Merge pull request #2842 from rust-lang/rustc-pull

Rustc pull update

compiler/rustc_attr_parsing/src/attributes/doc.rs

@@ -15,7 +15,8 @@
 use crate::context::{AcceptContext, FinalizeContext, Stage};
 use crate::errors::{
     DocAliasDuplicated, DocAutoCfgExpectsHideOrShow, DocAutoCfgHideShowExpectsList,
-    DocAutoCfgHideShowUnexpectedItem, DocUnknownInclude, IllFormedAttributeInput,
+    DocAutoCfgHideShowUnexpectedItem, DocAutoCfgWrongLiteral, DocUnknownAny, DocUnknownInclude,
+    DocUnknownPasses, DocUnknownPlugins, DocUnknownSpotlight, IllFormedAttributeInput,
 };
 use crate::parser::{ArgParser, MetaItemOrLitParser, MetaItemParser, OwnedPathParser};
 use crate::session_diagnostics::{
@@ -442,9 +443,9 @@ fn parse_auto_cfg<S: Stage>(
             ArgParser::NameValue(nv) => {
                 let MetaItemLit { kind: LitKind::Bool(bool_value), span, .. } = nv.value_as_lit()
                 else {
-                    cx.emit_lint(
+                    cx.emit_dyn_lint(
                         rustc_session::lint::builtin::INVALID_DOC_ATTRIBUTES,
-                        AttributeLintKind::DocAutoCfgWrongLiteral,
+                        move |dcx, level| DocAutoCfgWrongLiteral.into_diag(dcx, level),
                         nv.value_span,
                     );
                     return;
@@ -613,10 +614,11 @@ macro_rules! string_arg_and_crate_level {
                 }
             }
             Some(sym::spotlight) => {
-                cx.emit_lint(
+                let span = path.span();
+                cx.emit_dyn_lint(
                     rustc_session::lint::builtin::INVALID_DOC_ATTRIBUTES,
-                    AttributeLintKind::DocUnknownSpotlight { span: path.span() },
-                    path.span(),
+                    move |dcx, level| DocUnknownSpotlight { sugg_span: span }.into_diag(dcx, level),
+                    span,
                 );
             }
             Some(sym::include) if let Some(nv) = args.name_value() => {
@@ -640,32 +642,37 @@ macro_rules! string_arg_and_crate_level {
                 );
             }
             Some(name @ (sym::passes | sym::no_default_passes)) => {
-                cx.emit_lint(
+                let span = path.span();
+                cx.emit_dyn_lint(
                     rustc_session::lint::builtin::INVALID_DOC_ATTRIBUTES,
-                    AttributeLintKind::DocUnknownPasses { name, span: path.span() },
-                    path.span(),
+                    move |dcx, level| {
+                        DocUnknownPasses { name, note_span: span }.into_diag(dcx, level)
+                    },
+                    span,
                 );
             }
             Some(sym::plugins) => {
-                cx.emit_lint(
+                let span = path.span();
+                cx.emit_dyn_lint(
                     rustc_session::lint::builtin::INVALID_DOC_ATTRIBUTES,
-                    AttributeLintKind::DocUnknownPlugins { span: path.span() },
-                    path.span(),
+                    move |dcx, level| DocUnknownPlugins { label_span: span }.into_diag(dcx, level),
+                    span,
                 );
             }
             Some(name) => {
-                cx.emit_lint(
+                cx.emit_dyn_lint(
                     rustc_session::lint::builtin::INVALID_DOC_ATTRIBUTES,
-                    AttributeLintKind::DocUnknownAny { name },
+                    move |dcx, level| DocUnknownAny { name }.into_diag(dcx, level),
                     path.span(),
                 );
             }
             None => {
                 let full_name =
                     path.segments().map(|s| s.as_str()).intersperse("::").collect::<String>();
-                cx.emit_lint(
+                let name = Symbol::intern(&full_name);
+                cx.emit_dyn_lint(
                     rustc_session::lint::builtin::INVALID_DOC_ATTRIBUTES,
-                    AttributeLintKind::DocUnknownAny { name: Symbol::intern(&full_name) },
+                    move |dcx, level| DocUnknownAny { name }.into_diag(dcx, level),
                     path.span(),
                 );
             }

compiler/rustc_attr_parsing/src/errors.rs

@@ -205,3 +205,50 @@ pub(crate) struct DocUnknownInclude {
     )]
     pub sugg: (Span, Applicability),
 }
+
+#[derive(Diagnostic)]
+#[diag("unknown `doc` attribute `spotlight`")]
+#[note("`doc(spotlight)` was renamed to `doc(notable_trait)`")]
+#[note("`doc(spotlight)` is now a no-op")]
+pub(crate) struct DocUnknownSpotlight {
+    #[suggestion(
+        "use `notable_trait` instead",
+        style = "short",
+        applicability = "machine-applicable",
+        code = "notable_trait"
+    )]
+    pub sugg_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag("unknown `doc` attribute `{$name}`")]
+#[note(
+    "`doc` attribute `{$name}` no longer functions; see issue #44136 <https://github.com/rust-lang/rust/issues/44136>"
+)]
+#[note("`doc({$name})` is now a no-op")]
+pub(crate) struct DocUnknownPasses {
+    pub name: Symbol,
+    #[label("no longer functions")]
+    pub note_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag("unknown `doc` attribute `plugins`")]
+#[note(
+    "`doc` attribute `plugins` no longer functions; see issue #44136 <https://github.com/rust-lang/rust/issues/44136> and CVE-2018-1000622 <https://nvd.nist.gov/vuln/detail/CVE-2018-1000622>"
+)]
+#[note("`doc(plugins)` is now a no-op")]
+pub(crate) struct DocUnknownPlugins {
+    #[label("no longer functions")]
+    pub label_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag("unknown `doc` attribute `{$name}`")]
+pub(crate) struct DocUnknownAny {
+    pub name: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag("expected boolean for `#[doc(auto_cfg = ...)]`")]
+pub(crate) struct DocAutoCfgWrongLiteral;

compiler/rustc_borrowck/src/diagnostics/region_name.rs

@@ -499,18 +499,18 @@ fn give_name_if_anonymous_region_appears_in_arguments(
         fr: RegionVid,
     ) -> Option<RegionName> {
         let implicit_inputs = self.regioncx.universal_regions().defining_ty.implicit_inputs();
-        let argument_index = self.regioncx.get_argument_index_for_region(self.infcx.tcx, fr)?;
+        let user_arg_index = self.regioncx.get_user_arg_index_for_region(self.infcx.tcx, fr)?;
 
         let arg_ty = self.regioncx.universal_regions().unnormalized_input_tys
-            [implicit_inputs + argument_index];
+            [implicit_inputs + user_arg_index];
         let (_, span) = self.regioncx.get_argument_name_and_span_for_region(
             self.body,
             self.local_names(),
-            argument_index,
+            user_arg_index,
         );
 
         let highlight = self
-            .get_argument_hir_ty_for_highlighting(argument_index)
+            .get_argument_hir_ty_for_highlighting(user_arg_index)
             .and_then(|arg_hir_ty| self.highlight_if_we_can_match_hir_ty(fr, arg_ty, arg_hir_ty))
             .unwrap_or_else(|| {
                 // `highlight_if_we_cannot_match_hir_ty` needs to know the number we will give to
@@ -528,10 +528,11 @@ fn give_name_if_anonymous_region_appears_in_arguments(
 
     fn get_argument_hir_ty_for_highlighting(
         &self,
-        argument_index: usize,
+        user_arg_index: usize,
     ) -> Option<&hir::Ty<'tcx>> {
         let fn_decl = self.infcx.tcx.hir_fn_decl_by_hir_id(self.mir_hir_id())?;
-        let argument_hir_ty: &hir::Ty<'_> = fn_decl.inputs.get(argument_index)?;
+        // Closures don't have implicit self arguments in HIR, so use `user_arg_index` directly.
+        let argument_hir_ty: &hir::Ty<'_> = fn_decl.inputs.get(user_arg_index)?;
         match argument_hir_ty.kind {
             // This indicates a variable with no type annotation, like
             // `|x|`... in that case, we can't highlight the type but

compiler/rustc_borrowck/src/diagnostics/var_name.rs

@@ -31,7 +31,7 @@ pub(crate) fn get_var_name_and_span_for_region(
             })
             .or_else(|| {
                 debug!("get_var_name_and_span_for_region: attempting argument");
-                self.get_argument_index_for_region(tcx, fr).and_then(|index| {
+                self.get_user_arg_index_for_region(tcx, fr).and_then(|index| {
                     let local = self.user_arg_index_to_local(body, index);
                     if body_uses_local(body, local) {
                         Some(self.get_argument_name_and_span_for_region(body, local_names, index))
@@ -93,26 +93,26 @@ pub(crate) fn get_upvar_name_and_span_for_region(
     ///
     /// N.B., in the case of a closure, the index is indexing into the signature as seen by the
     /// user - in particular, index 0 is not the implicit self parameter.
-    pub(crate) fn get_argument_index_for_region(
+    pub(crate) fn get_user_arg_index_for_region(
         &self,
         tcx: TyCtxt<'tcx>,
         fr: RegionVid,
     ) -> Option<usize> {
         let implicit_inputs = self.universal_regions().defining_ty.implicit_inputs();
-        let argument_index =
+        let user_arg_index =
             self.universal_regions().unnormalized_input_tys.iter().skip(implicit_inputs).position(
                 |arg_ty| {
-                    debug!("get_argument_index_for_region: arg_ty = {arg_ty:?}");
+                    debug!("get_user_arg_index_for_region: arg_ty = {arg_ty:?}");
                     tcx.any_free_region_meets(arg_ty, |r| r.as_var() == fr)
                 },
             )?;
 
         debug!(
-            "get_argument_index_for_region: found {fr:?} in argument {argument_index} which has type {:?}",
-            self.universal_regions().unnormalized_input_tys[argument_index],
+            "get_user_arg_index_for_region: found {fr:?} in argument {user_arg_index} which has type {:?}",
+            self.universal_regions().unnormalized_input_tys[user_arg_index],
         );
 
-        Some(argument_index)
+        Some(user_arg_index)
     }
 
     /// Given the index of an argument as seen from the user (i.e. excluding
@@ -128,9 +128,9 @@ pub(crate) fn get_argument_name_and_span_for_region(
         &self,
         body: &Body<'tcx>,
         local_names: &IndexSlice<Local, Option<Symbol>>,
-        argument_index: usize,
+        user_arg_index: usize,
     ) -> (Option<Symbol>, Span) {
-        let argument_local = self.user_arg_index_to_local(body, argument_index);
+        let argument_local = self.user_arg_index_to_local(body, user_arg_index);
         debug!("get_argument_name_and_span_for_region: argument_local={argument_local:?}");
 
         let argument_name = local_names[argument_local];

compiler/rustc_borrowck/src/polonius/dump.rs

@@ -107,7 +107,7 @@ fn on_successor_discovered(&mut self, current_node: LocalizedNode, successor: Lo
 /// - a mermaid graph of the NLL regions and the constraints between them
 /// - a mermaid graph of the NLL SCCs and the constraints between them
 fn emit_polonius_dump<'tcx>(
-    dumper: &MirDumper<'_, '_, 'tcx>,
+    dumper: &MirDumper<'_, 'tcx>,
     body: &Body<'tcx>,
     regioncx: &RegionInferenceContext<'tcx>,
     borrow_set: &BorrowSet<'tcx>,
@@ -186,7 +186,7 @@ fn emit_polonius_dump<'tcx>(
 
 /// Emits the polonius MIR, as escaped HTML.
 fn emit_html_mir<'tcx>(
-    dumper: &MirDumper<'_, '_, 'tcx>,
+    dumper: &MirDumper<'_, 'tcx>,
     body: &Body<'tcx>,
     out: &mut dyn io::Write,
 ) -> io::Result<()> {

compiler/rustc_borrowck/src/universal_regions.rs

@@ -79,6 +79,9 @@ pub(crate) struct UniversalRegions<'tcx> {
     ///
     /// N.B., associated types in these types have not been normalized,
     /// as the name suggests. =)
+    ///
+    /// N.B., in the case of a closure, index 0 is the implicit self parameter,
+    /// and not the first input as seen by the user.
     pub unnormalized_input_tys: &'tcx [Ty<'tcx>],
 
     pub yield_ty: Option<Ty<'tcx>>,

compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs

@@ -448,14 +448,19 @@ pub(crate) fn gen_define_handling<'ll>(
         transfer.iter().map(|m| m.intersection(valid_begin_mappings).bits()).collect();
     let transfer_from: Vec<u64> =
         transfer.iter().map(|m| m.intersection(MappingFlags::FROM).bits()).collect();
+    let valid_kernel_mappings = MappingFlags::LITERAL | MappingFlags::IMPLICIT;
     // FIXME(offload): add `OMP_MAP_TARGET_PARAM = 0x20` only if necessary
-    let transfer_kernel = vec![MappingFlags::TARGET_PARAM.bits(); transfer_to.len()];
+    let transfer_kernel: Vec<u64> = transfer
+        .iter()
+        .map(|m| (m.intersection(valid_kernel_mappings) | MappingFlags::TARGET_PARAM).bits())
+        .collect();
 
     let actual_sizes = sizes
         .iter()
         .map(|s| match s {
             OffloadSize::Static(sz) => *sz,
-            OffloadSize::Dynamic => 0,
+            // NOTE(Sa4dUs): set `.offload_sizes` entry to 0 for sizes that we determine at runtime, just like clang
+            _ => 0,
         })
         .collect::<Vec<_>>();
     let offload_sizes =
@@ -542,12 +547,20 @@ pub(crate) fn scalar_width<'ll>(cx: &'ll SimpleCx<'_>, ty: &'ll Type) -> u64 {
 }
 
 fn get_runtime_size<'ll, 'tcx>(
-    _cx: &CodegenCx<'ll, 'tcx>,
-    _val: &'ll Value,
-    _meta: &OffloadMetadata,
+    builder: &mut Builder<'_, 'll, 'tcx>,
+    args: &[&'ll Value],
+    index: usize,
+    meta: &OffloadMetadata,
 ) -> &'ll Value {
-    // FIXME(Sa4dUs): handle dynamic-size data (e.g. slices)
-    bug!("offload does not support dynamic sizes yet");
+    match meta.payload_size {
+        OffloadSize::Slice { element_size } => {
+            let length_idx = index + 1;
+            let length = args[length_idx];
+            let length_i64 = builder.intcast(length, builder.cx.type_i64(), false);
+            builder.mul(length_i64, builder.cx.get_const_i64(element_size))
+        }
+        _ => bug!("unexpected offload size {:?}", meta.payload_size),
+    }
 }
 
 // For each kernel *call*, we now use some of our previous declared globals to move data to and from
@@ -588,7 +601,7 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>(
     let OffloadKernelDims { num_workgroups, threads_per_block, workgroup_dims, thread_dims } =
         offload_dims;
 
-    let has_dynamic = metadata.iter().any(|m| matches!(m.payload_size, OffloadSize::Dynamic));
+    let has_dynamic = metadata.iter().any(|m| !matches!(m.payload_size, OffloadSize::Static(_)));
 
     let tgt_decl = offload_globals.launcher_fn;
     let tgt_target_kernel_ty = offload_globals.launcher_ty;
@@ -683,9 +696,9 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>(
         let gep2 = builder.inbounds_gep(ty, a2, &[i32_0, idx]);
         builder.store(geps[i as usize], gep2, Align::EIGHT);
 
-        if matches!(metadata[i as usize].payload_size, OffloadSize::Dynamic) {
+        if !matches!(metadata[i as usize].payload_size, OffloadSize::Static(_)) {
             let gep3 = builder.inbounds_gep(ty2, a4, &[i32_0, idx]);
-            let size_val = get_runtime_size(cx, args[i as usize], &metadata[i as usize]);
+            let size_val = get_runtime_size(builder, args, i as usize, &metadata[i as usize]);
             builder.store(size_val, gep3, Align::EIGHT);
         }
     }

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -1813,9 +1813,20 @@ fn codegen_offload<'ll, 'tcx>(
     let sig = tcx.instantiate_bound_regions_with_erased(sig);
     let inputs = sig.inputs();
 
-    let metadata = inputs.iter().map(|ty| OffloadMetadata::from_ty(tcx, *ty)).collect::<Vec<_>>();
+    let fn_abi = cx.fn_abi_of_instance(fn_target, ty::List::empty());
 
-    let types = inputs.iter().map(|ty| cx.layout_of(*ty).llvm_type(cx)).collect::<Vec<_>>();
+    let mut metadata = Vec::new();
+    let mut types = Vec::new();
+
+    for (i, arg_abi) in fn_abi.args.iter().enumerate() {
+        let ty = inputs[i];
+        let decomposed = OffloadMetadata::handle_abi(cx, tcx, ty, arg_abi);
+
+        for (meta, entry_ty) in decomposed {
+            metadata.push(meta);
+            types.push(bx.cx.layout_of(entry_ty).llvm_type(bx.cx));
+        }
+    }
 
     let offload_globals_ref = cx.offload_globals.borrow();
     let offload_globals = match offload_globals_ref.as_ref() {

compiler/rustc_feature/src/unstable.rs

@@ -303,6 +303,8 @@ pub fn internal(&self, feature: Symbol) -> bool {
     (internal, panic_runtime, "1.10.0", Some(32837)),
     /// Allows using pattern types.
     (internal, pattern_types, "1.79.0", Some(123646)),
+    /// Allows `repr(simd)` and importing the various simd intrinsics.
+    (internal, repr_simd, "1.4.0", Some(27731)),
     /// Allows using compiler's own crates.
     (unstable, rustc_private, "1.0.0", Some(27812)),
     /// Allows using internal rustdoc features like `doc(keyword)`.
@@ -657,8 +659,6 @@ pub fn internal(&self, feature: Symbol) -> bool {
     (incomplete, ref_pat_eat_one_layer_2024_structural, "1.81.0", Some(123076)),
     /// Allows using the `#[register_tool]` attribute.
     (unstable, register_tool, "1.41.0", Some(66079)),
-    /// Allows `repr(simd)` and importing the various simd intrinsics.
-    (unstable, repr_simd, "1.4.0", Some(27731)),
     /// Allows bounding the return type of AFIT/RPITIT.
     (unstable, return_type_notation, "1.70.0", Some(109417)),
     /// Target features on riscv.

compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs

@@ -1901,14 +1901,14 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     }
 }
 
-struct FnCallDiagCtxt<'a, 'b, 'tcx> {
-    arg_matching_ctxt: ArgMatchingCtxt<'a, 'b, 'tcx>,
+struct FnCallDiagCtxt<'a, 'tcx> {
+    arg_matching_ctxt: ArgMatchingCtxt<'a, 'tcx>,
     errors: Vec<Error<'tcx>>,
     matched_inputs: IndexVec<ExpectedIdx, Option<ProvidedIdx>>,
 }
 
-impl<'a, 'b, 'tcx> Deref for FnCallDiagCtxt<'a, 'b, 'tcx> {
-    type Target = ArgMatchingCtxt<'a, 'b, 'tcx>;
+impl<'a, 'tcx> Deref for FnCallDiagCtxt<'a, 'tcx> {
+    type Target = ArgMatchingCtxt<'a, 'tcx>;
 
     fn deref(&self) -> &Self::Target {
         &self.arg_matching_ctxt
@@ -1921,9 +1921,9 @@ enum ArgumentsFormatting {
     Multiline { fallback_indent: String, brace_indent: String },
 }
 
-impl<'a, 'b, 'tcx> FnCallDiagCtxt<'a, 'b, 'tcx> {
+impl<'a, 'tcx> FnCallDiagCtxt<'a, 'tcx> {
     fn new(
-        arg: &'a FnCtxt<'b, 'tcx>,
+        arg: &'a FnCtxt<'a, 'tcx>,
         compatibility_diagonal: IndexVec<ProvidedIdx, Compatibility<'tcx>>,
         formal_and_expected_inputs: IndexVec<ExpectedIdx, (Ty<'tcx>, Ty<'tcx>)>,
         provided_args: IndexVec<ProvidedIdx, &'tcx Expr<'tcx>>,
@@ -2629,7 +2629,7 @@ fn has_error_or_infer<'tcx>(tys: impl IntoIterator<Item = Ty<'tcx>>) -> bool {
         (suggestions, labels, suggestion_text)
     }
 
-    fn label_generic_mismatches(&self, err: &mut Diag<'b>) {
+    fn label_generic_mismatches(&self, err: &mut Diag<'a>) {
         self.fn_ctxt.label_generic_mismatches(
             err,
             self.fn_def_id,
@@ -2805,22 +2805,22 @@ fn maybe_suggest_expect_for_unwrap(&self, provided_ty: Ty<'tcx>) -> Option<Span>
     }
 }
 
-struct ArgMatchingCtxt<'a, 'b, 'tcx> {
-    args_ctxt: ArgsCtxt<'a, 'b, 'tcx>,
+struct ArgMatchingCtxt<'a, 'tcx> {
+    args_ctxt: ArgsCtxt<'a, 'tcx>,
     provided_arg_tys: IndexVec<ProvidedIdx, (Ty<'tcx>, Span)>,
 }
 
-impl<'a, 'b, 'tcx> Deref for ArgMatchingCtxt<'a, 'b, 'tcx> {
-    type Target = ArgsCtxt<'a, 'b, 'tcx>;
+impl<'a, 'tcx> Deref for ArgMatchingCtxt<'a, 'tcx> {
+    type Target = ArgsCtxt<'a, 'tcx>;
 
     fn deref(&self) -> &Self::Target {
         &self.args_ctxt
     }
 }
 
-impl<'a, 'b, 'tcx> ArgMatchingCtxt<'a, 'b, 'tcx> {
+impl<'a, 'tcx> ArgMatchingCtxt<'a, 'tcx> {
     fn new(
-        arg: &'a FnCtxt<'b, 'tcx>,
+        arg: &'a FnCtxt<'a, 'tcx>,
         compatibility_diagonal: IndexVec<ProvidedIdx, Compatibility<'tcx>>,
         formal_and_expected_inputs: IndexVec<ExpectedIdx, (Ty<'tcx>, Ty<'tcx>)>,
         provided_args: IndexVec<ProvidedIdx, &'tcx Expr<'tcx>>,
@@ -2951,23 +2951,23 @@ fn remove_idx_is_perfect(&self, idx: usize) -> bool {
     }
 }
 
-struct ArgsCtxt<'a, 'b, 'tcx> {
-    call_ctxt: CallCtxt<'a, 'b, 'tcx>,
+struct ArgsCtxt<'a, 'tcx> {
+    call_ctxt: CallCtxt<'a, 'tcx>,
     call_metadata: CallMetadata,
     args_span: Span,
 }
 
-impl<'a, 'b, 'tcx> Deref for ArgsCtxt<'a, 'b, 'tcx> {
-    type Target = CallCtxt<'a, 'b, 'tcx>;
+impl<'a, 'tcx> Deref for ArgsCtxt<'a, 'tcx> {
+    type Target = CallCtxt<'a, 'tcx>;
 
     fn deref(&self) -> &Self::Target {
         &self.call_ctxt
     }
 }
 
-impl<'a, 'b, 'tcx> ArgsCtxt<'a, 'b, 'tcx> {
+impl<'a, 'tcx> ArgsCtxt<'a, 'tcx> {
     fn new(
-        arg: &'a FnCtxt<'b, 'tcx>,
+        arg: &'a FnCtxt<'a, 'tcx>,
         compatibility_diagonal: IndexVec<ProvidedIdx, Compatibility<'tcx>>,
         formal_and_expected_inputs: IndexVec<ExpectedIdx, (Ty<'tcx>, Ty<'tcx>)>,
         provided_args: IndexVec<ProvidedIdx, &'tcx Expr<'tcx>>,
@@ -2978,7 +2978,7 @@ fn new(
         call_expr: &'tcx Expr<'tcx>,
         tuple_arguments: TupleArgumentsFlag,
     ) -> Self {
-        let call_ctxt: CallCtxt<'_, '_, '_> = CallCtxt::new(
+        let call_ctxt: CallCtxt<'_, '_> = CallCtxt::new(
             arg,
             compatibility_diagonal,
             formal_and_expected_inputs,
@@ -3085,8 +3085,8 @@ struct CallMetadata {
     is_method: bool,
 }
 
-struct CallCtxt<'a, 'b, 'tcx> {
-    fn_ctxt: &'a FnCtxt<'b, 'tcx>,
+struct CallCtxt<'a, 'tcx> {
+    fn_ctxt: &'a FnCtxt<'a, 'tcx>,
     compatibility_diagonal: IndexVec<ProvidedIdx, Compatibility<'tcx>>,
     formal_and_expected_inputs: IndexVec<ExpectedIdx, (Ty<'tcx>, Ty<'tcx>)>,
     provided_args: IndexVec<ProvidedIdx, &'tcx hir::Expr<'tcx>>,
@@ -3100,17 +3100,17 @@ struct CallCtxt<'a, 'b, 'tcx> {
     callee_ty: Option<Ty<'tcx>>,
 }
 
-impl<'a, 'b, 'tcx> Deref for CallCtxt<'a, 'b, 'tcx> {
-    type Target = &'a FnCtxt<'b, 'tcx>;
+impl<'a, 'tcx> Deref for CallCtxt<'a, 'tcx> {
+    type Target = &'a FnCtxt<'a, 'tcx>;
 
     fn deref(&self) -> &Self::Target {
         &self.fn_ctxt
     }
 }
 
-impl<'a, 'b, 'tcx> CallCtxt<'a, 'b, 'tcx> {
+impl<'a, 'tcx> CallCtxt<'a, 'tcx> {
     fn new(
-        fn_ctxt: &'a FnCtxt<'b, 'tcx>,
+        fn_ctxt: &'a FnCtxt<'a, 'tcx>,
         compatibility_diagonal: IndexVec<ProvidedIdx, Compatibility<'tcx>>,
         formal_and_expected_inputs: IndexVec<ExpectedIdx, (Ty<'tcx>, Ty<'tcx>)>,
         provided_args: IndexVec<ProvidedIdx, &'tcx hir::Expr<'tcx>>,
@@ -3120,7 +3120,7 @@ fn new(
         call_span: Span,
         call_expr: &'tcx hir::Expr<'tcx>,
         tuple_arguments: TupleArgumentsFlag,
-    ) -> CallCtxt<'a, 'b, 'tcx> {
+    ) -> CallCtxt<'a, 'tcx> {
         let callee_expr = match &call_expr.peel_blocks().kind {
             hir::ExprKind::Call(callee, _) => Some(*callee),
             hir::ExprKind::MethodCall(_, receiver, ..) => {

compiler/rustc_hir_typeck/src/method/prelude_edition_lints.rs

@@ -16,15 +16,15 @@
 use crate::FnCtxt;
 use crate::method::probe::{self, Pick};
 
-struct AmbiguousTraitMethodCall<'a, 'b, 'tcx> {
+struct AmbiguousTraitMethodCall<'a, 'tcx> {
     segment_name: Symbol,
     self_expr_span: Span,
     pick: &'a Pick<'tcx>,
     tcx: TyCtxt<'tcx>,
-    edition: &'b str,
+    edition: &'static str,
 }
 
-impl<'a, 'b, 'c, 'tcx> Diagnostic<'a, ()> for AmbiguousTraitMethodCall<'b, 'c, 'tcx> {
+impl<'a, 'b, 'tcx> Diagnostic<'a, ()> for AmbiguousTraitMethodCall<'b, 'tcx> {
     fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, ()> {
         let Self { segment_name, self_expr_span, pick, tcx, edition } = self;
         let mut lint = Diag::new(
@@ -80,20 +80,18 @@ fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, ()> {
     }
 }
 
-struct AmbiguousTraitMethod<'a, 'b, 'tcx, 'pcx, 'fnctx> {
-    segment: &'a hir::PathSegment<'pcx>,
+struct AmbiguousTraitMethod<'a, 'tcx, 'fnctx> {
+    segment: &'a hir::PathSegment<'tcx>,
     call_expr: &'tcx hir::Expr<'tcx>,
     self_expr: &'tcx hir::Expr<'tcx>,
     pick: &'a Pick<'tcx>,
     args: &'tcx [hir::Expr<'tcx>],
-    edition: &'b str,
+    edition: &'static str,
     span: Span,
     this: &'a FnCtxt<'fnctx, 'tcx>,
 }
 
-impl<'a, 'b, 'c, 'tcx, 'pcx, 'fnctx> Diagnostic<'a, ()>
-    for AmbiguousTraitMethod<'b, 'c, 'tcx, 'pcx, 'fnctx>
-{
+impl<'a, 'c, 'tcx, 'fnctx> Diagnostic<'a, ()> for AmbiguousTraitMethod<'c, 'tcx, 'fnctx> {
     fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, ()> {
         let Self { segment, call_expr, self_expr, pick, args, edition, span, this } = self;
         let mut lint = Diag::new(
@@ -158,7 +156,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub(super) fn lint_edition_dependent_dot_call(
         &self,
         self_ty: Ty<'tcx>,
-        segment: &hir::PathSegment<'_>,
+        segment: &hir::PathSegment<'tcx>,
         span: Span,
         call_expr: &'tcx hir::Expr<'tcx>,
         self_expr: &'tcx hir::Expr<'tcx>,

compiler/rustc_hir_typeck/src/upvar.rs

@@ -958,15 +958,15 @@ fn perform_2229_migration_analysis(
         capture_clause: hir::CaptureBy,
         span: Span,
     ) {
-        struct MigrationLint<'a, 'b, 'tcx> {
+        struct MigrationLint<'a, 'tcx> {
             closure_def_id: LocalDefId,
-            this: &'a FnCtxt<'b, 'tcx>,
+            this: &'a FnCtxt<'a, 'tcx>,
             body_id: hir::BodyId,
             need_migrations: Vec<NeededMigration>,
             migration_message: String,
         }
 
-        impl<'a, 'b, 'c, 'tcx> Diagnostic<'a, ()> for MigrationLint<'b, 'c, 'tcx> {
+        impl<'a, 'b, 'tcx> Diagnostic<'a, ()> for MigrationLint<'b, 'tcx> {
             fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, ()> {
                 let Self { closure_def_id, this, body_id, need_migrations, migration_message } =
                     self;
@@ -2084,8 +2084,8 @@ fn drop_location_span(tcx: TyCtxt<'_>, hir_id: HirId) -> Span {
     tcx.sess.source_map().end_point(owner_span)
 }
 
-struct InferBorrowKind<'fcx, 'a, 'tcx> {
-    fcx: &'fcx FnCtxt<'a, 'tcx>,
+struct InferBorrowKind<'a, 'tcx> {
+    fcx: &'a FnCtxt<'a, 'tcx>,
     // The def-id of the closure whose kind and upvar accesses are being inferred.
     closure_def_id: LocalDefId,
 
@@ -2119,7 +2119,7 @@ struct InferBorrowKind<'fcx, 'a, 'tcx> {
     fake_reads: Vec<(Place<'tcx>, FakeReadCause, HirId)>,
 }
 
-impl<'fcx, 'a, 'tcx> euv::Delegate<'tcx> for InferBorrowKind<'fcx, 'a, 'tcx> {
+impl<'a, 'tcx> euv::Delegate<'tcx> for InferBorrowKind<'a, 'tcx> {
     #[instrument(skip(self), level = "debug")]
     fn fake_read(
         &mut self,

compiler/rustc_lint/src/default_could_be_derived.rs

@@ -156,15 +156,15 @@ fn check_impl_item(&mut self, cx: &LateContext<'_>, impl_item: &hir::ImplItem<'_
     }
 }
 
-struct WrongDefaultImpl<'a, 'hir, 'tcx> {
+struct WrongDefaultImpl<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     type_def_id: DefId,
-    orig_fields: FxHashMap<Symbol, &'a hir::FieldDef<'hir>>,
-    fields: &'a [hir::ExprField<'hir>],
+    orig_fields: FxHashMap<Symbol, &'a hir::FieldDef<'tcx>>,
+    fields: &'a [hir::ExprField<'tcx>],
     impl_span: Span,
 }
 
-impl<'a, 'b, 'hir, 'tcx> Diagnostic<'a, ()> for WrongDefaultImpl<'b, 'hir, 'tcx> {
+impl<'a, 'b, 'tcx> Diagnostic<'a, ()> for WrongDefaultImpl<'b, 'tcx> {
     fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, ()> {
         let Self { tcx, type_def_id, orig_fields, fields, impl_span } = self;
         let mut diag =

compiler/rustc_lint/src/early/diagnostics.rs

@@ -43,26 +43,6 @@ fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, ()> {
                     .into_diag(dcx, level)
             }
 
-            &AttributeLintKind::DocUnknownSpotlight { span } => {
-                lints::DocUnknownSpotlight { sugg_span: span }.into_diag(dcx, level)
-            }
-
-            &AttributeLintKind::DocUnknownPasses { name, span } => {
-                lints::DocUnknownPasses { name, note_span: span }.into_diag(dcx, level)
-            }
-
-            &AttributeLintKind::DocUnknownPlugins { span } => {
-                lints::DocUnknownPlugins { label_span: span }.into_diag(dcx, level)
-            }
-
-            &AttributeLintKind::DocUnknownAny { name } => {
-                lints::DocUnknownAny { name }.into_diag(dcx, level)
-            }
-
-            &AttributeLintKind::DocAutoCfgWrongLiteral => {
-                lints::DocAutoCfgWrongLiteral.into_diag(dcx, level)
-            }
-
             &AttributeLintKind::DocTestTakesList => lints::DocTestTakesList.into_diag(dcx, level),
 
             &AttributeLintKind::DocTestUnknown { name } => {

compiler/rustc_lint/src/lints.rs

@@ -3303,53 +3303,6 @@ fn add_to_diag<G: EmissionGuarantee>(self, diag: &mut Diag<'_, G>) {
 )]
 pub(crate) struct ExpectedNameValue;
 
-#[derive(Diagnostic)]
-#[diag("unknown `doc` attribute `spotlight`")]
-#[note("`doc(spotlight)` was renamed to `doc(notable_trait)`")]
-#[note("`doc(spotlight)` is now a no-op")]
-pub(crate) struct DocUnknownSpotlight {
-    #[suggestion(
-        "use `notable_trait` instead",
-        style = "short",
-        applicability = "machine-applicable",
-        code = "notable_trait"
-    )]
-    pub sugg_span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag("unknown `doc` attribute `{$name}`")]
-#[note(
-    "`doc` attribute `{$name}` no longer functions; see issue #44136 <https://github.com/rust-lang/rust/issues/44136>"
-)]
-#[note("`doc({$name})` is now a no-op")]
-pub(crate) struct DocUnknownPasses {
-    pub name: Symbol,
-    #[label("no longer functions")]
-    pub note_span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag("unknown `doc` attribute `plugins`")]
-#[note(
-    "`doc` attribute `plugins` no longer functions; see issue #44136 <https://github.com/rust-lang/rust/issues/44136> and CVE-2018-1000622 <https://nvd.nist.gov/vuln/detail/CVE-2018-1000622>"
-)]
-#[note("`doc(plugins)` is now a no-op")]
-pub(crate) struct DocUnknownPlugins {
-    #[label("no longer functions")]
-    pub label_span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag("unknown `doc` attribute `{$name}`")]
-pub(crate) struct DocUnknownAny {
-    pub name: Symbol,
-}
-
-#[derive(Diagnostic)]
-#[diag("expected boolean for `#[doc(auto_cfg = ...)]`")]
-pub(crate) struct DocAutoCfgWrongLiteral;
-
 #[derive(Diagnostic)]
 #[diag("`#[doc(test(...)]` takes a list of attributes")]
 pub(crate) struct DocTestTakesList;

compiler/rustc_lint/src/nonstandard_style.rs

@@ -51,31 +51,27 @@
 /// be upper cased or lower cased. For the purposes of the lint suggestion, we care about being able
 /// to change the char's case.
 fn char_has_case(c: char) -> bool {
-    let mut l = c.to_lowercase();
-    let mut u = c.to_uppercase();
-    while let Some(l) = l.next() {
-        match u.next() {
-            Some(u) if l != u => return true,
-            _ => {}
-        }
-    }
-    u.next().is_some()
+    !c.to_lowercase().eq(c.to_uppercase())
+}
+
+// contains a capitalisable character followed by, or preceded by, an underscore
+fn has_underscore_case(s: &str) -> bool {
+    let mut last = '\0';
+    s.chars().any(|c| match (std::mem::replace(&mut last, c), c) {
+        ('_', cs) | (cs, '_') => char_has_case(cs),
+        _ => false,
+    })
 }
 
 fn is_camel_case(name: &str) -> bool {
     let name = name.trim_matches('_');
-    if name.is_empty() {
+    let Some(first) = name.chars().next() else {
         return true;
-    }
+    };
 
-    // start with a non-lowercase letter rather than non-uppercase
+    // start with a non-lowercase letter rather than uppercase
     // ones (some scripts don't have a concept of upper/lowercase)
-    !name.chars().next().unwrap().is_lowercase()
-        && !name.contains("__")
-        && !name.chars().collect::<Vec<_>>().array_windows().any(|&[fst, snd]| {
-            // contains a capitalisable character followed by, or preceded by, an underscore
-            char_has_case(fst) && snd == '_' || char_has_case(snd) && fst == '_'
-        })
+    !(first.is_lowercase() || name.contains("__") || has_underscore_case(name))
 }
 
 fn to_camel_case(s: &str) -> String {

compiler/rustc_lint_defs/src/builtin.rs

@@ -109,6 +109,7 @@
         SHADOWING_SUPERTRAIT_ITEMS,
         SINGLE_USE_LIFETIMES,
         STABLE_FEATURES,
+        TAIL_CALL_TRACK_CALLER,
         TAIL_EXPR_DROP_ORDER,
         TEST_UNSTABLE_LINT,
         TEXT_DIRECTION_CODEPOINT_IN_COMMENT,

compiler/rustc_lint_defs/src/lib.rs

@@ -656,11 +656,6 @@ pub enum DeprecatedSinceKind {
 pub enum AttributeLintKind {
     UnexpectedCfgName((Symbol, Span), Option<(Symbol, Span)>),
     UnexpectedCfgValue((Symbol, Span), Option<(Symbol, Span)>),
-    DocUnknownSpotlight { span: Span },
-    DocUnknownPasses { name: Symbol, span: Span },
-    DocUnknownPlugins { span: Span },
-    DocUnknownAny { name: Symbol },
-    DocAutoCfgWrongLiteral,
     DocTestTakesList,
     DocTestUnknown { name: Symbol },
     DocTestLiteral,

compiler/rustc_middle/src/ich/impls_syntax.rs

@@ -3,9 +3,6 @@
 
 use rustc_ast as ast;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_hir as hir;
-use rustc_span::{Symbol, sym};
-use smallvec::SmallVec;
 
 use super::StableHashingContext;
 
@@ -16,44 +13,6 @@ fn hash_stable(&self, _: &mut StableHashingContext<'a>, _: &mut StableHasher) {
     }
 }
 
-impl<'a> HashStable<StableHashingContext<'a>> for [hir::Attribute] {
-    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        if self.is_empty() {
-            self.len().hash_stable(hcx, hasher);
-            return;
-        }
-
-        // Some attributes are always ignored during hashing.
-        let filtered: SmallVec<[&hir::Attribute; 8]> = self
-            .iter()
-            .filter(|attr| {
-                attr.is_doc_comment().is_none()
-                    // FIXME(jdonszelmann) have a better way to handle ignored attrs
-                    && !attr.name().is_some_and(|ident| is_ignored_attr(ident))
-            })
-            .collect();
-
-        filtered.len().hash_stable(hcx, hasher);
-        for attr in filtered {
-            attr.hash_stable(hcx, hasher);
-        }
-    }
-}
-
-#[inline]
-fn is_ignored_attr(name: Symbol) -> bool {
-    const IGNORED_ATTRIBUTES: &[Symbol] = &[
-        sym::cfg_trace, // FIXME(#138844) should this really be ignored?
-        sym::rustc_if_this_changed,
-        sym::rustc_then_this_would_need,
-        sym::rustc_clean,
-        sym::rustc_partition_reused,
-        sym::rustc_partition_codegened,
-        sym::rustc_expected_cgu_reuse,
-    ];
-    IGNORED_ATTRIBUTES.contains(&name)
-}
-
 impl<'tcx> HashStable<StableHashingContext<'tcx>> for rustc_feature::Features {
     fn hash_stable(&self, hcx: &mut StableHashingContext<'tcx>, hasher: &mut StableHasher) {
         // Unfortunately we cannot exhaustively list fields here, since the

compiler/rustc_middle/src/mir/pretty.rs

@@ -61,14 +61,14 @@ pub fn from_cli(tcx: TyCtxt<'_>) -> Self {
 /// Manages MIR dumping, which is MIR writing done to a file with a specific name. In particular,
 /// it makes it impossible to dump MIR to one of these files when it hasn't been requested from the
 /// command line. Layered on top of `MirWriter`, which does the actual writing.
-pub struct MirDumper<'dis, 'de, 'tcx> {
+pub struct MirDumper<'a, 'tcx> {
     show_pass_num: bool,
     pass_name: &'static str,
-    disambiguator: &'dis dyn Display,
-    writer: MirWriter<'de, 'tcx>,
+    disambiguator: &'a dyn Display,
+    writer: MirWriter<'a, 'tcx>,
 }
 
-impl<'dis, 'de, 'tcx> MirDumper<'dis, 'de, 'tcx> {
+impl<'a, 'tcx> MirDumper<'a, 'tcx> {
     // If dumping should be performed (e.g. because it was requested on the
     // CLI), returns a `MirDumper` with default values for the following fields:
     // - `show_pass_num`: `false`
@@ -112,7 +112,7 @@ pub fn set_show_pass_num(mut self) -> Self {
     }
 
     #[must_use]
-    pub fn set_disambiguator(mut self, disambiguator: &'dis dyn Display) -> Self {
+    pub fn set_disambiguator(mut self, disambiguator: &'a dyn Display) -> Self {
         self.disambiguator = disambiguator;
         self
     }
@@ -120,7 +120,7 @@ pub fn set_disambiguator(mut self, disambiguator: &'dis dyn Display) -> Self {
     #[must_use]
     pub fn set_extra_data(
         mut self,
-        extra_data: &'de dyn Fn(PassWhere, &mut dyn io::Write) -> io::Result<()>,
+        extra_data: &'a dyn Fn(PassWhere, &mut dyn io::Write) -> io::Result<()>,
     ) -> Self {
         self.writer.extra_data = extra_data;
         self
@@ -369,13 +369,13 @@ pub fn write_mir_pretty<'tcx>(
 }
 
 /// Does the writing of MIR to output, e.g. a file.
-pub struct MirWriter<'de, 'tcx> {
+pub struct MirWriter<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    extra_data: &'de dyn Fn(PassWhere, &mut dyn io::Write) -> io::Result<()>,
+    extra_data: &'a dyn Fn(PassWhere, &mut dyn io::Write) -> io::Result<()>,
     options: PrettyPrintMirOptions,
 }
 
-impl<'de, 'tcx> MirWriter<'de, 'tcx> {
+impl<'a, 'tcx> MirWriter<'a, 'tcx> {
     pub fn new(tcx: TyCtxt<'tcx>) -> Self {
         MirWriter { tcx, extra_data: &|_, _| Ok(()), options: PrettyPrintMirOptions::from_cli(tcx) }
     }
@@ -709,7 +709,7 @@ pub fn dump_mir_def_ids(tcx: TyCtxt<'_>, single: Option<DefId>) -> Vec<DefId> {
 ///////////////////////////////////////////////////////////////////////////
 // Basic blocks and their parts (statements, terminators, ...)
 
-impl<'de, 'tcx> MirWriter<'de, 'tcx> {
+impl<'a, 'tcx> MirWriter<'a, 'tcx> {
     /// Write out a human-readable textual representation for the given basic block.
     fn write_basic_block(
         &self,

compiler/rustc_middle/src/ty/context.rs

@@ -700,6 +700,7 @@ pub fn feed_anon_const_type(self, key: LocalDefId, value: ty::EarlyBinder<'tcx,
 
     /// Feeds the HIR delayed owner during AST -> HIR delayed lowering.
     pub fn feed_delayed_owner(self, key: LocalDefId, owner: MaybeOwner<'tcx>) {
+        self.dep_graph.assert_ignored();
         TyCtxtFeed { tcx: self, key }.delayed_owner(owner);
     }
 }

compiler/rustc_middle/src/ty/offload_meta.rs

@@ -1,7 +1,10 @@
 use bitflags::bitflags;
+use rustc_abi::{BackendRepr, TyAbiInterface};
+use rustc_target::callconv::ArgAbi;
 
 use crate::ty::{self, PseudoCanonicalInput, Ty, TyCtxt, TypingEnv};
 
+#[derive(Debug, Copy, Clone)]
 pub struct OffloadMetadata {
     pub payload_size: OffloadSize,
     pub mode: MappingFlags,
@@ -9,13 +12,13 @@ pub struct OffloadMetadata {
 
 #[derive(Debug, Copy, Clone)]
 pub enum OffloadSize {
-    Dynamic,
     Static(u64),
+    Slice { element_size: u64 },
 }
 
 bitflags! {
     /// Mirrors `OpenMPOffloadMappingFlags` from Clang/OpenMP.
-    #[derive(Debug, Copy, Clone)]
+    #[derive(Debug, Copy, Clone, PartialEq, Eq)]
     #[repr(transparent)]
     pub struct MappingFlags: u64 {
         /// No flags.
@@ -62,11 +65,38 @@ pub fn from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Self {
             mode: MappingFlags::from_ty(tcx, ty),
         }
     }
+
+    pub fn handle_abi<'tcx, C>(
+        cx: &C,
+        tcx: TyCtxt<'tcx>,
+        ty: Ty<'tcx>,
+        arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
+    ) -> Vec<(Self, Ty<'tcx>)>
+    where
+        Ty<'tcx>: TyAbiInterface<'tcx, C>,
+    {
+        match arg_abi.layout.backend_repr {
+            BackendRepr::ScalarPair(_, _) => (0..2)
+                .map(|i| {
+                    let ty = arg_abi.layout.field(cx, i).ty;
+                    (OffloadMetadata::from_ty(tcx, ty), ty)
+                })
+                .collect(),
+            _ => vec![(OffloadMetadata::from_ty(tcx, ty), ty)],
+        }
+    }
 }
 
 // FIXME(Sa4dUs): implement a solid logic to determine the payload size
 fn get_payload_size<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> OffloadSize {
     match ty.kind() {
+        ty::Slice(elem_ty) => {
+            let layout = tcx.layout_of(PseudoCanonicalInput {
+                typing_env: TypingEnv::fully_monomorphized(),
+                value: *elem_ty,
+            });
+            OffloadSize::Slice { element_size: layout.unwrap().size.bytes() }
+        }
         ty::RawPtr(inner, _) | ty::Ref(_, inner, _) => get_payload_size(tcx, *inner),
         _ => OffloadSize::Static(
             tcx.layout_of(PseudoCanonicalInput {

compiler/rustc_mir_build/src/check_tail_calls.rs

@@ -4,12 +4,13 @@
 use rustc_hir::LangItem;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::CRATE_DEF_ID;
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::span_bug;
 use rustc_middle::thir::visit::{self, Visitor};
 use rustc_middle::thir::{BodyTy, Expr, ExprId, ExprKind, Thir};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::def_id::{DefId, LocalDefId};
-use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span};
+use rustc_span::{ErrorGuaranteed, Span};
 
 pub(crate) fn check_tail_calls(tcx: TyCtxt<'_>, def: LocalDefId) -> Result<(), ErrorGuaranteed> {
     let (thir, expr) = tcx.thir_body(def)?;
@@ -21,7 +22,6 @@ pub(crate) fn check_tail_calls(tcx: TyCtxt<'_>, def: LocalDefId) -> Result<(), E
     }
 
     let is_closure = matches!(tcx.def_kind(def), DefKind::Closure);
-    let caller_ty = tcx.type_of(def).skip_binder();
 
     let mut visitor = TailCallCkVisitor {
         tcx,
@@ -30,7 +30,7 @@ pub(crate) fn check_tail_calls(tcx: TyCtxt<'_>, def: LocalDefId) -> Result<(), E
         // FIXME(#132279): we're clearly in a body here.
         typing_env: ty::TypingEnv::non_body_analysis(tcx, def),
         is_closure,
-        caller_ty,
+        caller_def_id: def,
     };
 
     visitor.visit_expr(&thir[expr]);
@@ -47,8 +47,8 @@ struct TailCallCkVisitor<'a, 'tcx> {
     /// The result of the checks, `Err(_)` if there was a problem with some
     /// tail call, `Ok(())` if all of them were fine.
     found_errors: Result<(), ErrorGuaranteed>,
-    /// Type of the caller function.
-    caller_ty: Ty<'tcx>,
+    /// `LocalDefId` of the caller function.
+    caller_def_id: LocalDefId,
 }
 
 impl<'tcx> TailCallCkVisitor<'_, 'tcx> {
@@ -148,11 +148,13 @@ fn check_tail_call(&mut self, call: &Expr<'_>, expr: &Expr<'_>) {
         // we should think what is the expected behavior here.
         // (we should probably just accept this by revealing opaques?)
         if caller_sig.inputs_and_output != callee_sig.inputs_and_output {
+            let caller_ty = self.tcx.type_of(self.caller_def_id).skip_binder();
+
             self.report_signature_mismatch(
                 expr.span,
                 self.tcx.liberate_late_bound_regions(
                     CRATE_DEF_ID.to_def_id(),
-                    self.caller_ty.fn_sig(self.tcx),
+                    caller_ty.fn_sig(self.tcx),
                 ),
                 self.tcx.liberate_late_bound_regions(CRATE_DEF_ID.to_def_id(), ty.fn_sig(self.tcx)),
             );
@@ -173,7 +175,7 @@ fn check_tail_call(&mut self, call: &Expr<'_>, expr: &Expr<'_>) {
             // coercing the function to an `fn()` pointer. (although in that case the tailcall is
             // basically useless -- the shim calls the actual function, so tailcalling the shim is
             // equivalent to calling the function)
-            let caller_needs_location = self.needs_location(self.caller_ty);
+            let caller_needs_location = self.caller_needs_location();
 
             if caller_needs_location {
                 self.report_track_caller_caller(expr.span);
@@ -189,19 +191,11 @@ fn check_tail_call(&mut self, call: &Expr<'_>, expr: &Expr<'_>) {
         }
     }
 
-    /// Returns true if function of type `ty` needs location argument
-    /// (i.e. if a function is marked as `#[track_caller]`).
-    ///
-    /// Panics if the function's instance can't be immediately resolved.
-    fn needs_location(&self, ty: Ty<'tcx>) -> bool {
-        if let &ty::FnDef(did, substs) = ty.kind() {
-            let instance =
-                ty::Instance::expect_resolve(self.tcx, self.typing_env, did, substs, DUMMY_SP);
-
-            instance.def.requires_caller_location(self.tcx)
-        } else {
-            false
-        }
+    /// Returns true if the caller function needs a location argument
+    /// (i.e. if a function is marked as `#[track_caller]`)
+    fn caller_needs_location(&self) -> bool {
+        let flags = self.tcx.codegen_fn_attrs(self.caller_def_id).flags;
+        flags.contains(CodegenFnAttrFlags::TRACK_CALLER)
     }
 
     fn report_in_closure(&mut self, expr: &Expr<'_>) {

compiler/rustc_mir_build/src/errors.rs

@@ -649,14 +649,14 @@ pub(crate) enum UnusedUnsafeEnclosing {
     },
 }
 
-pub(crate) struct NonExhaustivePatternsTypeNotEmpty<'p, 'tcx, 'm> {
-    pub(crate) cx: &'m RustcPatCtxt<'p, 'tcx>,
+pub(crate) struct NonExhaustivePatternsTypeNotEmpty<'a, 'tcx> {
+    pub(crate) cx: &'a RustcPatCtxt<'a, 'tcx>,
     pub(crate) scrut_span: Span,
     pub(crate) braces_span: Option<Span>,
     pub(crate) ty: Ty<'tcx>,
 }
 
-impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for NonExhaustivePatternsTypeNotEmpty<'_, '_, '_> {
+impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for NonExhaustivePatternsTypeNotEmpty<'_, '_> {
     fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, G> {
         let mut diag =
             Diag::new(dcx, level, msg!("non-exhaustive patterns: type `{$ty}` is non-empty"));

compiler/rustc_monomorphize/src/graph_checks/statics.rs

@@ -37,14 +37,14 @@ struct StaticSccIdx {}
 // Adjacency-list graph for statics using `StaticNodeIdx` as node type.
 // We cannot use `DefId` as the node type directly because each node must be
 // represented by an index in the range `0..num_nodes`.
-struct StaticRefGraph<'a, 'b, 'tcx> {
+struct StaticRefGraph<'a, 'tcx> {
     // maps from `StaticNodeIdx` to `DefId` and vice versa
     statics: &'a FxIndexSet<DefId>,
     // contains for each `MonoItem` the `MonoItem`s it uses
-    used_map: &'b UnordMap<MonoItem<'tcx>, Vec<MonoItem<'tcx>>>,
+    used_map: &'a UnordMap<MonoItem<'tcx>, Vec<MonoItem<'tcx>>>,
 }
 
-impl<'a, 'b, 'tcx> DirectedGraph for StaticRefGraph<'a, 'b, 'tcx> {
+impl<'a, 'tcx> DirectedGraph for StaticRefGraph<'a, 'tcx> {
     type Node = StaticNodeIdx;
 
     fn num_nodes(&self) -> usize {
@@ -52,7 +52,7 @@ fn num_nodes(&self) -> usize {
     }
 }
 
-impl<'a, 'b, 'tcx> Successors for StaticRefGraph<'a, 'b, 'tcx> {
+impl<'a, 'tcx> Successors for StaticRefGraph<'a, 'tcx> {
     fn successors(&self, node_idx: StaticNodeIdx) -> impl Iterator<Item = StaticNodeIdx> {
         let def_id = self.statics[node_idx.index()];
         self.used_map[&MonoItem::Static(def_id)].iter().filter_map(|&mono_item| match mono_item {

compiler/rustc_parse/src/parser/expr.rs

@@ -3464,7 +3464,9 @@ pub(super) fn parse_arm(&mut self) -> PResult<'a, Arm> {
     }
 
     pub(crate) fn eat_metavar_guard(&mut self) -> Option<Box<Guard>> {
-        self.eat_metavar_seq(MetaVarKind::Guard, |this| this.parse_match_arm_guard()).flatten()
+        self.eat_metavar_seq(MetaVarKind::Guard, |this| {
+            this.expect_match_arm_guard(ForceCollect::Yes)
+        })
     }
 
     fn parse_match_arm_guard(&mut self) -> PResult<'a, Option<Box<Guard>>> {

compiler/rustc_resolve/src/build_reduced_graph.rs

@@ -12,10 +12,8 @@
     self as ast, AssocItem, AssocItemKind, Block, ConstItem, Delegation, Fn, ForeignItem,
     ForeignItemKind, Inline, Item, ItemKind, NodeId, StaticItem, StmtKind, TraitAlias, TyAlias,
 };
-use rustc_attr_parsing as attr;
 use rustc_attr_parsing::AttributeParser;
 use rustc_expand::base::ResolverExpand;
-use rustc_expand::expand::AstFragment;
 use rustc_hir::Attribute;
 use rustc_hir::attrs::{AttributeKind, MacroUseArgs};
 use rustc_hir::def::{self, *};
@@ -31,7 +29,7 @@
 use tracing::debug;
 
 use crate::Namespace::{MacroNS, TypeNS, ValueNS};
-use crate::def_collector::collect_definitions;
+use crate::def_collector::DefCollector;
 use crate::diagnostics::StructCtor;
 use crate::imports::{ImportData, ImportKind, OnUnknownData};
 use crate::macros::{MacroRulesDecl, MacroRulesScope, MacroRulesScopeRef};
@@ -237,30 +235,18 @@ pub(crate) fn register_macros_for_all_crates(&mut self) {
         }
     }
 
-    pub(crate) fn build_reduced_graph(
-        &mut self,
-        fragment: &AstFragment,
-        parent_scope: ParentScope<'ra>,
-    ) -> MacroRulesScopeRef<'ra> {
-        collect_definitions(self, fragment, parent_scope.expansion);
-        let mut visitor = BuildReducedGraphVisitor { r: self, parent_scope };
-        fragment.visit_with(&mut visitor);
-        visitor.parent_scope.macro_rules
-    }
-
     pub(crate) fn build_reduced_graph_external(&self, module: ExternModule<'ra>) {
         let def_id = module.def_id();
         let children = self.tcx.module_children(def_id);
-        let parent_scope = ParentScope::module(module.to_module(), self.arenas);
         for (i, child) in children.iter().enumerate() {
-            self.build_reduced_graph_for_external_crate_res(child, parent_scope, i, None)
+            self.build_reduced_graph_for_external_crate_res(child, module, i, None)
         }
         for (i, child) in
             self.cstore().ambig_module_children_untracked(self.tcx, def_id).enumerate()
         {
             self.build_reduced_graph_for_external_crate_res(
                 &child.main,
-                parent_scope,
+                module,
                 children.len() + i,
                 Some(&child.second),
             )
@@ -271,11 +257,10 @@ pub(crate) fn build_reduced_graph_external(&self, module: ExternModule<'ra>) {
     fn build_reduced_graph_for_external_crate_res(
         &self,
         child: &ModChild,
-        parent_scope: ParentScope<'ra>,
+        parent: ExternModule<'ra>,
         child_index: usize,
         ambig_child: Option<&ModChild>,
     ) {
-        let parent = parent_scope.module.expect_extern();
         let child_span = |this: &Self, reexport_chain: &[Reexport], res: def::Res<_>| {
             this.def_span(
                 reexport_chain
@@ -288,7 +273,7 @@ fn build_reduced_graph_for_external_crate_res(
         let ident = IdentKey::new(orig_ident);
         let span = child_span(self, reexport_chain, res);
         let res = res.expect_non_local();
-        let expansion = parent_scope.expansion;
+        let expansion = LocalExpnId::ROOT;
         let ambig = ambig_child.map(|ambig_child| {
             let ModChild { ident: _, res, vis, ref reexport_chain } = *ambig_child;
             let span = child_span(self, reexport_chain, res);
@@ -364,18 +349,13 @@ fn build_reduced_graph_for_external_crate_res(
     }
 }
 
-struct BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
-    r: &'a mut Resolver<'ra, 'tcx>,
-    parent_scope: ParentScope<'ra>,
-}
-
-impl<'ra, 'tcx> AsMut<Resolver<'ra, 'tcx>> for BuildReducedGraphVisitor<'_, 'ra, 'tcx> {
+impl<'ra, 'tcx> AsMut<Resolver<'ra, 'tcx>> for DefCollector<'_, 'ra, 'tcx> {
     fn as_mut(&mut self) -> &mut Resolver<'ra, 'tcx> {
         self.r
     }
 }
 
-impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
+impl<'a, 'ra, 'tcx> DefCollector<'a, 'ra, 'tcx> {
     fn res(&self, def_id: impl Into<DefId>) -> Res {
         let def_id = def_id.into();
         Res::Def(self.r.tcx.def_kind(def_id), def_id)
@@ -755,6 +735,7 @@ fn build_reduced_graph_for_use_tree(
             }
             ast::UseTreeKind::Nested { ref items, .. } => {
                 for &(ref tree, id) in items {
+                    self.create_def(id, None, DefKind::Use, use_tree.span());
                     self.build_reduced_graph_for_use_tree(
                         // This particular use tree
                         tree, id, &prefix, true, false, // The whole `use` item
@@ -1077,7 +1058,11 @@ fn build_reduced_graph_for_extern_crate(
     }
 
     /// Constructs the reduced graph for one foreign item.
-    fn build_reduced_graph_for_foreign_item(&mut self, item: &ForeignItem, ident: Ident) {
+    pub(crate) fn build_reduced_graph_for_foreign_item(
+        &mut self,
+        item: &ForeignItem,
+        ident: Ident,
+    ) {
         let feed = self.r.feed(item.id);
         let local_def_id = feed.key();
         let def_id = local_def_id.to_def_id();
@@ -1209,7 +1194,7 @@ fn process_macro_use_imports(&mut self, item: &Item, module: Module<'ra>) -> boo
     }
 
     /// Returns `true` if this attribute list contains `macro_use`.
-    fn contains_macro_use(&self, attrs: &[ast::Attribute]) -> bool {
+    pub(crate) fn contains_macro_use(&self, attrs: &[ast::Attribute]) -> bool {
         for attr in attrs {
             if attr.has_name(sym::macro_escape) {
                 let inner_attribute = matches!(attr.style, ast::AttrStyle::Inner);
@@ -1229,7 +1214,7 @@ fn contains_macro_use(&self, attrs: &[ast::Attribute]) -> bool {
         false
     }
 
-    fn visit_invoc(&mut self, id: NodeId) -> LocalExpnId {
+    pub(crate) fn visit_invoc(&mut self, id: NodeId) -> LocalExpnId {
         let invoc_id = id.placeholder_to_expn_id();
         let old_parent_scope = self.r.invocation_parent_scopes.insert(invoc_id, self.parent_scope);
         assert!(old_parent_scope.is_none(), "invocation data is reset for an invocation");
@@ -1238,7 +1223,7 @@ fn visit_invoc(&mut self, id: NodeId) -> LocalExpnId {
 
     /// Visit invocation in context in which it can emit a named item (possibly `macro_rules`)
     /// directly into its parent scope's module.
-    fn visit_invoc_in_module(&mut self, id: NodeId) -> MacroRulesScopeRef<'ra> {
+    pub(crate) fn visit_invoc_in_module(&mut self, id: NodeId) -> MacroRulesScopeRef<'ra> {
         let invoc_id = self.visit_invoc(id);
         self.parent_scope.module.unexpanded_invocations.borrow_mut(self.r).insert(invoc_id);
         self.r.arenas.alloc_macro_rules_scope(MacroRulesScope::Invocation(invoc_id))
@@ -1375,24 +1360,8 @@ fn define_macro(&mut self, item: &ast::Item) -> MacroRulesScopeRef<'ra> {
     }
 }
 
-macro_rules! method {
-    ($visit:ident: $ty:ty, $invoc:path, $walk:ident) => {
-        fn $visit(&mut self, node: &'a $ty) {
-            if let $invoc(..) = node.kind {
-                self.visit_invoc(node.id);
-            } else {
-                visit::$walk(self, node);
-            }
-        }
-    };
-}
-
-impl<'a, 'ra, 'tcx> Visitor<'a> for BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
-    method!(visit_expr: ast::Expr, ast::ExprKind::MacCall, walk_expr);
-    method!(visit_pat: ast::Pat, ast::PatKind::MacCall, walk_pat);
-    method!(visit_ty: ast::Ty, ast::TyKind::MacCall, walk_ty);
-
-    fn visit_item(&mut self, item: &'a Item) {
+impl<'a, 'ra, 'tcx> DefCollector<'a, 'ra, 'tcx> {
+    pub(crate) fn brg_visit_item(&mut self, item: &'a Item) {
         let orig_module_scope = self.parent_scope.module;
         self.parent_scope.macro_rules = match item.kind {
             ItemKind::MacroDef(..) => {
@@ -1425,30 +1394,11 @@ fn visit_item(&mut self, item: &'a Item) {
         self.parent_scope.module = orig_module_scope;
     }
 
-    fn visit_stmt(&mut self, stmt: &'a ast::Stmt) {
-        if let ast::StmtKind::MacCall(..) = stmt.kind {
-            self.parent_scope.macro_rules = self.visit_invoc_in_module(stmt.id);
-        } else {
-            visit::walk_stmt(self, stmt);
-        }
+    pub(crate) fn brg_visit_stmt_mac_call(&mut self, stmt: &'a ast::Stmt) {
+        self.parent_scope.macro_rules = self.visit_invoc_in_module(stmt.id);
     }
 
-    fn visit_foreign_item(&mut self, foreign_item: &'a ForeignItem) {
-        let ident = match foreign_item.kind {
-            ForeignItemKind::Static(box StaticItem { ident, .. })
-            | ForeignItemKind::Fn(box Fn { ident, .. })
-            | ForeignItemKind::TyAlias(box TyAlias { ident, .. }) => ident,
-            ForeignItemKind::MacCall(_) => {
-                self.visit_invoc_in_module(foreign_item.id);
-                return;
-            }
-        };
-
-        self.build_reduced_graph_for_foreign_item(foreign_item, ident);
-        visit::walk_item(self, foreign_item);
-    }
-
-    fn visit_block(&mut self, block: &'a Block) {
+    pub(crate) fn brg_visit_block(&mut self, block: &'a Block) {
         let orig_current_module = self.parent_scope.module;
         let orig_current_macro_rules_scope = self.parent_scope.macro_rules;
         self.build_reduced_graph_for_block(block);
@@ -1457,38 +1407,13 @@ fn visit_block(&mut self, block: &'a Block) {
         self.parent_scope.macro_rules = orig_current_macro_rules_scope;
     }
 
-    fn visit_assoc_item(&mut self, item: &'a AssocItem, ctxt: AssocCtxt) {
-        let (ident, ns) = match item.kind {
-            AssocItemKind::Const(box ConstItem { ident, .. })
-            | AssocItemKind::Fn(box Fn { ident, .. })
-            | AssocItemKind::Delegation(box Delegation { ident, .. }) => (ident, ValueNS),
-
-            AssocItemKind::Type(box TyAlias { ident, .. }) => (ident, TypeNS),
-
-            AssocItemKind::MacCall(_) => {
-                match ctxt {
-                    AssocCtxt::Trait => {
-                        self.visit_invoc_in_module(item.id);
-                    }
-                    AssocCtxt::Impl { .. } => {
-                        let invoc_id = item.id.placeholder_to_expn_id();
-                        if !self.r.glob_delegation_invoc_ids.contains(&invoc_id) {
-                            self.r
-                                .impl_unexpanded_invocations
-                                .entry(self.r.invocation_parent(invoc_id))
-                                .or_default()
-                                .insert(invoc_id);
-                        }
-                        self.visit_invoc(item.id);
-                    }
-                }
-                return;
-            }
-
-            AssocItemKind::DelegationMac(..) => {
-                span_bug!(item.span, "delegation mac should already have been removed")
-            }
-        };
+    pub(crate) fn brg_visit_assoc_item(
+        &mut self,
+        item: &'a AssocItem,
+        ctxt: AssocCtxt,
+        ident: Ident,
+        ns: Namespace,
+    ) {
         let vis = self.resolve_visibility(&item.vis);
         let feed = self.r.feed(item.id);
         let local_def_id = feed.key();
@@ -1519,73 +1444,38 @@ fn visit_assoc_item(&mut self, item: &'a AssocItem, ctxt: AssocCtxt) {
         visit::walk_assoc_item(self, item, ctxt);
     }
 
-    fn visit_attribute(&mut self, attr: &'a ast::Attribute) {
-        if !attr.is_doc_comment() && attr::is_builtin_attr(attr) {
-            self.r
-                .builtin_attrs
-                .push((attr.get_normal_item().path.segments[0].ident, self.parent_scope));
-        }
-        visit::walk_attribute(self, attr);
-    }
-
-    fn visit_arm(&mut self, arm: &'a ast::Arm) {
-        if arm.is_placeholder {
-            self.visit_invoc(arm.id);
-        } else {
-            visit::walk_arm(self, arm);
+    pub(crate) fn visit_assoc_item_mac_call(
+        &mut self,
+        item: &'a Item<AssocItemKind>,
+        ctxt: AssocCtxt,
+    ) {
+        match ctxt {
+            AssocCtxt::Trait => {
+                self.visit_invoc_in_module(item.id);
+            }
+            AssocCtxt::Impl { .. } => {
+                let invoc_id = item.id.placeholder_to_expn_id();
+                if !self.r.glob_delegation_invoc_ids.contains(&invoc_id) {
+                    self.r
+                        .impl_unexpanded_invocations
+                        .entry(self.r.invocation_parent(invoc_id))
+                        .or_default()
+                        .insert(invoc_id);
+                }
+                self.visit_invoc(item.id);
+            }
         }
     }
 
-    fn visit_expr_field(&mut self, f: &'a ast::ExprField) {
-        if f.is_placeholder {
-            self.visit_invoc(f.id);
-        } else {
-            visit::walk_expr_field(self, f);
-        }
-    }
-
-    fn visit_pat_field(&mut self, fp: &'a ast::PatField) {
-        if fp.is_placeholder {
-            self.visit_invoc(fp.id);
-        } else {
-            visit::walk_pat_field(self, fp);
-        }
-    }
-
-    fn visit_generic_param(&mut self, param: &'a ast::GenericParam) {
-        if param.is_placeholder {
-            self.visit_invoc(param.id);
-        } else {
-            visit::walk_generic_param(self, param);
-        }
-    }
-
-    fn visit_param(&mut self, p: &'a ast::Param) {
-        if p.is_placeholder {
-            self.visit_invoc(p.id);
-        } else {
-            visit::walk_param(self, p);
-        }
-    }
-
-    fn visit_field_def(&mut self, sf: &'a ast::FieldDef) {
-        if sf.is_placeholder {
-            self.visit_invoc(sf.id);
-        } else {
-            let vis = self.resolve_visibility(&sf.vis);
-            self.r.feed_visibility(self.r.feed(sf.id), vis);
-            visit::walk_field_def(self, sf);
-        }
+    pub(crate) fn brg_visit_field_def(&mut self, sf: &'a ast::FieldDef) {
+        let vis = self.resolve_visibility(&sf.vis);
+        self.r.feed_visibility(self.r.feed(sf.id), vis);
+        visit::walk_field_def(self, sf);
     }
 
     // Constructs the reduced graph for one variant. Variants exist in the
     // type and value namespaces.
-    fn visit_variant(&mut self, variant: &'a ast::Variant) {
-        if variant.is_placeholder {
-            self.visit_invoc_in_module(variant.id);
-            return;
-        }
-
+    pub(crate) fn brg_visit_variant(&mut self, variant: &'a ast::Variant) {
         let parent = self.parent_scope.module.expect_local();
         let expn_id = self.parent_scope.expansion;
         let ident = variant.ident;
@@ -1620,24 +1510,4 @@ fn visit_variant(&mut self, variant: &'a ast::Variant) {
 
         visit::walk_variant(self, variant);
     }
-
-    fn visit_where_predicate(&mut self, p: &'a ast::WherePredicate) {
-        if p.is_placeholder {
-            self.visit_invoc(p.id);
-        } else {
-            visit::walk_where_predicate(self, p);
-        }
-    }
-
-    fn visit_crate(&mut self, krate: &'a ast::Crate) {
-        if krate.is_placeholder {
-            self.visit_invoc_in_module(krate.id);
-        } else {
-            // Visit attributes after items for backward compatibility.
-            // This way they can use `macro_rules` defined later.
-            visit::walk_list!(self, visit_item, &krate.items);
-            visit::walk_list!(self, visit_attribute, &krate.attrs);
-            self.contains_macro_use(&krate.attrs);
-        }
-    }
 }

compiler/rustc_resolve/src/def_collector.rs

@@ -2,39 +2,42 @@
 
 use rustc_ast::visit::FnKind;
 use rustc_ast::*;
+use rustc_attr_parsing as attr;
 use rustc_attr_parsing::{AttributeParser, Early, OmitDoc, ShouldEmit};
 use rustc_expand::expand::AstFragment;
 use rustc_hir as hir;
 use rustc_hir::Target;
+use rustc_hir::def::Namespace::{TypeNS, ValueNS};
 use rustc_hir::def::{CtorKind, CtorOf, DefKind};
 use rustc_hir::def_id::LocalDefId;
 use rustc_middle::span_bug;
-use rustc_span::hygiene::LocalExpnId;
 use rustc_span::{Span, Symbol, sym};
 use tracing::{debug, instrument};
 
-use crate::{ConstArgContext, ImplTraitContext, InvocationParent, Resolver};
+use crate::macros::MacroRulesScopeRef;
+use crate::{ConstArgContext, ImplTraitContext, InvocationParent, ParentScope, Resolver};
 
-pub(crate) fn collect_definitions(
-    resolver: &mut Resolver<'_, '_>,
+pub(crate) fn collect_definitions<'ra>(
+    resolver: &mut Resolver<'ra, '_>,
     fragment: &AstFragment,
-    expansion: LocalExpnId,
-) {
-    let invocation_parent = resolver.invocation_parents[&expansion];
+    parent_scope: ParentScope<'ra>,
+) -> MacroRulesScopeRef<'ra> {
+    let invocation_parent = resolver.invocation_parents[&parent_scope.expansion];
     debug!("new fragment to visit with invocation_parent: {invocation_parent:?}");
-    let mut visitor = DefCollector { resolver, expansion, invocation_parent };
+    let mut visitor = DefCollector { r: resolver, invocation_parent, parent_scope };
     fragment.visit_with(&mut visitor);
+    visitor.parent_scope.macro_rules
 }
 
 /// Creates `DefId`s for nodes in the AST.
-struct DefCollector<'a, 'ra, 'tcx> {
-    resolver: &'a mut Resolver<'ra, 'tcx>,
+pub(crate) struct DefCollector<'a, 'ra, 'tcx> {
+    pub(crate) r: &'a mut Resolver<'ra, 'tcx>,
     invocation_parent: InvocationParent,
-    expansion: LocalExpnId,
+    pub(crate) parent_scope: ParentScope<'ra>,
 }
 
 impl<'a, 'ra, 'tcx> DefCollector<'a, 'ra, 'tcx> {
-    fn create_def(
+    pub(super) fn create_def(
         &mut self,
         node_id: NodeId,
         name: Option<Symbol>,
@@ -46,13 +49,13 @@ fn create_def(
             "create_def(node_id={:?}, def_kind={:?}, parent_def={:?})",
             node_id, def_kind, parent_def
         );
-        self.resolver
+        self.r
             .create_def(
                 parent_def,
                 node_id,
                 name,
                 def_kind,
-                self.expansion.to_expn_id(),
+                self.parent_scope.expansion.to_expn_id(),
                 span.with_parent(None),
             )
             .def_id()
@@ -84,19 +87,20 @@ fn with_const_arg<F: FnOnce(&mut Self)>(&mut self, ctxt: ConstArgContext, f: F)
     fn collect_field(&mut self, field: &'a FieldDef, index: Option<usize>) {
         let index = |this: &Self| {
             index.unwrap_or_else(|| {
-                let node_id = NodeId::placeholder_from_expn_id(this.expansion);
-                this.resolver.placeholder_field_indices[&node_id]
+                let node_id = NodeId::placeholder_from_expn_id(this.parent_scope.expansion);
+                this.r.placeholder_field_indices[&node_id]
             })
         };
 
         if field.is_placeholder {
-            let old_index = self.resolver.placeholder_field_indices.insert(field.id, index(self));
+            let old_index = self.r.placeholder_field_indices.insert(field.id, index(self));
             assert!(old_index.is_none(), "placeholder field index is reset for a node ID");
             self.visit_macro_invoc(field.id);
+            self.visit_invoc(field.id);
         } else {
             let name = field.ident.map_or_else(|| sym::integer(index(self)), |ident| ident.name);
             let def = self.create_def(field.id, Some(name), DefKind::Field, field.span);
-            self.with_parent(def, |this| visit::walk_field_def(this, field));
+            self.with_parent(def, |this| this.brg_visit_field_def(field));
         }
     }
 
@@ -105,7 +109,7 @@ fn visit_macro_invoc(&mut self, id: NodeId) {
         debug!(?self.invocation_parent);
 
         let id = id.placeholder_to_expn_id();
-        let old_parent = self.resolver.invocation_parents.insert(id, self.invocation_parent);
+        let old_parent = self.r.invocation_parents.insert(id, self.invocation_parent);
         assert!(old_parent.is_none(), "parent `LocalDefId` is reset for an invocation");
     }
 }
@@ -144,8 +148,8 @@ fn visit_item(&mut self, i: &'a Item) {
                 // FIXME(jdonszelmann) don't care about tools here maybe? Just parse what we can.
                 // Does that prevents errors from happening? maybe
                 let mut parser = AttributeParser::<'_, Early>::new(
-                    &self.resolver.tcx.sess,
-                    self.resolver.tcx.features(),
+                    &self.r.tcx.sess,
+                    self.r.tcx.features(),
                     Vec::new(),
                     Early { emit_errors: ShouldEmit::Nothing },
                 );
@@ -162,8 +166,7 @@ fn visit_item(&mut self, i: &'a Item) {
                     },
                 );
 
-                let macro_data =
-                    self.resolver.compile_macro(def, *ident, &attrs, i.span, i.id, edition);
+                let macro_data = self.r.compile_macro(def, *ident, &attrs, i.span, i.id, edition);
                 let macro_kinds = macro_data.ext.macro_kinds();
                 opt_macro_data = Some(macro_data);
                 DefKind::Macro(macro_kinds)
@@ -171,17 +174,20 @@ fn visit_item(&mut self, i: &'a Item) {
             ItemKind::GlobalAsm(..) => DefKind::GlobalAsm,
             ItemKind::Use(_) => {
                 self.create_def(i.id, None, DefKind::Use, i.span);
-                return visit::walk_item(self, i);
+                self.brg_visit_item(i);
+                return;
             }
             ItemKind::MacCall(..) | ItemKind::DelegationMac(..) => {
-                return self.visit_macro_invoc(i.id);
+                self.visit_macro_invoc(i.id);
+                self.brg_visit_item(i);
+                return;
             }
         };
         let def_id =
             self.create_def(i.id, i.kind.ident().map(|ident| ident.name), def_kind, i.span);
 
         if let Some(macro_data) = opt_macro_data {
-            self.resolver.new_local_macro(def_id, macro_data);
+            self.r.new_local_macro(def_id, macro_data);
         }
 
         self.with_parent(def_id, |this| {
@@ -201,11 +207,15 @@ fn visit_item(&mut self, i: &'a Item) {
                     }
                     _ => {}
                 }
-                visit::walk_item(this, i);
+                this.brg_visit_item(i);
             })
         });
     }
 
+    fn visit_block(&mut self, block: &'a Block) {
+        self.brg_visit_block(block);
+    }
+
     fn visit_fn(&mut self, fn_kind: FnKind<'a>, _: &AttrVec, span: Span, _: NodeId) {
         match fn_kind {
             FnKind::Fn(
@@ -260,11 +270,6 @@ fn visit_fn(&mut self, fn_kind: FnKind<'a>, _: &AttrVec, span: Span, _: NodeId)
         }
     }
 
-    fn visit_nested_use_tree(&mut self, use_tree: &'a UseTree, id: NodeId) {
-        self.create_def(id, None, DefKind::Use, use_tree.span());
-        visit::walk_use_tree(self, use_tree);
-    }
-
     fn visit_foreign_item(&mut self, fi: &'a ForeignItem) {
         let (ident, def_kind) = match fi.kind {
             ForeignItemKind::Static(box StaticItem {
@@ -285,17 +290,26 @@ fn visit_foreign_item(&mut self, fi: &'a ForeignItem) {
             }
             ForeignItemKind::Fn(box Fn { ident, .. }) => (ident, DefKind::Fn),
             ForeignItemKind::TyAlias(box TyAlias { ident, .. }) => (ident, DefKind::ForeignTy),
-            ForeignItemKind::MacCall(_) => return self.visit_macro_invoc(fi.id),
+            ForeignItemKind::MacCall(_) => {
+                self.visit_invoc_in_module(fi.id);
+                self.visit_macro_invoc(fi.id);
+                return;
+            }
         };
 
         let def = self.create_def(fi.id, Some(ident.name), def_kind, fi.span);
 
-        self.with_parent(def, |this| visit::walk_item(this, fi));
+        self.with_parent(def, |this| {
+            this.build_reduced_graph_for_foreign_item(fi, ident);
+            visit::walk_item(this, fi)
+        });
     }
 
     fn visit_variant(&mut self, v: &'a Variant) {
         if v.is_placeholder {
-            return self.visit_macro_invoc(v.id);
+            self.visit_macro_invoc(v.id);
+            self.visit_invoc_in_module(v.id);
+            return;
         }
         let def = self.create_def(v.id, Some(v.ident.name), DefKind::Variant, v.span);
         self.with_parent(def, |this| {
@@ -307,13 +321,14 @@ fn visit_variant(&mut self, v: &'a Variant) {
                     v.span,
                 );
             }
-            visit::walk_variant(this, v)
+            this.brg_visit_variant(v);
         });
     }
 
     fn visit_where_predicate(&mut self, pred: &'a WherePredicate) {
         if pred.is_placeholder {
-            self.visit_macro_invoc(pred.id)
+            self.visit_macro_invoc(pred.id);
+            self.visit_invoc(pred.id);
         } else {
             visit::walk_where_predicate(self, pred)
         }
@@ -331,6 +346,7 @@ fn visit_variant_data(&mut self, data: &'a VariantData) {
     fn visit_generic_param(&mut self, param: &'a GenericParam) {
         if param.is_placeholder {
             self.visit_macro_invoc(param.id);
+            self.visit_invoc(param.id);
             return;
         }
         let def_kind = match param.kind {
@@ -352,18 +368,23 @@ fn visit_generic_param(&mut self, param: &'a GenericParam) {
     }
 
     fn visit_assoc_item(&mut self, i: &'a AssocItem, ctxt: visit::AssocCtxt) {
-        let (ident, def_kind) = match &i.kind {
+        let (ident, def_kind, ns) = match &i.kind {
             AssocItemKind::Fn(box Fn { ident, .. })
-            | AssocItemKind::Delegation(box Delegation { ident, .. }) => (*ident, DefKind::AssocFn),
+            | AssocItemKind::Delegation(box Delegation { ident, .. }) => {
+                (*ident, DefKind::AssocFn, ValueNS)
+            }
             AssocItemKind::Const(box ConstItem { ident, rhs_kind, .. }) => (
                 *ident,
                 DefKind::AssocConst {
                     is_type_const: matches!(rhs_kind, ConstItemRhsKind::TypeConst { .. }),
                 },
+                ValueNS,
             ),
-            AssocItemKind::Type(box TyAlias { ident, .. }) => (*ident, DefKind::AssocTy),
+            AssocItemKind::Type(box TyAlias { ident, .. }) => (*ident, DefKind::AssocTy, TypeNS),
             AssocItemKind::MacCall(..) => {
-                return self.visit_macro_invoc(i.id);
+                self.visit_macro_invoc(i.id);
+                self.visit_assoc_item_mac_call(i, ctxt);
+                return;
             }
             AssocItemKind::DelegationMac(..) => {
                 span_bug!(i.span, "degation mac invoc should have already been handled")
@@ -371,12 +392,15 @@ fn visit_assoc_item(&mut self, i: &'a AssocItem, ctxt: visit::AssocCtxt) {
         };
 
         let def = self.create_def(i.id, Some(ident.name), def_kind, i.span);
-        self.with_parent(def, |this| visit::walk_assoc_item(this, i, ctxt));
+        self.with_parent(def, |this| this.brg_visit_assoc_item(i, ctxt, ident, ns));
     }
 
     fn visit_pat(&mut self, pat: &'a Pat) {
         match pat.kind {
-            PatKind::MacCall(..) => self.visit_macro_invoc(pat.id),
+            PatKind::MacCall(..) => {
+                self.visit_macro_invoc(pat.id);
+                self.visit_invoc(pat.id);
+            }
             _ => visit::walk_pat(self, pat),
         }
     }
@@ -385,7 +409,7 @@ fn visit_anon_const(&mut self, constant: &'a AnonConst) {
         // `MgcaDisambiguation::Direct` is set even when MGCA is disabled, so
         // to avoid affecting stable we have to feature gate the not creating
         // anon consts
-        if !self.resolver.tcx.features().min_generic_const_args() {
+        if !self.r.tcx.features().min_generic_const_args() {
             let parent =
                 self.create_def(constant.id, None, DefKind::AnonConst, constant.value.span);
             return self.with_parent(parent, |this| visit::walk_anon_const(this, constant));
@@ -410,7 +434,11 @@ fn visit_expr(&mut self, expr: &'a Expr) {
         debug!(?self.invocation_parent);
 
         let parent_def = match &expr.kind {
-            ExprKind::MacCall(..) => return self.visit_macro_invoc(expr.id),
+            ExprKind::MacCall(..) => {
+                self.visit_macro_invoc(expr.id);
+                self.visit_invoc(expr.id);
+                return;
+            }
             ExprKind::Closure(..) | ExprKind::Gen(..) => {
                 self.create_def(expr.id, None, DefKind::Closure, expr.span)
             }
@@ -462,10 +490,13 @@ fn visit_expr(&mut self, expr: &'a Expr) {
 
     fn visit_ty(&mut self, ty: &'a Ty) {
         match ty.kind {
-            TyKind::MacCall(..) => self.visit_macro_invoc(ty.id),
+            TyKind::MacCall(..) => {
+                self.visit_macro_invoc(ty.id);
+                self.visit_invoc(ty.id);
+            }
             TyKind::ImplTrait(opaque_id, _) => {
                 let name = *self
-                    .resolver
+                    .r
                     .impl_trait_names
                     .get(&ty.id)
                     .unwrap_or_else(|| span_bug!(ty.span, "expected this opaque to be named"));
@@ -491,7 +522,10 @@ fn visit_ty(&mut self, ty: &'a Ty) {
 
     fn visit_stmt(&mut self, stmt: &'a Stmt) {
         match stmt.kind {
-            StmtKind::MacCall(..) => self.visit_macro_invoc(stmt.id),
+            StmtKind::MacCall(..) => {
+                self.brg_visit_stmt_mac_call(stmt);
+                self.visit_macro_invoc(stmt.id)
+            }
             // FIXME(impl_trait_in_bindings): We don't really have a good way of
             // introducing the right `ImplTraitContext` here for all the cases we
             // care about, in case we want to introduce ITIB to other positions
@@ -504,12 +538,18 @@ fn visit_stmt(&mut self, stmt: &'a Stmt) {
     }
 
     fn visit_arm(&mut self, arm: &'a Arm) {
-        if arm.is_placeholder { self.visit_macro_invoc(arm.id) } else { visit::walk_arm(self, arm) }
+        if arm.is_placeholder {
+            self.visit_macro_invoc(arm.id);
+            self.visit_invoc(arm.id);
+        } else {
+            visit::walk_arm(self, arm)
+        }
     }
 
     fn visit_expr_field(&mut self, f: &'a ExprField) {
         if f.is_placeholder {
-            self.visit_macro_invoc(f.id)
+            self.visit_macro_invoc(f.id);
+            self.visit_invoc(f.id);
         } else {
             visit::walk_expr_field(self, f)
         }
@@ -517,7 +557,8 @@ fn visit_expr_field(&mut self, f: &'a ExprField) {
 
     fn visit_pat_field(&mut self, fp: &'a PatField) {
         if fp.is_placeholder {
-            self.visit_macro_invoc(fp.id)
+            self.visit_macro_invoc(fp.id);
+            self.visit_invoc(fp.id);
         } else {
             visit::walk_pat_field(self, fp)
         }
@@ -525,7 +566,8 @@ fn visit_pat_field(&mut self, fp: &'a PatField) {
 
     fn visit_param(&mut self, p: &'a Param) {
         if p.is_placeholder {
-            self.visit_macro_invoc(p.id)
+            self.visit_macro_invoc(p.id);
+            self.visit_invoc(p.id);
         } else {
             self.with_impl_trait(ImplTraitContext::Universal, |this| visit::walk_param(this, p))
         }
@@ -539,14 +581,24 @@ fn visit_field_def(&mut self, field: &'a FieldDef) {
 
     fn visit_crate(&mut self, krate: &'a Crate) {
         if krate.is_placeholder {
-            self.visit_macro_invoc(krate.id)
+            self.visit_macro_invoc(krate.id);
+            self.visit_invoc_in_module(krate.id);
         } else {
-            visit::walk_crate(self, krate)
+            // Visit attributes after items for backward compatibility.
+            // This way they can use `macro_rules` defined later.
+            visit::walk_list!(self, visit_item, &krate.items);
+            visit::walk_list!(self, visit_attribute, &krate.attrs);
+            self.contains_macro_use(&krate.attrs);
         }
     }
 
-    fn visit_attribute(&mut self, attr: &'a Attribute) -> Self::Result {
+    fn visit_attribute(&mut self, attr: &'a Attribute) {
         let orig_in_attr = mem::replace(&mut self.invocation_parent.in_attr, true);
+        if !attr.is_doc_comment() && attr::is_builtin_attr(attr) {
+            self.r
+                .builtin_attrs
+                .push((attr.get_normal_item().path.segments[0].ident, self.parent_scope));
+        }
         visit::walk_attribute(self, attr);
         self.invocation_parent.in_attr = orig_in_attr;
     }

compiler/rustc_resolve/src/ident.rs

@@ -5,7 +5,6 @@
 use rustc_ast::{self as ast, NodeId};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def::{DefKind, MacroKinds, Namespace, NonMacroAttrKind, PartialRes, PerNS};
-use rustc_middle::ty::Visibility;
 use rustc_middle::{bug, span_bug};
 use rustc_session::lint::builtin::PROC_MACRO_DERIVE_RESOLUTION_FALLBACK;
 use rustc_session::parse::feature_err;
@@ -24,9 +23,9 @@
 use crate::macros::{MacroRulesScope, sub_namespace_match};
 use crate::{
     AmbiguityError, AmbiguityKind, AmbiguityWarning, BindingKey, CmResolver, Decl, DeclKind,
-    Determinacy, Finalize, IdentKey, ImportKind, LateDecl, LocalModule, Module, ModuleKind,
-    ModuleOrUniformRoot, ParentScope, PathResult, PrivacyError, Res, ResolutionError, Resolver,
-    Scope, ScopeSet, Segment, Stage, Symbol, Used, errors,
+    Determinacy, Finalize, IdentKey, ImportKind, ImportSummary, LateDecl, LocalModule, Module,
+    ModuleKind, ModuleOrUniformRoot, ParentScope, PathResult, PrivacyError, Res, ResolutionError,
+    Resolver, Scope, ScopeSet, Segment, Stage, Symbol, Used, errors,
 };
 
 #[derive(Copy, Clone)]
@@ -485,11 +484,11 @@ fn resolve_ident_in_scope_set_inner<'r>(
                         // We do not need to report them if we are either in speculative resolution,
                         // or in late resolution when everything is already imported and expanded
                         // and no ambiguities exist.
-                        let import_vis = match finalize {
+                        let import = match finalize {
                             None | Some(Finalize { stage: Stage::Late, .. }) => {
                                 return ControlFlow::Break(Ok(decl));
                             }
-                            Some(Finalize { import_vis, .. }) => import_vis,
+                            Some(Finalize { import, .. }) => import,
                         };
 
                         if let Some(&(innermost_decl, _)) = innermost_results.first() {
@@ -503,7 +502,7 @@ fn resolve_ident_in_scope_set_inner<'r>(
                                 decl,
                                 scope,
                                 &innermost_results,
-                                import_vis,
+                                import,
                             ) {
                                 // No need to search for more potential ambiguities, one is enough.
                                 return ControlFlow::Break(Ok(innermost_decl));
@@ -790,19 +789,18 @@ fn maybe_push_ambiguity(
         decl: Decl<'ra>,
         scope: Scope<'ra>,
         innermost_results: &[(Decl<'ra>, Scope<'ra>)],
-        import_vis: Option<Visibility>,
+        import: Option<ImportSummary>,
     ) -> bool {
         let (innermost_decl, innermost_scope) = innermost_results[0];
         let (res, innermost_res) = (decl.res(), innermost_decl.res());
         let ambig_vis = if res != innermost_res {
             None
-        } else if let Some(import_vis) = import_vis
-            && let min =
-                (|d: Decl<'_>| d.vis().min(import_vis.to_def_id(), self.tcx).expect_local())
-            && let (min1, min2) = (min(decl), min(innermost_decl))
-            && min1 != min2
+        } else if let Some(import) = import
+            && let vis1 = self.import_decl_vis(decl, import)
+            && let vis2 = self.import_decl_vis(innermost_decl, import)
+            && vis1 != vis2
         {
-            Some((min1, min2))
+            Some((vis1, vis2))
         } else {
             return false;
         };

compiler/rustc_resolve/src/imports.rs

@@ -37,8 +37,9 @@
 use crate::ref_mut::CmCell;
 use crate::{
     AmbiguityError, BindingKey, CmResolver, Decl, DeclData, DeclKind, Determinacy, Finalize,
-    IdentKey, ImportSuggestion, LocalModule, ModuleOrUniformRoot, ParentScope, PathResult, PerNS,
-    Res, ResolutionError, Resolver, ScopeSet, Segment, Used, module_to_string, names_to_string,
+    IdentKey, ImportSuggestion, ImportSummary, LocalModule, ModuleOrUniformRoot, ParentScope,
+    PathResult, PerNS, Res, ResolutionError, Resolver, ScopeSet, Segment, Used, module_to_string,
+    names_to_string,
 };
 
 /// A potential import declaration in the process of being planted into a module.
@@ -267,6 +268,14 @@ pub(crate) fn simplify(&self, r: &Resolver<'_, '_>) -> Reexport {
             ImportKind::MacroExport => Reexport::MacroExport,
         }
     }
+
+    fn summary(&self) -> ImportSummary {
+        ImportSummary {
+            vis: self.vis,
+            nearest_parent_mod: self.parent_scope.module.nearest_parent_mod().expect_local(),
+            is_single: matches!(self.kind, ImportKind::Single { .. }),
+        }
+    }
 }
 
 /// Records information about the resolution of a name in a namespace of a module.
@@ -327,9 +336,9 @@ struct UnresolvedImportError {
 
 // Reexports of the form `pub use foo as bar;` where `foo` is `extern crate foo;`
 // are permitted for backward-compatibility under a deprecation lint.
-fn pub_use_of_private_extern_crate_hack(import: Import<'_>, decl: Decl<'_>) -> Option<NodeId> {
-    match (&import.kind, &decl.kind) {
-        (ImportKind::Single { .. }, DeclKind::Import { import: decl_import, .. })
+fn pub_use_of_private_extern_crate_hack(import: ImportSummary, decl: Decl<'_>) -> Option<NodeId> {
+    match (import.is_single, decl.kind) {
+        (true, DeclKind::Import { import: decl_import, .. })
             if let ImportKind::ExternCrate { id, .. } = decl_import.kind
                 && import.vis.is_public() =>
         {
@@ -361,23 +370,42 @@ fn remove_same_import<'ra>(d1: Decl<'ra>, d2: Decl<'ra>) -> (Decl<'ra>, Decl<'ra
 }
 
 impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
+    pub(crate) fn import_decl_vis(&self, decl: Decl<'ra>, import: ImportSummary) -> Visibility {
+        assert!(import.vis.is_accessible_from(import.nearest_parent_mod, self.tcx));
+        let decl_vis = decl.vis();
+        if decl_vis.is_at_least(import.vis, self.tcx) {
+            // Ordered, import is less visible than the imported declaration, or the same,
+            // use the import's visibility.
+            import.vis
+        } else if decl_vis.is_accessible_from(import.nearest_parent_mod, self.tcx) {
+            // Ordered, imported declaration is less visible than the import, but is still visible
+            // from the current module, use the declaration's visibility.
+            assert!(import.vis.is_at_least(decl_vis, self.tcx));
+            if pub_use_of_private_extern_crate_hack(import, decl).is_some() {
+                import.vis
+            } else {
+                decl_vis.expect_local()
+            }
+        } else {
+            // Ordered or not, the imported declaration is too private for the current module.
+            // It doesn't matter what visibility we choose here (except in the `PRIVATE_MACRO_USE`
+            // case), because either some error will be reported, or the import declaration
+            // will be thrown away (unfortunately cannot use delayed bug here for this reason).
+            // Use import visibility to keep the all declaration visibilities in a module ordered.
+            import.vis
+        }
+    }
+
     /// Given an import and the declaration that it points to,
     /// create the corresponding import declaration.
     pub(crate) fn new_import_decl(&self, decl: Decl<'ra>, import: Import<'ra>) -> Decl<'ra> {
-        let import_vis = import.vis.to_def_id();
-        let vis = if decl.vis().is_at_least(import_vis, self.tcx)
-            || pub_use_of_private_extern_crate_hack(import, decl).is_some()
-        {
-            import_vis
-        } else {
-            decl.vis()
-        };
+        let vis = self.import_decl_vis(decl, import.summary());
 
         if let ImportKind::Glob { ref max_vis, .. } = import.kind
-            && (vis == import_vis
+            && (vis == import.vis
                 || max_vis.get().is_none_or(|max_vis| vis.is_at_least(max_vis, self.tcx)))
         {
-            max_vis.set_unchecked(Some(vis.expect_local()))
+            max_vis.set_unchecked(Some(vis))
         }
 
         self.arenas.alloc_decl(DeclData {
@@ -385,7 +413,7 @@ pub(crate) fn new_import_decl(&self, decl: Decl<'ra>, import: Import<'ra>) -> De
             ambiguity: CmCell::new(None),
             warn_ambiguity: CmCell::new(false),
             span: import.span,
-            vis: CmCell::new(vis),
+            vis: CmCell::new(vis.to_def_id()),
             expansion: import.parent_scope.expansion,
             parent_module: Some(import.parent_scope.module),
         })
@@ -448,6 +476,7 @@ fn select_glob_decl(
             }
         } else if !old_glob_decl.vis().is_at_least(glob_decl.vis(), self.tcx) {
             // We are glob-importing the same item but with greater visibility.
+            // All visibilities here are ordered because all of them are ancestors of `module`.
             // FIXME: Update visibility in place, but without regressions
             // (#152004, #151124, #152347).
             glob_decl
@@ -471,7 +500,10 @@ pub(crate) fn try_plant_decl_into_local_module(
         decl: Decl<'ra>,
         warn_ambiguity: bool,
     ) -> Result<(), Decl<'ra>> {
+        assert!(!decl.warn_ambiguity.get());
+        assert!(decl.ambiguity.get().is_none());
         let module = decl.parent_module.unwrap().expect_local();
+        assert!(self.is_accessible_from(decl.vis(), module.to_module()));
         let res = decl.res();
         self.check_reserved_macro_name(ident.name, orig_ident_span, res);
         // Even if underscore names cannot be looked up, we still need to add them to modules,
@@ -487,7 +519,6 @@ pub(crate) fn try_plant_decl_into_local_module(
             orig_ident_span,
             warn_ambiguity,
             |this, resolution| {
-                assert!(!decl.warn_ambiguity.get());
                 if decl.is_glob_import() {
                     resolution.glob_decl = Some(match resolution.glob_decl {
                         Some(old_decl) => this.select_glob_decl(
@@ -1261,7 +1292,7 @@ fn finalize_import(&mut self, import: Import<'ra>) -> Option<UnresolvedImportErr
                     &import.parent_scope,
                     Some(Finalize {
                         report_private: false,
-                        import_vis: Some(import.vis),
+                        import: Some(import.summary()),
                         ..finalize
                     }),
                     bindings[ns].get().decl(),
@@ -1461,7 +1492,9 @@ fn finalize_import(&mut self, import: Import<'ra>) -> Option<UnresolvedImportErr
         // All namespaces must be re-exported with extra visibility for an error to occur.
         if !any_successful_reexport {
             let (ns, binding) = reexport_error.unwrap();
-            if let Some(extern_crate_id) = pub_use_of_private_extern_crate_hack(import, binding) {
+            if let Some(extern_crate_id) =
+                pub_use_of_private_extern_crate_hack(import.summary(), binding)
+            {
                 let extern_crate_sp = self.tcx.source_span(self.local_def_id(extern_crate_id));
                 self.lint_buffer.buffer_lint(
                     PUB_USE_OF_PRIVATE_EXTERN_CRATE,

compiler/rustc_resolve/src/late.rs

@@ -365,6 +365,9 @@ enum LifetimeRibKind {
 
     /// This rib acts as a barrier to forbid reference to lifetimes of a parent item.
     Item,
+
+    /// Lifetimes cannot be elided in `impl Trait` types without `#![feature(anonymous_lifetime_in_impl_trait)]`.
+    ImplTrait,
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -941,7 +944,7 @@ fn visit_ty(&mut self, ty: &'ast Ty) {
             }
             TyKind::ImplTrait(..) => {
                 let candidates = self.lifetime_elision_candidates.take();
-                visit::walk_ty(self, ty);
+                self.with_lifetime_rib(LifetimeRibKind::ImplTrait, |this| visit::walk_ty(this, ty));
                 self.lifetime_elision_candidates = candidates;
             }
             TyKind::TraitObject(bounds, ..) => {
@@ -1353,6 +1356,7 @@ fn visit_path_segment(&mut self, path_segment: &'ast PathSegment) {
                         LifetimeRibKind::AnonymousCreateParameter { .. }
                         | LifetimeRibKind::AnonymousReportError
                         | LifetimeRibKind::StaticIfNoLifetimeInScope { .. }
+                        | LifetimeRibKind::ImplTrait
                         | LifetimeRibKind::Elided(_)
                         | LifetimeRibKind::ElisionFailure
                         | LifetimeRibKind::ConcreteAnonConst(_)
@@ -1780,6 +1784,15 @@ fn resolve_lifetime(&mut self, lifetime: &'ast Lifetime, use_ctxt: visit::Lifeti
                                     LifetimeRibKind::ConcreteAnonConst(_) => {
                                         span_bug!(ident.span, "unexpected rib kind: {:?}", rib.kind)
                                     }
+
+                                    LifetimeRibKind::ImplTrait => {
+                                        if self.r.tcx.features().anonymous_lifetime_in_impl_trait()
+                                        {
+                                            None
+                                        } else {
+                                            Some(LifetimeUseSet::Many)
+                                        }
+                                    }
                                 })
                                 .unwrap_or(LifetimeUseSet::Many);
                             debug!(?use_ctxt, ?use_set);
@@ -1819,6 +1832,7 @@ fn resolve_lifetime(&mut self, lifetime: &'ast Lifetime, use_ctxt: visit::Lifeti
                 | LifetimeRibKind::Generics { .. }
                 | LifetimeRibKind::ElisionFailure
                 | LifetimeRibKind::AnonymousReportError
+                | LifetimeRibKind::ImplTrait
                 | LifetimeRibKind::StaticIfNoLifetimeInScope { .. } => {}
             }
         }
@@ -2000,7 +2014,9 @@ fn resolve_anonymous_lifetime(
                     return;
                 }
                 LifetimeRibKind::Item => break,
-                LifetimeRibKind::Generics { .. } | LifetimeRibKind::ConstParamTy => {}
+                LifetimeRibKind::Generics { .. }
+                | LifetimeRibKind::ConstParamTy
+                | LifetimeRibKind::ImplTrait => {}
                 LifetimeRibKind::ConcreteAnonConst(_) => {
                     // There is always an `Elided(LifetimeRes::Infer)` inside an `AnonConst`.
                     span_bug!(lifetime.ident.span, "unexpected rib kind: {:?}", rib.kind)
@@ -2323,7 +2339,9 @@ fn resolve_elided_lifetimes_in_path(
                         }
                         break;
                     }
-                    LifetimeRibKind::Generics { .. } | LifetimeRibKind::ConstParamTy => {}
+                    LifetimeRibKind::Generics { .. }
+                    | LifetimeRibKind::ConstParamTy
+                    | LifetimeRibKind::ImplTrait => {}
                     LifetimeRibKind::ConcreteAnonConst(_) => {
                         // There is always an `Elided(LifetimeRes::Infer)` inside an `AnonConst`.
                         span_bug!(elided_lifetime_span, "unexpected rib kind: {:?}", rib.kind)

compiler/rustc_resolve/src/lib.rs

@@ -2722,6 +2722,15 @@ enum Stage {
     Late,
 }
 
+/// Parts of import data required for finalizing import resolution.
+/// Does not carry a lifetime, so it can be stored in `Finalize`.
+#[derive(Copy, Clone, Debug)]
+struct ImportSummary {
+    vis: Visibility,
+    nearest_parent_mod: LocalDefId,
+    is_single: bool,
+}
+
 /// Invariant: if `Finalize` is used, expansion and import resolution must be complete.
 #[derive(Copy, Clone, Debug)]
 struct Finalize {
@@ -2740,8 +2749,8 @@ struct Finalize {
     used: Used = Used::Other,
     /// Finalizing early or late resolution.
     stage: Stage = Stage::Early,
-    /// Nominal visibility of the import item, in case we are resolving an import's final segment.
-    import_vis: Option<Visibility> = None,
+    /// Some import data, in case we are resolving an import's final segment.
+    import: Option<ImportSummary> = None,
 }
 
 impl Finalize {

compiler/rustc_resolve/src/macros.rs

@@ -34,6 +34,7 @@
 use rustc_span::{DUMMY_SP, Ident, Span, Symbol, kw, sym};
 
 use crate::Namespace::*;
+use crate::def_collector::collect_definitions;
 use crate::errors::{
     self, AddAsNonDerive, CannotDetermineMacroResolution, CannotFindIdentInThisScope,
     MacroExpectedFound, RemoveSurroundingDerive,
@@ -189,7 +190,7 @@ fn visit_ast_fragment_with_placeholders(
         // Integrate the new AST fragment into all the definition and module structures.
         // We are inside the `expansion` now, but other parent scope components are still the same.
         let parent_scope = ParentScope { expansion, ..self.invocation_parent_scopes[&expansion] };
-        let output_macro_rules_scope = self.build_reduced_graph(fragment, parent_scope);
+        let output_macro_rules_scope = collect_definitions(self, fragment, parent_scope);
         self.output_macro_rules_scopes.insert(expansion, output_macro_rules_scope);
 
         parent_scope.module.unexpanded_invocations.borrow_mut(self).remove(&expansion);

compiler/rustc_session/src/config.rs

@@ -2641,7 +2641,7 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
     if unstable_opts.retpoline_external_thunk {
         unstable_opts.retpoline = true;
         collected_options.target_modifiers.insert(
-            OptionsTargetModifiers::UnstableOptions(UnstableOptionsTargetModifiers::retpoline),
+            OptionsTargetModifiers::UnstableOptions(UnstableOptionsTargetModifiers::Retpoline),
             "true".to_string(),
         );
     }

compiler/rustc_session/src/lib.rs

@@ -5,6 +5,7 @@
 #![feature(default_field_values)]
 #![feature(iter_intersperse)]
 #![feature(macro_derive)]
+#![feature(macro_metavar_expr)]
 #![feature(rustc_attrs)]
 // To generate CodegenOptionsTargetModifiers and UnstableOptionsTargetModifiers enums
 // with macro_rules, it is necessary to use recursive mechanic ("Incremental TT Munchers").

compiler/rustc_session/src/options.rs

@@ -141,10 +141,10 @@ pub fn consistent(&self, sess: &Session, other: Option<&TargetModifier>) -> bool
         assert!(other.is_none() || self.opt == other.unwrap().opt);
         match self.opt {
             OptionsTargetModifiers::UnstableOptions(unstable) => match unstable {
-                UnstableOptionsTargetModifiers::sanitizer => {
+                UnstableOptionsTargetModifiers::Sanitizer => {
                     return target_modifier_consistency_check::sanitizer(self, other);
                 }
-                UnstableOptionsTargetModifiers::sanitizer_cfi_normalize_integers => {
+                UnstableOptionsTargetModifiers::SanitizerCfiNormalizeIntegers => {
                     return target_modifier_consistency_check::sanitizer_cfi_normalize_integers(
                         sess, self, other,
                     );
@@ -170,164 +170,52 @@ fn tmod_push_impl(
     }
 }
 
-macro_rules! tmod_push {
-    ($struct_name:ident, $tmod_enum_name:ident, $opt_name:ident, $opt_expr:expr, $init:expr, $mods:expr, $tmod_vals:expr) => {
-        if *$opt_expr != $init {
-            tmod_push_impl(
-                OptionsTargetModifiers::$struct_name($tmod_enum_name::$opt_name),
-                $tmod_vals,
-                $mods,
-            );
-        }
-    };
-}
-
-macro_rules! gather_tmods {
-    ($struct_name:ident, $tmod_enum_name:ident, $opt_name:ident, $opt_expr:expr, $init:expr, $mods:expr, $tmod_vals:expr,
-        [SUBSTRUCT], [TARGET_MODIFIER]) => {
-        compile_error!("SUBSTRUCT can't be target modifier");
-    };
-    ($struct_name:ident, $tmod_enum_name:ident, $opt_name:ident, $opt_expr:expr, $init:expr, $mods:expr, $tmod_vals:expr,
-        [UNTRACKED], [TARGET_MODIFIER]) => {
-        tmod_push!($struct_name, $tmod_enum_name, $opt_name, $opt_expr, $init, $mods, $tmod_vals)
-    };
-    ($struct_name:ident, $tmod_enum_name:ident, $opt_name:ident, $opt_expr:expr, $init:expr, $mods:expr, $tmod_vals:expr,
-        [TRACKED], [TARGET_MODIFIER]) => {
-        tmod_push!($struct_name, $tmod_enum_name, $opt_name, $opt_expr, $init, $mods, $tmod_vals)
-    };
-    ($struct_name:ident, $tmod_enum_name:ident, $opt_name:ident, $opt_expr:expr, $init:expr, $mods:expr, $tmod_vals:expr,
-        [TRACKED_NO_CRATE_HASH], [TARGET_MODIFIER]) => {
-        tmod_push!($struct_name, $tmod_enum_name, $opt_name, $opt_expr, $init, $mods, $tmod_vals)
-    };
-    ($struct_name:ident, $tmod_enum_name:ident, $opt_name:ident, $opt_expr:expr, $init:expr, $mods:expr, $tmod_vals:expr,
-        [SUBSTRUCT], [$(MITIGATION)?]) => {
-        $opt_expr.gather_target_modifiers($mods, $tmod_vals);
-    };
-    ($struct_name:ident, $tmod_enum_name:ident, $opt_name:ident, $opt_expr:expr, $init:expr, $mods:expr, $tmod_vals:expr,
-        [UNTRACKED], [$(MITIGATION)?]) => {{}};
-    ($struct_name:ident, $tmod_enum_name:ident, $opt_name:ident, $opt_expr:expr, $init:expr, $mods:expr, $tmod_vals:expr,
-        [TRACKED], [$(MITIGATION)?]) => {{}};
-    ($struct_name:ident, $tmod_enum_name:ident, $opt_name:ident, $opt_expr:expr, $init:expr, $mods:expr, $tmod_vals:expr,
-        [TRACKED_NO_CRATE_HASH], [$(MITIGATION)?]) => {{}};
-}
-
-macro_rules! gather_tmods_top_level {
-    ($_opt_name:ident, $opt_expr:expr, $mods:expr, $tmod_vals:expr, [SUBSTRUCT $substruct_enum:ident]) => {
-        $opt_expr.gather_target_modifiers($mods, $tmod_vals);
-    };
-    ($opt_name:ident, $opt_expr:expr, $mods:expr, $tmod_vals:expr, [$non_substruct:ident TARGET_MODIFIER]) => {
-        compile_error!("Top level option can't be target modifier");
-    };
-    ($opt_name:ident, $opt_expr:expr, $mods:expr, $tmod_vals:expr, [$non_substruct:ident $(MITIGATION)?]) => {};
-}
-
-/// Macro for generating OptionsTargetsModifiers top-level enum with impl.
-/// Will generate something like:
-/// ```rust,ignore (illustrative)
-/// pub enum OptionsTargetModifiers {
-///     CodegenOptions(CodegenOptionsTargetModifiers),
-///     UnstableOptions(UnstableOptionsTargetModifiers),
-/// }
-/// impl OptionsTargetModifiers {
-///     pub fn reparse(&self, user_value: &str) -> ExtendedTargetModifierInfo {
-///         match self {
-///             Self::CodegenOptions(v) => v.reparse(user_value),
-///             Self::UnstableOptions(v) => v.reparse(user_value),
-///         }
-///     }
-///     pub fn is_target_modifier(flag_name: &str) -> bool {
-///         CodegenOptionsTargetModifiers::is_target_modifier(flag_name) ||
-///         UnstableOptionsTargetModifiers::is_target_modifier(flag_name)
-///     }
-/// }
-/// ```
-macro_rules! top_level_tmod_enum {
-    ($( {$($optinfo:tt)*} ),* $(,)*) => {
-        top_level_tmod_enum! { @parse {}, (user_value){}; $($($optinfo)*|)* }
-    };
-    // Termination
-    (
-        @parse
-        {$($variant:tt($substruct_enum:tt))*},
-        ($user_value:ident){$($pout:tt)*};
-    ) => {
-        #[allow(non_camel_case_types)]
-        #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Copy, Clone, Encodable, BlobDecodable)]
-        pub enum OptionsTargetModifiers {
-            $($variant($substruct_enum)),*
-        }
-        impl OptionsTargetModifiers {
-            #[allow(unused_variables)]
-            pub fn reparse(&self, $user_value: &str) -> ExtendedTargetModifierInfo {
-                #[allow(unreachable_patterns)]
-                match self {
-                    $($pout)*
-                    _ => panic!("unknown target modifier option: {:?}", *self)
-                }
-            }
-            pub fn is_target_modifier(flag_name: &str) -> bool {
-                $($substruct_enum::is_target_modifier(flag_name))||*
-            }
-        }
-    };
-    // Adding SUBSTRUCT option group into $eout
-    (
-        @parse {$($eout:tt)*}, ($puser_value:ident){$($pout:tt)*};
-            [SUBSTRUCT $substruct_enum:ident $variant:ident] |
-        $($tail:tt)*
-    ) => {
-        top_level_tmod_enum! {
-            @parse
-            {
-                $($eout)*
-                $variant($substruct_enum)
-            },
-            ($puser_value){
-                $($pout)*
-                Self::$variant(v) => v.reparse($puser_value),
-            };
-            $($tail)*
-        }
-    };
-    // Skipping non-target-modifier and non-substruct
-    (
-        @parse {$($eout:tt)*}, ($puser_value:ident){$($pout:tt)*};
-            [$non_substruct:ident] |
-        $($tail:tt)*
-    ) => {
-        top_level_tmod_enum! {
-            @parse
-            {
-                $($eout)*
-            },
-            ($puser_value){
-                $($pout)*
-            };
-            $($tail)*
-        }
-    };
-}
-
 macro_rules! top_level_options {
     (
         $(#[$top_level_attr:meta])*
         pub struct Options {
             $(
                 $(#[$attr:meta])*
-                $opt:ident : $t:ty [
-                    $dep_tracking_marker:ident
-                    $( $tmod:ident $variant:ident )?
-                ],
+                $opt:ident : $t:ty
+                [$dep_tracking_marker:ident]
+                $( { TARGET_MODIFIER: $tmod_variant:ident($tmod_enum:ident) } )?
+                ,
             )*
         }
     ) => {
-        top_level_tmod_enum!(
-            {
+        #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Copy, Clone, Encodable, BlobDecodable)]
+        pub enum OptionsTargetModifiers {
+            $(
                 $(
-                    [$dep_tracking_marker $($tmod $variant),*]
-                )|*
+                    $tmod_variant($tmod_enum),
+                )?
+            )*
+        }
+
+        impl OptionsTargetModifiers {
+            pub fn reparse(&self, user_value: &str) -> ExtendedTargetModifierInfo {
+                match self {
+                    $(
+                        $(
+                            Self::$tmod_variant(v) => v.reparse(user_value),
+                        )?
+                    )*
+                    #[allow(unreachable_patterns)]
+                    _ => panic!("unknown target modifier option: {self:?}"),
+                }
             }
-        );
+
+            pub fn is_target_modifier(flag_name: &str) -> bool {
+                $(
+                    $(
+                        if $tmod_enum::is_target_modifier(flag_name) {
+                            return true
+                        }
+                    )?
+                )*
+                false
+            }
+        }
 
         #[derive(Clone)]
         $(#[$top_level_attr])*
@@ -375,13 +263,11 @@ pub fn dep_tracking_hash(&self, for_crate_hash: bool) -> Hash64 {
             pub fn gather_target_modifiers(&self) -> Vec<TargetModifier> {
                 let mut mods = Vec::<TargetModifier>::new();
                 $(
-                    gather_tmods_top_level!(
-                        $opt,
-                        &self.$opt,
-                        &mut mods,
-                        &self.target_modifiers,
-                        [$dep_tracking_marker $($tmod),*]
-                    );
+                    $(
+                        // Only expand for flags that have `TARGET_MODIFIER`.
+                        ${ignore($tmod_enum)}
+                        self.$opt.gather_target_modifiers(&mut mods, &self.target_modifiers);
+                    )?
                 )*
                 mods.sort_by(|a, b| a.opt.cmp(&b.opt));
                 mods
@@ -451,9 +337,9 @@ pub struct Options {
         #[rustc_lint_opt_deny_field_access("should only be used via `Config::track_state`")]
         untracked_state_hash: Hash64 [TRACKED_NO_CRATE_HASH],
 
-        unstable_opts: UnstableOptions [SUBSTRUCT UnstableOptionsTargetModifiers UnstableOptions],
+        unstable_opts: UnstableOptions [SUBSTRUCT] { TARGET_MODIFIER: UnstableOptions(UnstableOptionsTargetModifiers) },
         prints: Vec<PrintRequest> [UNTRACKED],
-        cg: CodegenOptions [SUBSTRUCT CodegenOptionsTargetModifiers CodegenOptions],
+        cg: CodegenOptions [SUBSTRUCT] { TARGET_MODIFIER: CodegenOptions(CodegenOptionsTargetModifiers) },
         externs: Externs [UNTRACKED],
         crate_name: Option<String> [TRACKED],
         /// Indicates how the compiler should treat unstable features.
@@ -530,108 +416,6 @@ pub struct Options {
     }
 );
 
-macro_rules! mitigation_enum_opt {
-    ($opt:ident, MITIGATION) => {
-        Some(mitigation_coverage::DeniedPartialMitigationKind::$opt)
-    };
-    ($opt:ident, $(TARGET_MODIFIER)?) => {
-        None
-    };
-}
-
-macro_rules! tmod_enum_opt {
-    ($struct_name:ident, $tmod_enum_name:ident, $opt:ident, TARGET_MODIFIER) => {
-        Some(OptionsTargetModifiers::$struct_name($tmod_enum_name::$opt))
-    };
-    ($struct_name:ident, $tmod_enum_name:ident, $opt:ident, $(MITIGATION)?) => {
-        None
-    };
-}
-
-macro_rules! tmod_enum {
-    ($tmod_enum_name:ident, $prefix:expr, $( {$($optinfo:tt)*} ),* $(,)*) => {
-        tmod_enum! { $tmod_enum_name, $prefix, @parse {}, (user_value){}; $($($optinfo)*|)* }
-    };
-    // Termination
-    (
-        $tmod_enum_name:ident, $prefix:expr,
-        @parse
-        {$($eout:tt)*},
-        ($user_value:ident){$($pout:tt)*};
-    ) => {
-        #[allow(non_camel_case_types)]
-        #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Copy, Clone, Encodable, BlobDecodable)]
-        pub enum $tmod_enum_name {
-            $($eout),*
-        }
-        impl $tmod_enum_name {
-            #[allow(unused_variables)]
-            pub fn reparse(&self, $user_value: &str) -> ExtendedTargetModifierInfo {
-                #[allow(unreachable_patterns)]
-                match self {
-                    $($pout)*
-                    _ => panic!("unknown target modifier option: {:?}", *self)
-                }
-            }
-            pub fn is_target_modifier(flag_name: &str) -> bool {
-                match flag_name.replace('-', "_").as_str() {
-                    $(stringify!($eout) => true,)*
-                    _ => false,
-                }
-            }
-        }
-    };
-    // Adding target-modifier option into $eout
-    (
-        $tmod_enum_name:ident, $prefix:expr,
-        @parse {$($eout:tt)*}, ($puser_value:ident){$($pout:tt)*};
-            $opt:ident, $parse:ident, $t:ty, [TARGET_MODIFIER] |
-        $($tail:tt)*
-    ) => {
-        tmod_enum! {
-            $tmod_enum_name, $prefix,
-            @parse
-            {
-                $($eout)*
-                $opt
-            },
-            ($puser_value){
-                $($pout)*
-                Self::$opt => {
-                    let mut parsed : $t = Default::default();
-                    let val = if $puser_value.is_empty() { None } else { Some($puser_value) };
-                    parse::$parse(&mut parsed, val);
-                    ExtendedTargetModifierInfo {
-                        prefix: $prefix.to_string(),
-                        name: stringify!($opt).to_string().replace('_', "-"),
-                        tech_value: format!("{:?}", parsed),
-                    }
-                },
-            };
-            $($tail)*
-        }
-    };
-    // Skipping non-target-modifier
-    (
-        $tmod_enum_name:ident, $prefix:expr,
-        @parse {$($eout:tt)*}, ($puser_value:ident){$($pout:tt)*};
-            $opt:ident, $parse:ident, $t:ty, [$(MITIGATION)?] |
-        $($tail:tt)*
-    ) => {
-        tmod_enum! {
-            $tmod_enum_name, $prefix,
-            @parse
-            {
-                $($eout)*
-            },
-            ($puser_value){
-                $($pout)*
-            };
-            $($tail)*
-        }
-    };
-}
-
 #[derive(Default)]
 pub struct CollectedOptions {
     pub target_modifiers: BTreeMap<OptionsTargetModifiers, String>,
@@ -684,7 +468,7 @@ pub(super) fn $opt(
 macro_rules! options {
     (
         $struct_name:ident,
-        $tmod_enum_name:ident,
+        $tmod_enum:ident,
         $stat:ident,
         $optmod:ident,
         $prefix:expr,
@@ -695,8 +479,11 @@ macro_rules! options {
             $opt:ident : $t:ty = (
                 $init:expr,
                 $parse:ident,
-                [$dep_tracking_marker:ident $( $modifier_kind:ident )?],
-                $desc:expr
+                [$dep_tracking_marker:ident]
+                $( { TARGET_MODIFIER: $tmod_variant:ident } )?
+                $( { MITIGATION: $mitigation_variant:ident } )?
+                ,
+                $desc:literal
                 $(, removed: $removed:ident )?
             ),
         )*
@@ -710,15 +497,49 @@ pub struct $struct_name {
             )*
         }
 
-        tmod_enum!(
-            $tmod_enum_name,
-            $prefix,
-            {
-                $(
-                    $opt, $parse, $t, [$($modifier_kind),*]
-                )|*
+        #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Copy, Clone, Encodable, BlobDecodable)]
+        pub enum $tmod_enum {
+            $(
+                $( $tmod_variant, )?
+            )*
+        }
+
+        impl $tmod_enum {
+            pub fn reparse(&self, _user_value: &str) -> ExtendedTargetModifierInfo {
+                match self {
+                    $(
+                        $(
+                            Self::$tmod_variant => {
+                                let mut parsed: $t = Default::default();
+                                let val = if _user_value.is_empty() { None } else { Some(_user_value) };
+                                parse::$parse(&mut parsed, val);
+                                ExtendedTargetModifierInfo {
+                                    prefix: $prefix.to_string(),
+                                    name: stringify!($opt).to_string().replace('_', "-"),
+                                    tech_value: format!("{:?}", parsed),
+                                }
+                            }
+                        )?
+                    )*
+
+                    #[allow(unreachable_patterns)]
+                    _ => panic!("unknown target modifier option: {:?}", *self)
+                }
             }
-        );
+
+            pub fn is_target_modifier(flag_name: &str) -> bool {
+                match flag_name.replace('-', "_").as_str() {
+                    $(
+                        $(
+                            // Only expand for flags that have `TARGET_MODIFIER`.
+                            ${ignore($tmod_variant)}
+                            stringify!($opt) => true,
+                        )?
+                    )*
+                    _ => false,
+                }
+            }
+        }
 
         impl Default for $struct_name {
             fn default() -> $struct_name {
@@ -770,17 +591,15 @@ pub fn gather_target_modifiers(
                 _tmod_vals: &BTreeMap<OptionsTargetModifiers, String>,
             ) {
                 $(
-                    gather_tmods!(
-                        $struct_name,
-                        $tmod_enum_name,
-                        $opt,
-                        &self.$opt,
-                        $init,
-                        _mods,
-                        _tmod_vals,
-                        [$dep_tracking_marker],
-                        [$($modifier_kind),*]
-                    );
+                    $(
+                        if self.$opt != $init {
+                            tmod_push_impl(
+                                OptionsTargetModifiers::$struct_name($tmod_enum::$tmod_variant),
+                                _tmod_vals,
+                                _mods,
+                            );
+                        }
+                    )?
                 )*
             }
         }
@@ -793,8 +612,12 @@ pub fn gather_target_modifiers(
                     type_desc: desc::$parse,
                     desc: $desc,
                     removed: None $( .or(Some(RemovedOption::$removed)) )?,
-                    tmod: tmod_enum_opt!($struct_name, $tmod_enum_name, $opt, $($modifier_kind),*),
-                    mitigation: mitigation_enum_opt!($opt, $($modifier_kind),*),
+                    tmod: None $( .or(Some(
+                        OptionsTargetModifiers::$struct_name($tmod_enum::$tmod_variant)
+                    )))?,
+                    mitigation: None $( .or(Some(
+                        mitigation_coverage::DeniedPartialMitigationKind::$mitigation_variant
+                    )))?,
                 },
             )*
         ];
@@ -2230,7 +2053,7 @@ pub(crate) fn parse_assert_incr_state(
     collapse_macro_debuginfo: CollapseMacroDebuginfo = (CollapseMacroDebuginfo::Unspecified,
         parse_collapse_macro_debuginfo, [TRACKED],
         "set option to collapse debuginfo for macros"),
-    control_flow_guard: CFGuard = (CFGuard::Disabled, parse_cfguard, [TRACKED MITIGATION],
+    control_flow_guard: CFGuard = (CFGuard::Disabled, parse_cfguard, [TRACKED] { MITIGATION: ControlFlowGuard },
         "use Windows Control Flow Guard (default: no)"),
     debug_assertions: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "explicitly enable the `cfg(debug_assertions)` directive"),
@@ -2409,7 +2232,7 @@ pub(crate) fn parse_assert_incr_state(
         (default: no)"),
     box_noalias: bool = (true, parse_bool, [TRACKED],
         "emit noalias metadata for box (default: yes)"),
-    branch_protection: Option<BranchProtection> = (None, parse_branch_protection, [TRACKED TARGET_MODIFIER],
+    branch_protection: Option<BranchProtection> = (None, parse_branch_protection, [TRACKED] { TARGET_MODIFIER: BranchProtection },
         "set options for branch target identification and pointer authentication on AArch64"),
     build_sdylib_interface: bool = (false, parse_bool, [UNTRACKED],
         "whether the stable interface is being built"),
@@ -2510,7 +2333,7 @@ pub(crate) fn parse_assert_incr_state(
     fewer_names: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "reduce memory use by retaining fewer names within compilation artifacts (LLVM-IR) \
         (default: no)"),
-    fixed_x18: bool = (false, parse_bool, [TRACKED TARGET_MODIFIER],
+    fixed_x18: bool = (false, parse_bool, [TRACKED] { TARGET_MODIFIER: FixedX18 },
         "make the x18 register reserved on AArch64 (default: no)"),
     flatten_format_args: bool = (true, parse_bool, [TRACKED],
         "flatten nested format_args!() and literals into a simplified format_args!() call \
@@ -2554,7 +2377,7 @@ pub(crate) fn parse_assert_incr_state(
         - hashes of green query instances
         - hash collisions of query keys
         - hash collisions when creating dep-nodes"),
-    indirect_branch_cs_prefix: bool = (false, parse_bool, [TRACKED TARGET_MODIFIER],
+    indirect_branch_cs_prefix: bool = (false, parse_bool, [TRACKED] { TARGET_MODIFIER: IndirectBranchCsPrefix },
         "add `cs` prefix to `call` and `jmp` to indirect thunks (default: no)"),
     inline_llvm: bool = (true, parse_bool, [TRACKED],
         "enable LLVM inlining (default: yes)"),
@@ -2749,10 +2572,10 @@ pub(crate) fn parse_assert_incr_state(
         "enable queries of the dependency graph for regression testing (default: no)"),
     randomize_layout: bool = (false, parse_bool, [TRACKED],
         "randomize the layout of types (default: no)"),
-    reg_struct_return: bool = (false, parse_bool, [TRACKED TARGET_MODIFIER],
+    reg_struct_return: bool = (false, parse_bool, [TRACKED] { TARGET_MODIFIER: RegStructReturn },
         "On x86-32 targets, it overrides the default ABI to return small structs in registers.
         It is UNSOUND to link together crates that use different values for this flag!"),
-    regparm: Option<u32> = (None, parse_opt_number, [TRACKED TARGET_MODIFIER],
+    regparm: Option<u32> = (None, parse_opt_number, [TRACKED] { TARGET_MODIFIER: Regparm },
         "On x86-32 targets, setting this to N causes the compiler to pass N arguments \
         in registers EAX, EDX, and ECX instead of on the stack for\
         \"C\", \"cdecl\", and \"stdcall\" fn.\
@@ -2764,19 +2587,19 @@ pub(crate) fn parse_assert_incr_state(
     remark_dir: Option<PathBuf> = (None, parse_opt_pathbuf, [UNTRACKED],
         "directory into which to write optimization remarks (if not specified, they will be \
 written to standard error output)"),
-    retpoline: bool = (false, parse_bool, [TRACKED TARGET_MODIFIER],
+    retpoline: bool = (false, parse_bool, [TRACKED] { TARGET_MODIFIER: Retpoline },
         "enables retpoline-indirect-branches and retpoline-indirect-calls target features (default: no)"),
-    retpoline_external_thunk: bool = (false, parse_bool, [TRACKED TARGET_MODIFIER],
+    retpoline_external_thunk: bool = (false, parse_bool, [TRACKED] { TARGET_MODIFIER: RetpolineExternalThunk },
         "enables retpoline-external-thunk, retpoline-indirect-branches and retpoline-indirect-calls \
         target features (default: no)"),
     #[rustc_lint_opt_deny_field_access("use `Session::sanitizers()` instead of this field")]
-    sanitizer: SanitizerSet = (SanitizerSet::empty(), parse_sanitizers, [TRACKED TARGET_MODIFIER],
+    sanitizer: SanitizerSet = (SanitizerSet::empty(), parse_sanitizers, [TRACKED] { TARGET_MODIFIER: Sanitizer },
         "use a sanitizer"),
     sanitizer_cfi_canonical_jump_tables: Option<bool> = (Some(true), parse_opt_bool, [TRACKED],
         "enable canonical jump tables (default: yes)"),
     sanitizer_cfi_generalize_pointers: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "enable generalizing pointer types (default: no)"),
-    sanitizer_cfi_normalize_integers: Option<bool> = (None, parse_opt_bool, [TRACKED TARGET_MODIFIER],
+    sanitizer_cfi_normalize_integers: Option<bool> = (None, parse_opt_bool, [TRACKED] { TARGET_MODIFIER: SanitizerCfiNormalizeIntegers },
         "enable normalizing integer types (default: no)"),
     sanitizer_dataflow_abilist: Vec<String> = (Vec::new(), parse_comma_list, [TRACKED],
         "additional ABI list files that control how shadow parameters are passed (comma separated)"),
@@ -2836,7 +2659,7 @@ pub(crate) fn parse_assert_incr_state(
     src_hash_algorithm: Option<SourceFileHashAlgorithm> = (None, parse_src_file_hash, [TRACKED],
         "hash algorithm of source files in debug info (`md5`, `sha1`, or `sha256`)"),
     #[rustc_lint_opt_deny_field_access("use `Session::stack_protector` instead of this field")]
-    stack_protector: StackProtector = (StackProtector::None, parse_stack_protector, [TRACKED MITIGATION],
+    stack_protector: StackProtector = (StackProtector::None, parse_stack_protector, [TRACKED] { MITIGATION: StackProtector },
         "control stack smash protection strategy (`rustc --print stack-protector-strategies` for details)"),
     staticlib_allow_rdylib_deps: bool = (false, parse_bool, [TRACKED],
         "allow staticlibs to have rust dylib dependencies"),

compiler/rustc_session/src/options/mitigation_coverage.rs

@@ -133,7 +133,6 @@ macro_rules! intersperse {
 
 macro_rules! denied_partial_mitigations {
     ([$self:ident] enum $kind:ident {$(($name:ident, $text:expr, $since:ident, $code:expr)),*}) => {
-        #[allow(non_camel_case_types)]
         #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Encodable, BlobDecodable)]
         pub enum DeniedPartialMitigationKind {
             $($name),*
@@ -204,8 +203,8 @@ pub fn gather_enabled_denied_partial_mitigations(&$self) -> Vec<DeniedPartialMit
     enum DeniedPartialMitigationKind {
         // The mitigation name should match the option name in rustc_session::options,
         // to allow for resetting the mitigation
-        (stack_protector, "stack-protector", EditionFuture, self.stack_protector()),
-        (control_flow_guard, "control-flow-guard", EditionFuture, self.opts.cg.control_flow_guard == CFGuard::Checks)
+        (StackProtector, "stack-protector", EditionFuture, self.stack_protector()),
+        (ControlFlowGuard, "control-flow-guard", EditionFuture, self.opts.cg.control_flow_guard == CFGuard::Checks)
     }
 }
 

compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs

@@ -15,8 +15,9 @@
 use rustc_middle::ty::adjustment::{Adjust, Adjustment, AutoBorrow, DerefAdjustKind};
 use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter, Print, Printer};
 use rustc_middle::ty::{
-    self, GenericArg, GenericArgKind, GenericArgsRef, InferConst, IsSuggestable, Term, TermKind,
-    Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt, TypeckResults,
+    self, GenericArg, GenericArgKind, GenericArgsRef, GenericParamDefKind, InferConst,
+    IsSuggestable, Term, TermKind, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable,
+    TypeVisitableExt, TypeckResults,
 };
 use rustc_span::{BytePos, DUMMY_SP, Ident, Span, sym};
 use tracing::{debug, instrument, warn};
@@ -592,15 +593,19 @@ pub fn emit_inference_failure_err_with_type_hint(
                             (true, parent.prefix.to_string(), parent.name)
                         });
 
+                let param = &generics.own_params[argument_index];
+                let param_name = param.name.to_string();
+
                 infer_subdiags.push(SourceKindSubdiag::GenericLabel {
                     span,
                     is_type,
-                    param_name: generics.own_params[argument_index].name.to_string(),
+                    param_name: param_name.clone(),
                     parent_exists,
                     parent_prefix,
                     parent_name,
                 });
 
+                let mut used_fallback = false;
                 let args = if self.tcx.get_diagnostic_item(sym::iterator_collect_fn)
                     == Some(generics_def_id)
                 {
@@ -634,9 +639,9 @@ pub fn emit_inference_failure_err_with_type_hint(
                     let mut p = fmt_printer(self, Namespace::TypeNS);
                     p.comma_sep(generic_args.iter().copied().map(|arg| {
                         if arg.is_suggestable(self.tcx, true) {
+                            used_fallback = true;
                             return arg;
                         }
-
                         match arg.kind() {
                             GenericArgKind::Lifetime(_) => bug!("unexpected lifetime"),
                             GenericArgKind::Type(_) => self.next_ty_var(DUMMY_SP).into(),
@@ -648,11 +653,31 @@ pub fn emit_inference_failure_err_with_type_hint(
                 };
 
                 if !have_turbofish {
-                    infer_subdiags.push(SourceKindSubdiag::GenericSuggestion {
-                        span: insert_span,
-                        arg_count: generic_args.len(),
-                        args,
-                    });
+                    if generic_args.len() == 1 && used_fallback {
+                        match param.kind {
+                            GenericParamDefKind::Type { .. } => {
+                                infer_subdiags.push(SourceKindSubdiag::GenericTypeSuggestion {
+                                    span: insert_span,
+                                    param: param_name,
+                                });
+                            }
+                            GenericParamDefKind::Const { .. } => {
+                                infer_subdiags.push(SourceKindSubdiag::ConstGenericSuggestion {
+                                    span: insert_span,
+                                    param: param_name,
+                                });
+                            }
+                            GenericParamDefKind::Lifetime => {
+                                bug!("unexpected lifetime")
+                            }
+                        }
+                    } else {
+                        infer_subdiags.push(SourceKindSubdiag::GenericSuggestion {
+                            span: insert_span,
+                            arg_count: generic_args.len(),
+                            args,
+                        });
+                    }
                 }
             }
             InferSourceKind::FullyQualifiedMethodCall { receiver, successor, args, def_id } => {

compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs

@@ -2748,6 +2748,11 @@ pub fn note_obligation_cause(
                 obligation.param_env,
                 obligation.cause.code(),
             );
+            self.suggest_borrow_for_unsized_closure_return(
+                obligation.cause.body_id,
+                err,
+                obligation.predicate,
+            );
             self.suggest_unsized_bound_if_applicable(err, obligation);
             if let Some(span) = err.span.primary_span()
                 && let Some(mut diag) =

compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs

@@ -246,6 +246,15 @@ pub fn suggest_restriction<'tcx, G: EmissionGuarantee>(
     }
 }
 
+/// A single layer of `&` peeled from an expression, used by
+/// [`TypeErrCtxt::peel_expr_refs`].
+struct PeeledRef<'tcx> {
+    /// The span covering the `&` (and any whitespace/mutability keyword) to remove.
+    span: Span,
+    /// The type after peeling this layer (and all prior layers).
+    peeled_ty: Ty<'tcx>,
+}
+
 impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
     pub fn note_field_shadowed_by_private_candidate_in_cause(
         &self,
@@ -1882,6 +1891,85 @@ pub(super) fn suggest_borrowing_for_object_cast(
         );
     }
 
+    /// Peel `&`-borrows from an expression, following through untyped let-bindings.
+    /// Returns a list of removable `&` layers (each with the span to remove and the
+    /// resulting type), plus an optional terminal [`hir::Param`] when the chain ends
+    /// at a function parameter (including async-fn desugared parameters).
+    fn peel_expr_refs(
+        &self,
+        mut expr: &'tcx hir::Expr<'tcx>,
+        mut ty: Ty<'tcx>,
+    ) -> (Vec<PeeledRef<'tcx>>, Option<&'tcx hir::Param<'tcx>>) {
+        let mut refs = Vec::new();
+        'outer: loop {
+            while let hir::ExprKind::AddrOf(_, _, borrowed) = expr.kind {
+                let span =
+                    if let Some(borrowed_span) = borrowed.span.find_ancestor_inside(expr.span) {
+                        expr.span.until(borrowed_span)
+                    } else {
+                        break 'outer;
+                    };
+
+                // Double check that the span actually corresponds to a borrow,
+                // rather than some macro garbage.
+                // The span may include leading parens from parenthesized expressions
+                // (e.g., `(&expr)` where HIR removes the Paren but keeps the span).
+                // In that case, trim the span to start at the `&`.
+                let span = match self.tcx.sess.source_map().span_to_snippet(span) {
+                    Ok(ref snippet) if snippet.starts_with("&") => span,
+                    Ok(ref snippet) if let Some(amp) = snippet.find('&') => {
+                        span.with_lo(span.lo() + BytePos(amp as u32))
+                    }
+                    _ => break 'outer,
+                };
+
+                let ty::Ref(_, inner_ty, _) = ty.kind() else {
+                    break 'outer;
+                };
+                ty = *inner_ty;
+                refs.push(PeeledRef { span, peeled_ty: ty });
+                expr = borrowed;
+            }
+            if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = expr.kind
+                && let Res::Local(hir_id) = path.res
+                && let hir::Node::Pat(binding) = self.tcx.hir_node(hir_id)
+            {
+                match self.tcx.parent_hir_node(binding.hir_id) {
+                    // Untyped let-binding: follow to its initializer.
+                    hir::Node::LetStmt(local)
+                        if local.ty.is_none()
+                            && let Some(init) = local.init =>
+                    {
+                        expr = init;
+                        continue;
+                    }
+                    // Async fn desugared parameter: `let x = __arg0;` with AsyncFn source.
+                    // Follow to the original parameter.
+                    hir::Node::LetStmt(local)
+                        if matches!(local.source, hir::LocalSource::AsyncFn)
+                            && let Some(init) = local.init
+                            && let hir::ExprKind::Path(hir::QPath::Resolved(None, arg_path)) =
+                                init.kind
+                            && let Res::Local(arg_hir_id) = arg_path.res
+                            && let hir::Node::Pat(arg_binding) = self.tcx.hir_node(arg_hir_id)
+                            && let hir::Node::Param(param) =
+                                self.tcx.parent_hir_node(arg_binding.hir_id) =>
+                    {
+                        return (refs, Some(param));
+                    }
+                    // Direct parameter reference.
+                    hir::Node::Param(param) => {
+                        return (refs, Some(param));
+                    }
+                    _ => break 'outer,
+                }
+            } else {
+                break 'outer;
+            }
+        }
+        (refs, None)
+    }
+
     /// Whenever references are used by mistake, like `for (i, e) in &vec.iter().enumerate()`,
     /// suggest removing these references until we reach a type that implements the trait.
     pub(super) fn suggest_remove_reference(
@@ -1958,53 +2046,40 @@ pub(super) fn suggest_remove_reference(
         }
 
         // Maybe suggest removal of borrows from expressions, like in `for i in &&&foo {}`.
-        let Some(mut expr) = expr_finder.result else {
+        let Some(expr) = expr_finder.result else {
             return false;
         };
-        let mut count = 0;
-        let mut suggestions = vec![];
         // Skipping binder here, remapping below
-        let mut suggested_ty = trait_pred.self_ty().skip_binder();
-        'outer: loop {
-            while let hir::ExprKind::AddrOf(_, _, borrowed) = expr.kind {
-                count += 1;
-                let span =
-                    if let Some(borrowed_span) = borrowed.span.find_ancestor_inside(expr.span) {
-                        expr.span.until(borrowed_span)
-                    } else {
-                        break 'outer;
-                    };
+        let suggested_ty = trait_pred.self_ty().skip_binder();
+        let (peeled_refs, _) = self.peel_expr_refs(expr, suggested_ty);
+        for (i, peeled) in peeled_refs.iter().enumerate() {
+            let suggestions: Vec<_> =
+                peeled_refs[..=i].iter().map(|r| (r.span, String::new())).collect();
+            if maybe_suggest(peeled.peeled_ty, i + 1, suggestions) {
+                return true;
+            }
+        }
+        false
+    }
 
-                // Double check that the span we extracted actually corresponds to a borrow,
-                // rather than some macro garbage.
-                match self.tcx.sess.source_map().span_to_snippet(span) {
-                    Ok(snippet) if snippet.starts_with("&") => {}
-                    _ => break 'outer,
-                }
-
-                suggestions.push((span, String::new()));
-
-                let ty::Ref(_, inner_ty, _) = suggested_ty.kind() else {
-                    break 'outer;
-                };
-                suggested_ty = *inner_ty;
-
-                expr = borrowed;
-
-                if maybe_suggest(suggested_ty, count, suggestions.clone()) {
+    /// Suggest removing `&` from a function parameter type like `&impl Future`.
+    fn suggest_remove_ref_from_param(&self, param: &hir::Param<'_>, err: &mut Diag<'_>) -> bool {
+        if let Some(decl) = self.tcx.parent_hir_node(param.hir_id).fn_decl()
+            && let Some(input_ty) = decl.inputs.iter().find(|t| param.ty_span.contains(t.span))
+            && let hir::TyKind::Ref(_, mut_ty) = input_ty.kind
+        {
+            let ref_span = input_ty.span.until(mut_ty.ty.span);
+            match self.tcx.sess.source_map().span_to_snippet(ref_span) {
+                Ok(snippet) if snippet.starts_with("&") => {
+                    err.span_suggestion_verbose(
+                        ref_span,
+                        "consider removing the `&` from the parameter type",
+                        "",
+                        Applicability::MaybeIncorrect,
+                    );
                     return true;
                 }
-            }
-            if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = expr.kind
-                && let Res::Local(hir_id) = path.res
-                && let hir::Node::Pat(binding) = self.tcx.hir_node(hir_id)
-                && let hir::Node::LetStmt(local) = self.tcx.parent_hir_node(binding.hir_id)
-                && let None = local.ty
-                && let Some(binding_expr) = local.init
-            {
-                expr = binding_expr;
-            } else {
-                break 'outer;
+                _ => {}
             }
         }
         false
@@ -2023,6 +2098,89 @@ pub(super) fn suggest_remove_await(
             // could also check if it is an fn call (very likely) and suggest changing *that*, if
             // it is from the local crate.
 
+            // If the type is `&..&T` where `T: Future`, suggest removing `&`
+            // instead of removing `.await`.
+            if let ty::PredicateKind::Clause(ty::ClauseKind::Trait(pred)) =
+                obligation.predicate.kind().skip_binder()
+            {
+                let self_ty = pred.self_ty();
+                let future_trait =
+                    self.tcx.require_lang_item(LangItem::Future, obligation.cause.span);
+
+                // Peel through references to check if there's a Future underneath.
+                let has_future = {
+                    let mut ty = self_ty;
+                    loop {
+                        match *ty.kind() {
+                            ty::Ref(_, inner_ty, _)
+                                if !matches!(inner_ty.kind(), ty::Dynamic(..)) =>
+                            {
+                                if self
+                                    .type_implements_trait(
+                                        future_trait,
+                                        [inner_ty],
+                                        obligation.param_env,
+                                    )
+                                    .must_apply_modulo_regions()
+                                {
+                                    break true;
+                                }
+                                ty = inner_ty;
+                            }
+                            _ => break false,
+                        }
+                    }
+                };
+
+                if has_future {
+                    let (peeled_refs, terminal_param) = self.peel_expr_refs(expr, self_ty);
+
+                    // Try removing `&`s from the expression.
+                    for (i, peeled) in peeled_refs.iter().enumerate() {
+                        if self
+                            .type_implements_trait(
+                                future_trait,
+                                [peeled.peeled_ty],
+                                obligation.param_env,
+                            )
+                            .must_apply_modulo_regions()
+                        {
+                            let count = i + 1;
+                            let msg = if count == 1 {
+                                "consider removing the leading `&`-reference".to_string()
+                            } else {
+                                format!("consider removing {count} leading `&`-references")
+                            };
+                            let suggestions: Vec<_> =
+                                peeled_refs[..=i].iter().map(|r| (r.span, String::new())).collect();
+                            err.multipart_suggestion(
+                                msg,
+                                suggestions,
+                                Applicability::MachineApplicable,
+                            );
+                            return;
+                        }
+                    }
+
+                    // Try removing `&` from the parameter type, but only when there's
+                    // no `&` in the expression itself (otherwise removing from the param
+                    // alone wouldn't fix the error).
+                    if peeled_refs.is_empty()
+                        && let Some(param) = terminal_param
+                        && self.suggest_remove_ref_from_param(param, err)
+                    {
+                        return;
+                    }
+
+                    // Fallback: emit a help message when we can't provide a specific span.
+                    err.help(
+                        "a reference to a future is not a future; \
+                     consider removing the leading `&`-reference",
+                    );
+                    return;
+                }
+            }
+
             // use nth(1) to skip one layer of desugaring from `IntoIter::into_iter`
             if let Some((_, hir::Node::Expr(await_expr))) = self.tcx.hir_parent_iter(*hir_id).nth(1)
                 && let Some(expr_span) = expr.span.find_ancestor_inside_same_ctxt(await_expr.span)
@@ -2196,6 +2354,60 @@ pub(super) fn suggest_semicolon_removal(
         false
     }
 
+    pub(super) fn suggest_borrow_for_unsized_closure_return<G: EmissionGuarantee>(
+        &self,
+        body_id: LocalDefId,
+        err: &mut Diag<'_, G>,
+        predicate: ty::Predicate<'tcx>,
+    ) {
+        let Some(pred) = predicate.as_trait_clause() else {
+            return;
+        };
+        if !self.tcx.is_lang_item(pred.def_id(), LangItem::Sized) {
+            return;
+        }
+
+        let Some(span) = err.span.primary_span() else {
+            return;
+        };
+        let Some(node_body_id) = self.tcx.hir_node_by_def_id(body_id).body_id() else {
+            return;
+        };
+        let body = self.tcx.hir_body(node_body_id);
+        let mut expr_finder = FindExprBySpan::new(span, self.tcx);
+        expr_finder.visit_expr(body.value);
+        let Some(expr) = expr_finder.result else {
+            return;
+        };
+
+        let closure = match expr.kind {
+            hir::ExprKind::Call(_, args) => args.iter().find_map(|arg| match arg.kind {
+                hir::ExprKind::Closure(closure) => Some(closure),
+                _ => None,
+            }),
+            hir::ExprKind::MethodCall(_, _, args, _) => {
+                args.iter().find_map(|arg| match arg.kind {
+                    hir::ExprKind::Closure(closure) => Some(closure),
+                    _ => None,
+                })
+            }
+            _ => None,
+        };
+        let Some(closure) = closure else {
+            return;
+        };
+        if !matches!(closure.fn_decl.output, hir::FnRetTy::DefaultReturn(_)) {
+            return;
+        }
+
+        err.span_suggestion_verbose(
+            self.tcx.hir_body(closure.body).value.span.shrink_to_lo(),
+            "consider borrowing the value",
+            "&",
+            Applicability::MaybeIncorrect,
+        );
+    }
+
     pub(super) fn return_type_span(&self, obligation: &PredicateObligation<'tcx>) -> Option<Span> {
         let hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn { sig, .. }, .. }) =
             self.tcx.hir_node_by_def_id(obligation.cause.body_id)

compiler/rustc_trait_selection/src/errors.rs

@@ -331,6 +331,28 @@ pub(crate) enum SourceKindSubdiag<'a> {
         arg_count: usize,
         args: String,
     },
+    #[suggestion(
+        "consider specifying a concrete type for the type parameter `{$param}`",
+        style = "verbose",
+        code = "::</* Type */>",
+        applicability = "has-placeholders"
+    )]
+    GenericTypeSuggestion {
+        #[primary_span]
+        span: Span,
+        param: String,
+    },
+    #[suggestion(
+        "consider specifying a const for the const parameter `{$param}`",
+        style = "verbose",
+        code = "::</* CONST */>",
+        applicability = "has-placeholders"
+    )]
+    ConstGenericSuggestion {
+        #[primary_span]
+        span: Span,
+        param: String,
+    },
 }
 
 #[derive(Subdiagnostic)]

compiler/rustc_type_ir/src/inherent.rs

@@ -17,6 +17,7 @@
     self as ty, ClauseKind, CollectAndApply, FieldInfo, Interner, PredicateKind, UpcastFrom,
 };
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Ty<I: Interner<Ty = Self>>:
     Copy
     + Debug
@@ -195,6 +196,7 @@ fn is_guaranteed_unsized_raw(self) -> bool {
     }
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Tys<I: Interner<Tys = Self>>:
     Copy + Debug + Hash + Eq + SliceLike<Item = I::Ty> + TypeFoldable<I> + Default
 {
@@ -203,6 +205,7 @@ pub trait Tys<I: Interner<Tys = Self>>:
     fn output(self) -> I::Ty;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait FSigKind<I: Interner<FSigKind = Self>>: Copy + Debug + Hash + Eq {
     /// The identity function.
     fn fn_sig_kind(self) -> Self;
@@ -220,6 +223,7 @@ pub trait FSigKind<I: Interner<FSigKind = Self>>: Copy + Debug + Hash + Eq {
     fn c_variadic(self) -> bool;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Abi<I: Interner<Abi = Self>>: Copy + Debug + Hash + Eq {
     /// The identity function.
     fn abi(self) -> Self;
@@ -237,6 +241,7 @@ pub trait Abi<I: Interner<Abi = Self>>: Copy + Debug + Hash + Eq {
     fn unpack_abi(abi_index: u8) -> Self;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Safety<I: Interner<Safety = Self>>: Copy + Debug + Hash + Eq {
     /// The `safe` safety mode.
     fn safe() -> Self;
@@ -251,6 +256,7 @@ pub trait Safety<I: Interner<Safety = Self>>: Copy + Debug + Hash + Eq {
     fn prefix_str(self) -> &'static str;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Region<I: Interner<Region = Self>>:
     Copy
     + Debug
@@ -276,6 +282,7 @@ fn is_bound(self) -> bool {
     }
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Const<I: Interner<Const = Self>>:
     Copy
     + Debug
@@ -320,19 +327,23 @@ fn is_ct_error(self) -> bool {
     }
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait ValueConst<I: Interner<ValueConst = Self>>: Copy + Debug + Hash + Eq {
     fn ty(self) -> I::Ty;
     fn valtree(self) -> I::ValTree;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait ExprConst<I: Interner<ExprConst = Self>>: Copy + Debug + Hash + Eq + Relate<I> {
     fn args(self) -> I::GenericArgs;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait GenericsOf<I: Interner<GenericsOf = Self>> {
     fn count(&self) -> usize;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait GenericArg<I: Interner<GenericArg = Self>>:
     Copy
     + Debug
@@ -387,6 +398,7 @@ fn is_non_region_infer(self) -> bool {
     }
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Term<I: Interner<Term = Self>>:
     Copy + Debug + Hash + Eq + IntoKind<Kind = ty::TermKind<I>> + TypeFoldable<I> + Relate<I>
 {
@@ -434,6 +446,7 @@ fn to_alias_term(self) -> Option<ty::AliasTerm<I>> {
     }
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait GenericArgs<I: Interner<GenericArgs = Self>>:
     Copy + Debug + Hash + Eq + SliceLike<Item = I::GenericArg> + Default + Relate<I>
 {
@@ -473,6 +486,7 @@ fn as_coroutine(self) -> ty::CoroutineArgs<I> {
     }
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Predicate<I: Interner<Predicate = Self>>:
     Copy
     + Debug
@@ -528,6 +542,7 @@ fn allow_normalization(self) -> bool {
     }
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Clause<I: Interner<Clause = Self>>:
     Copy
     + Debug
@@ -577,6 +592,7 @@ fn as_projection_clause(self) -> Option<ty::Binder<I, ty::ProjectionPredicate<I>
     fn instantiate_supertrait(self, cx: I, trait_ref: ty::Binder<I, ty::TraitRef<I>>) -> Self;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Clauses<I: Interner<Clauses = Self>>:
     Copy
     + Debug
@@ -589,16 +605,19 @@ pub trait Clauses<I: Interner<Clauses = Self>>:
 {
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait IntoKind {
     type Kind;
 
     fn kind(self) -> Self::Kind;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait ParamLike: Copy + Debug + Hash + Eq {
     fn index(self) -> u32;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait AdtDef<I: Interner>: Copy + Debug + Hash + Eq {
     fn def_id(self) -> I::AdtId;
 
@@ -635,10 +654,12 @@ fn sizedness_constraint(
     fn destructor(self, interner: I) -> Option<AdtDestructorKind>;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait ParamEnv<I: Interner>: Copy + Debug + Hash + Eq + TypeFoldable<I> {
     fn caller_bounds(self) -> impl SliceLike<Item = I::Clause>;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Features<I: Interner>: Copy {
     fn generic_const_exprs(self) -> bool;
 
@@ -647,6 +668,7 @@ pub trait Features<I: Interner>: Copy {
     fn feature_bound_holds_in_crate(self, symbol: I::Symbol) -> bool;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait DefId<I: Interner>: Copy + Debug + Hash + Eq + TypeFoldable<I> {
     fn is_local(self) -> bool;
 
@@ -663,6 +685,7 @@ impl<I: Interner, T: DefId<I> + Into<I::DefId> + TryFrom<I::DefId, Error: std::f
 {
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait BoundExistentialPredicates<I: Interner>:
     Copy + Debug + Hash + Eq + Relate<I> + SliceLike<Item = ty::Binder<I, ty::ExistentialPredicate<I>>>
 {
@@ -677,10 +700,12 @@ fn projection_bounds(
     ) -> impl IntoIterator<Item = ty::Binder<I, ty::ExistentialProjection<I>>>;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Span<I: Interner>: Copy + Debug + Hash + Eq + TypeFoldable<I> {
     fn dummy() -> Self;
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait OpaqueTypeStorageEntries: Debug + Copy + Default {
     /// Whether the number of opaques has changed in a way that necessitates
     /// reevaluating a goal. For now, this is only when the number of non-duplicated
@@ -767,6 +792,7 @@ fn as_slice(&self) -> &[Self::Item] {
     }
 }
 
+#[rust_analyzer::prefer_underscore_import]
 pub trait Symbol<I>: Copy + Hash + PartialEq + Eq + Debug {
     fn is_kw_underscore_lifetime(self) -> bool;
 }

library/alloctests/benches/lib.rs

@@ -1,5 +1,6 @@
 // This is marked as `test = true` and hence picked up by `./x miri`, but that would be too slow.
 #![cfg(not(miri))]
+#![allow(internal_features)]
 #![feature(iter_next_chunk)]
 #![feature(repr_simd)]
 #![feature(slice_partition_dedup)]

library/core/src/io/error.rs

@@ -0,0 +1,372 @@
+#![unstable(feature = "core_io", issue = "154046")]
+
+use crate::fmt;
+
+/// A list specifying general categories of I/O error.
+///
+/// This list is intended to grow over time and it is not recommended to
+/// exhaustively match against it.
+///
+/// # Handling errors and matching on `ErrorKind`
+///
+/// In application code, use `match` for the `ErrorKind` values you are
+/// expecting; use `_` to match "all other errors".
+///
+/// In comprehensive and thorough tests that want to verify that a test doesn't
+/// return any known incorrect error kind, you may want to cut-and-paste the
+/// current full list of errors from here into your test code, and then match
+/// `_` as the correct case. This seems counterintuitive, but it will make your
+/// tests more robust. In particular, if you want to verify that your code does
+/// produce an unrecognized error kind, the robust solution is to check for all
+/// the recognized error kinds and fail in those cases.
+#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
+#[stable(feature = "rust1", since = "1.0.0")]
+#[cfg_attr(not(test), rustc_diagnostic_item = "io_errorkind")]
+#[allow(deprecated)]
+#[non_exhaustive]
+pub enum ErrorKind {
+    /// An entity was not found, often a file.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    NotFound,
+    /// The operation lacked the necessary privileges to complete.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    PermissionDenied,
+    /// The connection was refused by the remote server.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    ConnectionRefused,
+    /// The connection was reset by the remote server.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    ConnectionReset,
+    /// The remote host is not reachable.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    HostUnreachable,
+    /// The network containing the remote host is not reachable.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    NetworkUnreachable,
+    /// The connection was aborted (terminated) by the remote server.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    ConnectionAborted,
+    /// The network operation failed because it was not connected yet.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    NotConnected,
+    /// A socket address could not be bound because the address is already in
+    /// use elsewhere.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    AddrInUse,
+    /// A nonexistent interface was requested or the requested address was not
+    /// local.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    AddrNotAvailable,
+    /// The system's networking is down.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    NetworkDown,
+    /// The operation failed because a pipe was closed.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    BrokenPipe,
+    /// An entity already exists, often a file.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    AlreadyExists,
+    /// The operation needs to block to complete, but the blocking operation was
+    /// requested to not occur.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    WouldBlock,
+    /// A filesystem object is, unexpectedly, not a directory.
+    ///
+    /// For example, a filesystem path was specified where one of the intermediate directory
+    /// components was, in fact, a plain file.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    NotADirectory,
+    /// The filesystem object is, unexpectedly, a directory.
+    ///
+    /// A directory was specified when a non-directory was expected.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    IsADirectory,
+    /// A non-empty directory was specified where an empty directory was expected.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    DirectoryNotEmpty,
+    /// The filesystem or storage medium is read-only, but a write operation was attempted.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    ReadOnlyFilesystem,
+    /// Loop in the filesystem or IO subsystem; often, too many levels of symbolic links.
+    ///
+    /// There was a loop (or excessively long chain) resolving a filesystem object
+    /// or file IO object.
+    ///
+    /// On Unix this is usually the result of a symbolic link loop; or, of exceeding the
+    /// system-specific limit on the depth of symlink traversal.
+    #[unstable(feature = "io_error_more", issue = "86442")]
+    FilesystemLoop,
+    /// Stale network file handle.
+    ///
+    /// With some network filesystems, notably NFS, an open file (or directory) can be invalidated
+    /// by problems with the network or server.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    StaleNetworkFileHandle,
+    /// A parameter was incorrect.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    InvalidInput,
+    /// Data not valid for the operation were encountered.
+    ///
+    /// Unlike [`InvalidInput`], this typically means that the operation
+    /// parameters were valid, however the error was caused by malformed
+    /// input data.
+    ///
+    /// For example, a function that reads a file into a string will error with
+    /// `InvalidData` if the file's contents are not valid UTF-8.
+    ///
+    /// [`InvalidInput`]: ErrorKind::InvalidInput
+    #[stable(feature = "io_invalid_data", since = "1.2.0")]
+    InvalidData,
+    /// The I/O operation's timeout expired, causing it to be canceled.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    TimedOut,
+    /// An error returned when an operation could not be completed because a
+    /// call to an underlying writer returned [`Ok(0)`].
+    ///
+    /// This typically means that an operation could only succeed if it wrote a
+    /// particular number of bytes but only a smaller number of bytes could be
+    /// written.
+    ///
+    /// [`Ok(0)`]: Ok
+    #[stable(feature = "rust1", since = "1.0.0")]
+    WriteZero,
+    /// The underlying storage (typically, a filesystem) is full.
+    ///
+    /// This does not include out of quota errors.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    StorageFull,
+    /// Seek on unseekable file.
+    ///
+    /// Seeking was attempted on an open file handle which is not suitable for seeking - for
+    /// example, on Unix, a named pipe opened with `File::open`.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    NotSeekable,
+    /// Filesystem quota or some other kind of quota was exceeded.
+    #[stable(feature = "io_error_quota_exceeded", since = "1.85.0")]
+    QuotaExceeded,
+    /// File larger than allowed or supported.
+    ///
+    /// This might arise from a hard limit of the underlying filesystem or file access API, or from
+    /// an administratively imposed resource limitation.  Simple disk full, and out of quota, have
+    /// their own errors.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    FileTooLarge,
+    /// Resource is busy.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    ResourceBusy,
+    /// Executable file is busy.
+    ///
+    /// An attempt was made to write to a file which is also in use as a running program.  (Not all
+    /// operating systems detect this situation.)
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    ExecutableFileBusy,
+    /// Deadlock (avoided).
+    ///
+    /// A file locking operation would result in deadlock.  This situation is typically detected, if
+    /// at all, on a best-effort basis.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    Deadlock,
+    /// Cross-device or cross-filesystem (hard) link or rename.
+    #[stable(feature = "io_error_crosses_devices", since = "1.85.0")]
+    CrossesDevices,
+    /// Too many (hard) links to the same filesystem object.
+    ///
+    /// The filesystem does not support making so many hardlinks to the same file.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    TooManyLinks,
+    /// A filename was invalid.
+    ///
+    /// This error can also occur if a length limit for a name was exceeded.
+    #[stable(feature = "io_error_invalid_filename", since = "1.87.0")]
+    InvalidFilename,
+    /// Program argument list too long.
+    ///
+    /// When trying to run an external program, a system or process limit on the size of the
+    /// arguments would have been exceeded.
+    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
+    ArgumentListTooLong,
+    /// This operation was interrupted.
+    ///
+    /// Interrupted operations can typically be retried.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    Interrupted,
+
+    /// This operation is unsupported on this platform.
+    ///
+    /// This means that the operation can never succeed.
+    #[stable(feature = "unsupported_error", since = "1.53.0")]
+    Unsupported,
+
+    // ErrorKinds which are primarily categorisations for OS error
+    // codes should be added above.
+    //
+    /// An error returned when an operation could not be completed because an
+    /// "end of file" was reached prematurely.
+    ///
+    /// This typically means that an operation could only succeed if it read a
+    /// particular number of bytes but only a smaller number of bytes could be
+    /// read.
+    #[stable(feature = "read_exact", since = "1.6.0")]
+    UnexpectedEof,
+
+    /// An operation could not be completed, because it failed
+    /// to allocate enough memory.
+    #[stable(feature = "out_of_memory_error", since = "1.54.0")]
+    OutOfMemory,
+
+    /// The operation was partially successful and needs to be checked
+    /// later on due to not blocking.
+    #[unstable(feature = "io_error_inprogress", issue = "130840")]
+    InProgress,
+
+    // "Unusual" error kinds which do not correspond simply to (sets
+    // of) OS error codes, should be added just above this comment.
+    // `Other` and `Uncategorized` should remain at the end:
+    //
+    /// A custom error that does not fall under any other I/O error kind.
+    ///
+    /// This can be used to construct your own errors that do not match any
+    /// [`ErrorKind`].
+    ///
+    /// This [`ErrorKind`] is not used by the standard library.
+    ///
+    /// Errors from the standard library that do not fall under any of the I/O
+    /// error kinds cannot be `match`ed on, and will only match a wildcard (`_`) pattern.
+    /// New [`ErrorKind`]s might be added in the future for some of those.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    Other,
+
+    /// Any I/O error from the standard library that's not part of this list.
+    ///
+    /// Errors that are `Uncategorized` now may move to a different or a new
+    /// [`ErrorKind`] variant in the future. It is not recommended to match
+    /// an error against `Uncategorized`; use a wildcard match (`_`) instead.
+    #[unstable(feature = "io_error_uncategorized", issue = "none")]
+    #[doc(hidden)]
+    Uncategorized,
+}
+
+impl ErrorKind {
+    pub(crate) const fn as_str(&self) -> &'static str {
+        use ErrorKind::*;
+        match *self {
+            // tidy-alphabetical-start
+            AddrInUse => "address in use",
+            AddrNotAvailable => "address not available",
+            AlreadyExists => "entity already exists",
+            ArgumentListTooLong => "argument list too long",
+            BrokenPipe => "broken pipe",
+            ConnectionAborted => "connection aborted",
+            ConnectionRefused => "connection refused",
+            ConnectionReset => "connection reset",
+            CrossesDevices => "cross-device link or rename",
+            Deadlock => "deadlock",
+            DirectoryNotEmpty => "directory not empty",
+            ExecutableFileBusy => "executable file busy",
+            FileTooLarge => "file too large",
+            FilesystemLoop => "filesystem loop or indirection limit (e.g. symlink loop)",
+            HostUnreachable => "host unreachable",
+            InProgress => "in progress",
+            Interrupted => "operation interrupted",
+            InvalidData => "invalid data",
+            InvalidFilename => "invalid filename",
+            InvalidInput => "invalid input parameter",
+            IsADirectory => "is a directory",
+            NetworkDown => "network down",
+            NetworkUnreachable => "network unreachable",
+            NotADirectory => "not a directory",
+            NotConnected => "not connected",
+            NotFound => "entity not found",
+            NotSeekable => "seek on unseekable file",
+            Other => "other error",
+            OutOfMemory => "out of memory",
+            PermissionDenied => "permission denied",
+            QuotaExceeded => "quota exceeded",
+            ReadOnlyFilesystem => "read-only filesystem or storage medium",
+            ResourceBusy => "resource busy",
+            StaleNetworkFileHandle => "stale network file handle",
+            StorageFull => "no storage space",
+            TimedOut => "timed out",
+            TooManyLinks => "too many links",
+            Uncategorized => "uncategorized error",
+            UnexpectedEof => "unexpected end of file",
+            Unsupported => "unsupported",
+            WouldBlock => "operation would block",
+            WriteZero => "write zero",
+            // tidy-alphabetical-end
+        }
+    }
+
+    // This compiles to the same code as the check+transmute, but doesn't require
+    // unsafe, or to hard-code max ErrorKind or its size in a way the compiler
+    // couldn't verify.
+    #[inline]
+    #[unstable(feature = "core_io_internals", reason = "exposed only for libstd", issue = "none")]
+    #[doc(hidden)]
+    pub const fn from_prim(ek: u32) -> Option<Self> {
+        macro_rules! from_prim {
+            ($prim:expr => $Enum:ident { $($Variant:ident),* $(,)? }) => {{
+                // Force a compile error if the list gets out of date.
+                const _: fn(e: $Enum) = |e: $Enum| match e {
+                    $($Enum::$Variant => (),)*
+                };
+                match $prim {
+                    $(v if v == ($Enum::$Variant as _) => Some($Enum::$Variant),)*
+                    _ => None,
+                }
+            }}
+        }
+        from_prim!(ek => ErrorKind {
+            NotFound,
+            PermissionDenied,
+            ConnectionRefused,
+            ConnectionReset,
+            HostUnreachable,
+            NetworkUnreachable,
+            ConnectionAborted,
+            NotConnected,
+            AddrInUse,
+            AddrNotAvailable,
+            NetworkDown,
+            BrokenPipe,
+            AlreadyExists,
+            WouldBlock,
+            NotADirectory,
+            IsADirectory,
+            DirectoryNotEmpty,
+            ReadOnlyFilesystem,
+            FilesystemLoop,
+            StaleNetworkFileHandle,
+            InvalidInput,
+            InvalidData,
+            TimedOut,
+            WriteZero,
+            StorageFull,
+            NotSeekable,
+            QuotaExceeded,
+            FileTooLarge,
+            ResourceBusy,
+            ExecutableFileBusy,
+            Deadlock,
+            CrossesDevices,
+            TooManyLinks,
+            InvalidFilename,
+            ArgumentListTooLong,
+            Interrupted,
+            Other,
+            UnexpectedEof,
+            Unsupported,
+            OutOfMemory,
+            InProgress,
+            Uncategorized,
+        })
+    }
+}
+
+#[stable(feature = "io_errorkind_display", since = "1.60.0")]
+impl fmt::Display for ErrorKind {
+    /// Shows a human-readable description of the `ErrorKind`.
+    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt.write_str(self.as_str())
+    }
+}

library/core/src/io/mod.rs

@@ -1,6 +1,9 @@
 //! Traits, helpers, and type definitions for core I/O functionality.
 
 mod borrowed_buf;
+mod error;
 
 #[unstable(feature = "core_io_borrowed_buf", issue = "117693")]
 pub use self::borrowed_buf::{BorrowedBuf, BorrowedCursor};
+#[unstable(feature = "core_io", issue = "154046")]
+pub use self::error::ErrorKind;

library/core/src/lib.rs

@@ -305,7 +305,7 @@ pub mod autodiff {
 pub mod cell;
 pub mod char;
 pub mod ffi;
-#[unstable(feature = "core_io_borrowed_buf", issue = "117693")]
+#[unstable(feature = "core_io", issue = "154046")]
 pub mod io;
 pub mod iter;
 pub mod net;

library/coretests/tests/lib.rs

@@ -36,6 +36,7 @@
 #![feature(const_unsigned_bigint_helpers)]
 #![feature(core_intrinsics)]
 #![feature(core_intrinsics_fallbacks)]
+#![feature(core_io)]
 #![feature(core_io_borrowed_buf)]
 #![feature(core_private_bignum)]
 #![feature(core_private_diy_float)]

library/std/src/io/error.rs

@@ -1,6 +1,9 @@
 #[cfg(test)]
 mod tests;
 
+#[stable(feature = "rust1", since = "1.0.0")]
+pub use core::io::ErrorKind;
+
 // On 64-bit platforms, `io::Error` may use a bit-packed representation to
 // reduce size. However, this representation assumes that error codes are
 // always 32-bit wide.
@@ -206,323 +209,6 @@ struct Custom {
     error: Box<dyn error::Error + Send + Sync>,
 }
 
-/// A list specifying general categories of I/O error.
-///
-/// This list is intended to grow over time and it is not recommended to
-/// exhaustively match against it.
-///
-/// It is used with the [`io::Error`] type.
-///
-/// [`io::Error`]: Error
-///
-/// # Handling errors and matching on `ErrorKind`
-///
-/// In application code, use `match` for the `ErrorKind` values you are
-/// expecting; use `_` to match "all other errors".
-///
-/// In comprehensive and thorough tests that want to verify that a test doesn't
-/// return any known incorrect error kind, you may want to cut-and-paste the
-/// current full list of errors from here into your test code, and then match
-/// `_` as the correct case. This seems counterintuitive, but it will make your
-/// tests more robust. In particular, if you want to verify that your code does
-/// produce an unrecognized error kind, the robust solution is to check for all
-/// the recognized error kinds and fail in those cases.
-#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
-#[stable(feature = "rust1", since = "1.0.0")]
-#[cfg_attr(not(test), rustc_diagnostic_item = "io_errorkind")]
-#[allow(deprecated)]
-#[non_exhaustive]
-pub enum ErrorKind {
-    /// An entity was not found, often a file.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    NotFound,
-    /// The operation lacked the necessary privileges to complete.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    PermissionDenied,
-    /// The connection was refused by the remote server.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    ConnectionRefused,
-    /// The connection was reset by the remote server.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    ConnectionReset,
-    /// The remote host is not reachable.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    HostUnreachable,
-    /// The network containing the remote host is not reachable.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    NetworkUnreachable,
-    /// The connection was aborted (terminated) by the remote server.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    ConnectionAborted,
-    /// The network operation failed because it was not connected yet.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    NotConnected,
-    /// A socket address could not be bound because the address is already in
-    /// use elsewhere.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    AddrInUse,
-    /// A nonexistent interface was requested or the requested address was not
-    /// local.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    AddrNotAvailable,
-    /// The system's networking is down.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    NetworkDown,
-    /// The operation failed because a pipe was closed.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    BrokenPipe,
-    /// An entity already exists, often a file.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    AlreadyExists,
-    /// The operation needs to block to complete, but the blocking operation was
-    /// requested to not occur.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    WouldBlock,
-    /// A filesystem object is, unexpectedly, not a directory.
-    ///
-    /// For example, a filesystem path was specified where one of the intermediate directory
-    /// components was, in fact, a plain file.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    NotADirectory,
-    /// The filesystem object is, unexpectedly, a directory.
-    ///
-    /// A directory was specified when a non-directory was expected.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    IsADirectory,
-    /// A non-empty directory was specified where an empty directory was expected.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    DirectoryNotEmpty,
-    /// The filesystem or storage medium is read-only, but a write operation was attempted.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    ReadOnlyFilesystem,
-    /// Loop in the filesystem or IO subsystem; often, too many levels of symbolic links.
-    ///
-    /// There was a loop (or excessively long chain) resolving a filesystem object
-    /// or file IO object.
-    ///
-    /// On Unix this is usually the result of a symbolic link loop; or, of exceeding the
-    /// system-specific limit on the depth of symlink traversal.
-    #[unstable(feature = "io_error_more", issue = "86442")]
-    FilesystemLoop,
-    /// Stale network file handle.
-    ///
-    /// With some network filesystems, notably NFS, an open file (or directory) can be invalidated
-    /// by problems with the network or server.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    StaleNetworkFileHandle,
-    /// A parameter was incorrect.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    InvalidInput,
-    /// Data not valid for the operation were encountered.
-    ///
-    /// Unlike [`InvalidInput`], this typically means that the operation
-    /// parameters were valid, however the error was caused by malformed
-    /// input data.
-    ///
-    /// For example, a function that reads a file into a string will error with
-    /// `InvalidData` if the file's contents are not valid UTF-8.
-    ///
-    /// [`InvalidInput`]: ErrorKind::InvalidInput
-    #[stable(feature = "io_invalid_data", since = "1.2.0")]
-    InvalidData,
-    /// The I/O operation's timeout expired, causing it to be canceled.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    TimedOut,
-    /// An error returned when an operation could not be completed because a
-    /// call to [`write`] returned [`Ok(0)`].
-    ///
-    /// This typically means that an operation could only succeed if it wrote a
-    /// particular number of bytes but only a smaller number of bytes could be
-    /// written.
-    ///
-    /// [`write`]: crate::io::Write::write
-    /// [`Ok(0)`]: Ok
-    #[stable(feature = "rust1", since = "1.0.0")]
-    WriteZero,
-    /// The underlying storage (typically, a filesystem) is full.
-    ///
-    /// This does not include out of quota errors.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    StorageFull,
-    /// Seek on unseekable file.
-    ///
-    /// Seeking was attempted on an open file handle which is not suitable for seeking - for
-    /// example, on Unix, a named pipe opened with `File::open`.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    NotSeekable,
-    /// Filesystem quota or some other kind of quota was exceeded.
-    #[stable(feature = "io_error_quota_exceeded", since = "1.85.0")]
-    QuotaExceeded,
-    /// File larger than allowed or supported.
-    ///
-    /// This might arise from a hard limit of the underlying filesystem or file access API, or from
-    /// an administratively imposed resource limitation.  Simple disk full, and out of quota, have
-    /// their own errors.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    FileTooLarge,
-    /// Resource is busy.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    ResourceBusy,
-    /// Executable file is busy.
-    ///
-    /// An attempt was made to write to a file which is also in use as a running program.  (Not all
-    /// operating systems detect this situation.)
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    ExecutableFileBusy,
-    /// Deadlock (avoided).
-    ///
-    /// A file locking operation would result in deadlock.  This situation is typically detected, if
-    /// at all, on a best-effort basis.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    Deadlock,
-    /// Cross-device or cross-filesystem (hard) link or rename.
-    #[stable(feature = "io_error_crosses_devices", since = "1.85.0")]
-    CrossesDevices,
-    /// Too many (hard) links to the same filesystem object.
-    ///
-    /// The filesystem does not support making so many hardlinks to the same file.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    TooManyLinks,
-    /// A filename was invalid.
-    ///
-    /// This error can also occur if a length limit for a name was exceeded.
-    #[stable(feature = "io_error_invalid_filename", since = "1.87.0")]
-    InvalidFilename,
-    /// Program argument list too long.
-    ///
-    /// When trying to run an external program, a system or process limit on the size of the
-    /// arguments would have been exceeded.
-    #[stable(feature = "io_error_a_bit_more", since = "1.83.0")]
-    ArgumentListTooLong,
-    /// This operation was interrupted.
-    ///
-    /// Interrupted operations can typically be retried.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    Interrupted,
-
-    /// This operation is unsupported on this platform.
-    ///
-    /// This means that the operation can never succeed.
-    #[stable(feature = "unsupported_error", since = "1.53.0")]
-    Unsupported,
-
-    // ErrorKinds which are primarily categorisations for OS error
-    // codes should be added above.
-    //
-    /// An error returned when an operation could not be completed because an
-    /// "end of file" was reached prematurely.
-    ///
-    /// This typically means that an operation could only succeed if it read a
-    /// particular number of bytes but only a smaller number of bytes could be
-    /// read.
-    #[stable(feature = "read_exact", since = "1.6.0")]
-    UnexpectedEof,
-
-    /// An operation could not be completed, because it failed
-    /// to allocate enough memory.
-    #[stable(feature = "out_of_memory_error", since = "1.54.0")]
-    OutOfMemory,
-
-    /// The operation was partially successful and needs to be checked
-    /// later on due to not blocking.
-    #[unstable(feature = "io_error_inprogress", issue = "130840")]
-    InProgress,
-
-    // "Unusual" error kinds which do not correspond simply to (sets
-    // of) OS error codes, should be added just above this comment.
-    // `Other` and `Uncategorized` should remain at the end:
-    //
-    /// A custom error that does not fall under any other I/O error kind.
-    ///
-    /// This can be used to construct your own [`Error`]s that do not match any
-    /// [`ErrorKind`].
-    ///
-    /// This [`ErrorKind`] is not used by the standard library.
-    ///
-    /// Errors from the standard library that do not fall under any of the I/O
-    /// error kinds cannot be `match`ed on, and will only match a wildcard (`_`) pattern.
-    /// New [`ErrorKind`]s might be added in the future for some of those.
-    #[stable(feature = "rust1", since = "1.0.0")]
-    Other,
-
-    /// Any I/O error from the standard library that's not part of this list.
-    ///
-    /// Errors that are `Uncategorized` now may move to a different or a new
-    /// [`ErrorKind`] variant in the future. It is not recommended to match
-    /// an error against `Uncategorized`; use a wildcard match (`_`) instead.
-    #[unstable(feature = "io_error_uncategorized", issue = "none")]
-    #[doc(hidden)]
-    Uncategorized,
-}
-
-impl ErrorKind {
-    pub(crate) fn as_str(&self) -> &'static str {
-        use ErrorKind::*;
-        match *self {
-            // tidy-alphabetical-start
-            AddrInUse => "address in use",
-            AddrNotAvailable => "address not available",
-            AlreadyExists => "entity already exists",
-            ArgumentListTooLong => "argument list too long",
-            BrokenPipe => "broken pipe",
-            ConnectionAborted => "connection aborted",
-            ConnectionRefused => "connection refused",
-            ConnectionReset => "connection reset",
-            CrossesDevices => "cross-device link or rename",
-            Deadlock => "deadlock",
-            DirectoryNotEmpty => "directory not empty",
-            ExecutableFileBusy => "executable file busy",
-            FileTooLarge => "file too large",
-            FilesystemLoop => "filesystem loop or indirection limit (e.g. symlink loop)",
-            HostUnreachable => "host unreachable",
-            InProgress => "in progress",
-            Interrupted => "operation interrupted",
-            InvalidData => "invalid data",
-            InvalidFilename => "invalid filename",
-            InvalidInput => "invalid input parameter",
-            IsADirectory => "is a directory",
-            NetworkDown => "network down",
-            NetworkUnreachable => "network unreachable",
-            NotADirectory => "not a directory",
-            NotConnected => "not connected",
-            NotFound => "entity not found",
-            NotSeekable => "seek on unseekable file",
-            Other => "other error",
-            OutOfMemory => "out of memory",
-            PermissionDenied => "permission denied",
-            QuotaExceeded => "quota exceeded",
-            ReadOnlyFilesystem => "read-only filesystem or storage medium",
-            ResourceBusy => "resource busy",
-            StaleNetworkFileHandle => "stale network file handle",
-            StorageFull => "no storage space",
-            TimedOut => "timed out",
-            TooManyLinks => "too many links",
-            Uncategorized => "uncategorized error",
-            UnexpectedEof => "unexpected end of file",
-            Unsupported => "unsupported",
-            WouldBlock => "operation would block",
-            WriteZero => "write zero",
-            // tidy-alphabetical-end
-        }
-    }
-}
-
-#[stable(feature = "io_errorkind_display", since = "1.60.0")]
-impl fmt::Display for ErrorKind {
-    /// Shows a human-readable description of the `ErrorKind`.
-    ///
-    /// This is similar to `impl Display for Error`, but doesn't require first converting to Error.
-    ///
-    /// # Examples
-    /// ```
-    /// use std::io::ErrorKind;
-    /// assert_eq!("entity not found", ErrorKind::NotFound.to_string());
-    /// ```
-    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt.write_str(self.as_str())
-    }
-}
-
 /// Intended for use for errors not exposed to the user, where allocating onto
 /// the heap (for normal construction via Error::new) is too costly.
 #[stable(feature = "io_error_from_errorkind", since = "1.14.0")]
@@ -1051,7 +737,7 @@ fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
                 write!(fmt, "{detail} (os error {code})")
             }
             ErrorData::Custom(ref c) => c.error.fmt(fmt),
-            ErrorData::Simple(kind) => write!(fmt, "{}", kind.as_str()),
+            ErrorData::Simple(kind) => kind.fmt(fmt),
             ErrorData::SimpleMessage(msg) => msg.message.fmt(fmt),
         }
     }

library/std/src/io/error/repr_bitpacked.rs

@@ -253,7 +253,7 @@ unsafe fn decode_repr<C, F>(ptr: NonNull<()>, make_custom: F) -> ErrorData<C>
         }
         TAG_SIMPLE => {
             let kind_bits = (bits >> 32) as u32;
-            let kind = kind_from_prim(kind_bits).unwrap_or_else(|| {
+            let kind = ErrorKind::from_prim(kind_bits).unwrap_or_else(|| {
                 debug_assert!(false, "Invalid io::error::Repr bits: `Repr({:#018x})`", bits);
                 // This means the `ptr` passed in was not valid, which violates
                 // the unsafe contract of `decode_repr`.
@@ -283,69 +283,6 @@ unsafe fn decode_repr<C, F>(ptr: NonNull<()>, make_custom: F) -> ErrorData<C>
     }
 }
 
-// This compiles to the same code as the check+transmute, but doesn't require
-// unsafe, or to hard-code max ErrorKind or its size in a way the compiler
-// couldn't verify.
-#[inline]
-fn kind_from_prim(ek: u32) -> Option<ErrorKind> {
-    macro_rules! from_prim {
-        ($prim:expr => $Enum:ident { $($Variant:ident),* $(,)? }) => {{
-            // Force a compile error if the list gets out of date.
-            const _: fn(e: $Enum) = |e: $Enum| match e {
-                $($Enum::$Variant => ()),*
-            };
-            match $prim {
-                $(v if v == ($Enum::$Variant as _) => Some($Enum::$Variant),)*
-                _ => None,
-            }
-        }}
-    }
-    from_prim!(ek => ErrorKind {
-        NotFound,
-        PermissionDenied,
-        ConnectionRefused,
-        ConnectionReset,
-        HostUnreachable,
-        NetworkUnreachable,
-        ConnectionAborted,
-        NotConnected,
-        AddrInUse,
-        AddrNotAvailable,
-        NetworkDown,
-        BrokenPipe,
-        AlreadyExists,
-        WouldBlock,
-        NotADirectory,
-        IsADirectory,
-        DirectoryNotEmpty,
-        ReadOnlyFilesystem,
-        FilesystemLoop,
-        StaleNetworkFileHandle,
-        InvalidInput,
-        InvalidData,
-        TimedOut,
-        WriteZero,
-        StorageFull,
-        NotSeekable,
-        QuotaExceeded,
-        FileTooLarge,
-        ResourceBusy,
-        ExecutableFileBusy,
-        Deadlock,
-        CrossesDevices,
-        TooManyLinks,
-        InvalidFilename,
-        ArgumentListTooLong,
-        Interrupted,
-        Other,
-        UnexpectedEof,
-        Unsupported,
-        OutOfMemory,
-        InProgress,
-        Uncategorized,
-    })
-}
-
 // Some static checking to alert us if a change breaks any of the assumptions
 // that our encoding relies on for correctness and soundness. (Some of these are
 // a bit overly thorough/cautious, admittedly)

library/std/src/lib.rs

@@ -321,7 +321,9 @@
 #![feature(const_default)]
 #![feature(core_float_math)]
 #![feature(core_intrinsics)]
+#![feature(core_io)]
 #![feature(core_io_borrowed_buf)]
+#![feature(core_io_internals)]
 #![feature(cstr_display)]
 #![feature(drop_guard)]
 #![feature(duration_constants)]
@@ -344,6 +346,9 @@
 #![feature(hashmap_internals)]
 #![feature(hint_must_use)]
 #![feature(int_from_ascii)]
+#![feature(io_error_inprogress)]
+#![feature(io_error_more)]
+#![feature(io_error_uncategorized)]
 #![feature(ip)]
 #![feature(iter_advance_by)]
 #![feature(iter_next_chunk)]

library/std/src/sys/io/error/sgx.rs

@@ -60,6 +60,6 @@ pub fn error_string(errno: i32) -> String {
     } else if ((Error::UserRangeStart as _)..=(Error::UserRangeEnd as _)).contains(&errno) {
         format!("user-specified error {errno:08x}")
     } else {
-        decode_error_kind(errno).as_str().into()
+        format!("{}", decode_error_kind(errno))
     }
 }

src/bootstrap/src/core/config/config.rs

@@ -2196,6 +2196,42 @@ pub fn check_stage0_version(
     }
 }
 
+fn print_rustc_modifications(
+    dwn_ctx: &DownloadContext<'_>,
+    if_unchanged: bool,
+    mut modifications: Vec<PathBuf>,
+) -> Option<()> {
+    if !dwn_ctx.exec_ctx.is_verbose() {
+        modifications.retain(|path| !path.starts_with("compiler"));
+    }
+    if modifications.is_empty() {
+        // only compiler changes; still force a rebuild but don't say why.
+        eprintln!(
+            "skipping rustc download with `download-rustc = 'if-unchanged'` due to local changes"
+        );
+        return None;
+    }
+
+    eprintln!(
+        "NOTE: detected {} modifications that could affect a build of rustc",
+        modifications.len()
+    );
+    for file in modifications.iter().take(10) {
+        eprintln!("- {}", file.display());
+    }
+    if modifications.len() > 10 {
+        eprintln!("- ... and {} more", modifications.len() - 10);
+    }
+
+    if if_unchanged {
+        eprintln!("skipping rustc download due to `download-rustc = 'if-unchanged'`");
+        None
+    } else {
+        eprintln!("downloading unconditionally due to `download-rustc = true`");
+        Some(())
+    }
+}
+
 pub fn download_ci_rustc_commit<'a>(
     dwn_ctx: impl AsRef<DownloadContext<'a>>,
     rust_info: &channel::GitInfo,
@@ -2250,24 +2286,7 @@ pub fn download_ci_rustc_commit<'a>(
                     return None;
                 }
 
-                eprintln!(
-                    "NOTE: detected {} modifications that could affect a build of rustc",
-                    modifications.len()
-                );
-                for file in modifications.iter().take(10) {
-                    eprintln!("- {}", file.display());
-                }
-                if modifications.len() > 10 {
-                    eprintln!("- ... and {} more", modifications.len() - 10);
-                }
-
-                if if_unchanged {
-                    eprintln!("skipping rustc download due to `download-rustc = 'if-unchanged'`");
-                    return None;
-                } else {
-                    eprintln!("downloading unconditionally due to `download-rustc = true`");
-                }
-
+                print_rustc_modifications(dwn_ctx, if_unchanged, modifications)?;
                 upstream
             }
             PathFreshness::MissingUpstream => {

src/doc/rustc-dev-guide/rust-version

@@ -1 +1 @@
-d12e1e12ae6fb828df1b663a07c9191bc48c6d6d
+84c11900724736a2b9afac4822031d78a9ed8e86

src/doc/rustc-dev-guide/src/offload/installation.md

@@ -30,7 +30,7 @@ git clone git@github.com:llvm/llvm-project
 cd llvm-project
 mkdir build
 cd build
-cmake -G Ninja ../llvm -DLLVM_TARGETS_TO_BUILD="host,AMDGPU,NVPTX" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="offload,openmp" -DLLVM_ENABLE_PLUGINS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=.
+cmake -G Ninja ../llvm -DLLVM_TARGETS_TO_BUILD="host;AMDGPU;NVPTX" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="offload;openmp" -DLLVM_ENABLE_PLUGINS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=.
 ninja
 ninja install
 ```

src/tools/miri/Cargo.lock

@@ -1162,9 +1162,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
 
 [[package]]
 name = "rand"
-version = "0.9.2"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
+checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166"
 dependencies = [
  "rand_chacha",
  "rand_core",

src/tools/miri/cargo-miri/src/phases.rs

@@ -87,7 +87,9 @@ pub fn phase_cargo_miri(mut args: impl Iterator<Item = String>) {
                 println!("`cargo miri {verb}` supports the same flags as `cargo {verb}`:\n");
                 let mut cmd = cargo();
                 cmd.arg(verb);
-                cmd.arg("--help");
+                // Forward all arguments (some of them can influence the help output, e.g.
+                // the nextest verb).
+                cmd.args(args);
                 exec(cmd);
             }
             _ => {

src/tools/miri/genmc-sys/build.rs

@@ -28,7 +28,7 @@ mod downloading {
     /// The GenMC repository the we get our commit from.
     pub(crate) const GENMC_GITHUB_URL: &str = "https://github.com/MPI-SWS/genmc.git";
     /// The GenMC commit we depend on. It must be available on the specified GenMC repository.
-    pub(crate) const GENMC_COMMIT: &str = "22d3d0b44dedb4e8e1aae3330e546465e4664529";
+    pub(crate) const GENMC_COMMIT: &str = "29b03a66402c4453fc77901ef3be90bb55707cd4";
 
     /// Ensure that a local GenMC repo is present and set to the correct commit.
     /// Return the path of the GenMC repo clone.
@@ -159,6 +159,7 @@ fn compile_cpp_dependencies(genmc_path: &Path) {
         .out_dir(genmc_build_dir)
         .profile(GENMC_CMAKE_PROFILE)
         .define("BUILD_LLI", "OFF")
+        .define("EMIT_NA_LABELS", "OFF")
         .define("GENMC_DEBUG", if enable_genmc_debug { "ON" } else { "OFF" });
 
     // The actual compilation happens here:
@@ -172,7 +173,7 @@ fn compile_cpp_dependencies(genmc_path: &Path) {
     // Part 2:
     // Compile the cxx_bridge (the link between the Rust and C++ code).
 
-    let genmc_include_dir = genmc_install_dir.join("include").join("genmc");
+    let genmc_include_dir = genmc_install_dir.join("include");
 
     // These are all the C++ files we need to compile, which needs to be updated if more C++ files are added to Miri.
     // We use absolute paths since relative paths can confuse IDEs when attempting to go-to-source on a path in a compiler error.
@@ -181,10 +182,6 @@ fn compile_cpp_dependencies(genmc_path: &Path) {
         .map(|file| std::path::absolute(cpp_files_base_path.join(file)).unwrap());
 
     let mut bridge = cxx_build::bridge("src/lib.rs");
-    // FIXME(genmc,cmake): Remove once the GenMC debug setting is available in the config.h file.
-    if enable_genmc_debug {
-        bridge.define("ENABLE_GENMC_DEBUG", None);
-    }
     bridge
         .opt_level(2)
         .debug(true) // Same settings that GenMC uses (default for cmake `RelWithDebInfo`)

src/tools/miri/genmc-sys/cpp/include/MiriInterface.hpp

@@ -5,17 +5,17 @@
 #include "rust/cxx.h"
 
 // GenMC generated headers:
-#include "config.h"
+#include "genmc/config.h"
 
 // Miri `genmc-sys/src_cpp` headers:
 #include "ResultHandling.hpp"
 
 // GenMC headers:
-#include "ExecutionGraph/EventLabel.hpp"
-#include "Support/MemOrdering.hpp"
-#include "Support/RMWOps.hpp"
-#include "Verification/Config.hpp"
-#include "Verification/GenMCDriver.hpp"
+#include "genmc/Execution/EventLabel.hpp"
+#include "genmc/Support/MemOrdering.hpp"
+#include "genmc/Support/RMWOps.hpp"
+#include "genmc/Verification/Config.hpp"
+#include "genmc/Verification/GenMCDriver.hpp"
 
 // C++ headers:
 #include <cstdint>
@@ -36,6 +36,7 @@ struct StoreResult;
 struct ReadModifyWriteResult;
 struct CompareExchangeResult;
 struct MutexLockResult;
+struct MallocResult;
 
 // GenMC uses `int` for its thread IDs.
 using ThreadId = int;
@@ -86,13 +87,15 @@ struct MiriGenmcShim : private GenMCDriver {
 
     /**** Memory access handling ****/
 
-    [[nodiscard]] LoadResult handle_load(
+    [[nodiscard]] LoadResult handle_atomic_load(
         ThreadId thread_id,
         uint64_t address,
         uint64_t size,
         MemOrdering ord,
         GenmcScalar old_val
     );
+    [[nodiscard]] LoadResult
+    handle_non_atomic_load(ThreadId thread_id, uint64_t address, uint64_t size);
     [[nodiscard]] ReadModifyWriteResult handle_read_modify_write(
         ThreadId thread_id,
         uint64_t address,
@@ -113,7 +116,7 @@ struct MiriGenmcShim : private GenMCDriver {
         MemOrdering fail_load_ordering,
         bool can_fail_spuriously
     );
-    [[nodiscard]] StoreResult handle_store(
+    [[nodiscard]] StoreResult handle_atomic_store(
         ThreadId thread_id,
         uint64_t address,
         uint64_t size,
@@ -121,12 +124,14 @@ struct MiriGenmcShim : private GenMCDriver {
         GenmcScalar old_val,
         MemOrdering ord
     );
+    [[nodiscard]] StoreResult
+    handle_non_atomic_store(ThreadId thread_id, uint64_t address, uint64_t size);
 
     void handle_fence(ThreadId thread_id, MemOrdering ord);
 
     /**** Memory (de)allocation ****/
 
-    auto handle_malloc(ThreadId thread_id, uint64_t size, uint64_t alignment) -> uint64_t;
+    auto handle_malloc(ThreadId thread_id, uint64_t size, uint64_t alignment) -> MallocResult;
 
     /** Returns null on success, or an error string if an error occurs. */
     auto handle_free(ThreadId thread_id, uint64_t address) -> std::unique_ptr<std::string>;
@@ -203,33 +208,15 @@ struct MiriGenmcShim : private GenMCDriver {
     auto get_estimation_results() const -> EstimationResult;
 
   private:
-    /** Increment the event index in the given thread by 1 and return the new event. */
-    [[nodiscard]] inline auto inc_pos(ThreadId tid) -> Event {
+    /** Returns the current event for a given thread. */
+    inline auto curr_pos(ThreadId tid) -> Event {
         ERROR_ON(tid >= threads_action_.size(), "ThreadId out of bounds");
-        return ++threads_action_[tid].event;
+        return threads_action_[tid].event;
     }
-    /** Decrement the event index in the given thread by 1 and return the new event. */
-    inline auto dec_pos(ThreadId tid) -> Event {
+    /** Increment the event index in the given thread by `count`. */
+    inline void inc_pos(ThreadId tid, unsigned int count) {
         ERROR_ON(tid >= threads_action_.size(), "ThreadId out of bounds");
-        return --threads_action_[tid].event;
-    }
-
-    /**
-     * Helper function for loads that need to reset the event counter when no value is returned.
-     * Same syntax as `GenMCDriver::handleLoad`, but this takes a thread id instead of an Event.
-     * Automatically calls `inc_pos` and `dec_pos` where needed for the given thread.
-     */
-    template <EventLabel::EventLabelKind k, typename... Ts>
-    auto handle_load_reset_if_none(ThreadId tid, std::optional<SVal> old_val, Ts&&... params)
-        -> HandleResult<SVal> {
-        const auto pos = inc_pos(tid);
-        const auto ret =
-            GenMCDriver::handleLoad<k>(nullptr, pos, old_val, std::forward<Ts>(params)...);
-        // If we didn't get a value, we have to reset the index of the current thread.
-        if (!std::holds_alternative<SVal>(ret)) {
-            dec_pos(tid);
-        }
-        return ret;
+        threads_action_[tid].event.index += count;
     }
 
     /**
@@ -293,40 +280,55 @@ inline std::optional<SVal> try_to_sval(GenmcScalar scalar) {
 namespace LoadResultExt {
 inline LoadResult no_value() {
     return LoadResult {
+        .invalid = false,
         .error = std::unique_ptr<std::string>(nullptr),
-        .has_value = false,
         .read_value = GenmcScalarExt::uninit(),
     };
 }
 
 inline LoadResult from_value(SVal read_value) {
-    return LoadResult { .error = std::unique_ptr<std::string>(nullptr),
-                        .has_value = true,
+    return LoadResult { .invalid = false,
+                        .error = std::unique_ptr<std::string>(nullptr),
                         .read_value = GenmcScalarExt::from_sval(read_value) };
 }
 
 inline LoadResult from_error(std::unique_ptr<std::string> error) {
-    return LoadResult { .error = std::move(error),
-                        .has_value = false,
+    return LoadResult { .invalid = false,
+                        .error = std::move(error),
                         .read_value = GenmcScalarExt::uninit() };
 }
+
+inline LoadResult from_invalid() {
+    return LoadResult { .invalid = true, .error = nullptr, .read_value = GenmcScalarExt::uninit() };
+}
+
 } // namespace LoadResultExt
 
 namespace StoreResultExt {
 inline StoreResult ok(bool is_coherence_order_maximal_write) {
-    return StoreResult { /* error: */ std::unique_ptr<std::string>(nullptr),
-                         is_coherence_order_maximal_write };
+    return StoreResult { .invalid = false,
+                         .error = std::unique_ptr<std::string>(nullptr),
+                         .is_coherence_order_maximal_write = is_coherence_order_maximal_write };
 }
 
 inline StoreResult from_error(std::unique_ptr<std::string> error) {
-    return StoreResult { .error = std::move(error), .is_coherence_order_maximal_write = false };
+    return StoreResult { .invalid = false,
+                         .error = std::move(error),
+                         .is_coherence_order_maximal_write = false };
+}
+
+inline StoreResult from_invalid() {
+    return StoreResult { .invalid = true,
+                         .error = nullptr,
+                         .is_coherence_order_maximal_write = false };
 }
 } // namespace StoreResultExt
 
 namespace ReadModifyWriteResultExt {
 inline ReadModifyWriteResult
 ok(SVal old_value, SVal new_value, bool is_coherence_order_maximal_write) {
-    return ReadModifyWriteResult { .error = std::unique_ptr<std::string>(nullptr),
+    return ReadModifyWriteResult { .invalid = false,
+                                   .error = std::unique_ptr<std::string>(nullptr),
                                    .old_value = GenmcScalarExt::from_sval(old_value),
                                    .new_value = GenmcScalarExt::from_sval(new_value),
                                    .is_coherence_order_maximal_write =
@@ -334,7 +336,16 @@ ok(SVal old_value, SVal new_value, bool is_coherence_order_maximal_write) {
 }
 
 inline ReadModifyWriteResult from_error(std::unique_ptr<std::string> error) {
-    return ReadModifyWriteResult { .error = std::move(error),
+    return ReadModifyWriteResult { .invalid = false,
+                                   .error = std::move(error),
+                                   .old_value = GenmcScalarExt::uninit(),
+                                   .new_value = GenmcScalarExt::uninit(),
+                                   .is_coherence_order_maximal_write = false };
+}
+
+inline ReadModifyWriteResult from_invalid() {
+    return ReadModifyWriteResult { .invalid = true,
+                                   .error = nullptr,
                                    .old_value = GenmcScalarExt::uninit(),
                                    .new_value = GenmcScalarExt::uninit(),
                                    .is_coherence_order_maximal_write = false };
@@ -343,7 +354,8 @@ inline ReadModifyWriteResult from_error(std::unique_ptr<std::string> error) {
 
 namespace CompareExchangeResultExt {
 inline CompareExchangeResult success(SVal old_value, bool is_coherence_order_maximal_write) {
-    return CompareExchangeResult { .error = nullptr,
+    return CompareExchangeResult { .invalid = false,
+                                   .error = nullptr,
                                    .old_value = GenmcScalarExt::from_sval(old_value),
                                    .is_success = true,
                                    .is_coherence_order_maximal_write =
@@ -351,14 +363,24 @@ inline CompareExchangeResult success(SVal old_value, bool is_coherence_order_max
 }
 
 inline CompareExchangeResult failure(SVal old_value) {
-    return CompareExchangeResult { .error = nullptr,
+    return CompareExchangeResult { .invalid = false,
+                                   .error = nullptr,
                                    .old_value = GenmcScalarExt::from_sval(old_value),
                                    .is_success = false,
                                    .is_coherence_order_maximal_write = false };
 }
 
 inline CompareExchangeResult from_error(std::unique_ptr<std::string> error) {
-    return CompareExchangeResult { .error = std::move(error),
+    return CompareExchangeResult { .invalid = false,
+                                   .error = std::move(error),
+                                   .old_value = GenmcScalarExt::uninit(),
+                                   .is_success = false,
+                                   .is_coherence_order_maximal_write = false };
+}
+
+inline CompareExchangeResult from_invalid() {
+    return CompareExchangeResult { .invalid = true,
+                                   .error = nullptr,
                                    .old_value = GenmcScalarExt::uninit(),
                                    .is_success = false,
                                    .is_coherence_order_maximal_write = false };
@@ -367,20 +389,42 @@ inline CompareExchangeResult from_error(std::unique_ptr<std::string> error) {
 
 namespace MutexLockResultExt {
 inline MutexLockResult ok(bool is_lock_acquired) {
-    return MutexLockResult { /* error: */ nullptr, /* is_reset: */ false, is_lock_acquired };
+    return MutexLockResult { .invalid = false,
+                             .error = nullptr,
+                             .is_reset = false,
+                             .is_lock_acquired = is_lock_acquired };
 }
 
 inline MutexLockResult reset() {
-    return MutexLockResult { /* error: */ nullptr,
-                             /* is_reset: */ true,
-                             /* is_lock_acquired: */ false };
+    return MutexLockResult { .invalid = false,
+                             .error = nullptr,
+                             .is_reset = true,
+                             .is_lock_acquired = false };
 }
 
 inline MutexLockResult from_error(std::unique_ptr<std::string> error) {
-    return MutexLockResult { /* error: */ std::move(error),
-                             /* is_reset: */ false,
-                             /* is_lock_acquired: */ false };
+    return MutexLockResult { .invalid = false,
+                             .error = std::move(error),
+                             .is_reset = false,
+                             .is_lock_acquired = false };
+}
+
+inline MutexLockResult from_invalid() {
+    return MutexLockResult { .invalid = true,
+                             .error = nullptr,
+                             .is_reset = false,
+                             .is_lock_acquired = false };
 }
 } // namespace MutexLockResultExt
 
+namespace MallocResultExt {
+inline MallocResult ok(SVal addr) {
+    return MallocResult { .error = nullptr, .address = addr.get() };
+}
+
+inline MallocResult from_error(std::unique_ptr<std::string> error) {
+    return MallocResult { .error = std::move(error), .address = 0UL };
+}
+} // namespace MallocResultExt
+
 #endif /* GENMC_MIRI_INTERFACE_HPP */

src/tools/miri/genmc-sys/cpp/include/ResultHandling.hpp

@@ -5,7 +5,7 @@
 #include "rust/cxx.h"
 
 // GenMC headers:
-#include "Verification/VerificationError.hpp"
+#include "genmc/Verification/VerificationError.hpp"
 
 #include <format>
 #include <memory>

src/tools/miri/genmc-sys/cpp/src/MiriInterface/Exploration.cpp

@@ -7,22 +7,22 @@
 #include "genmc-sys/src/lib.rs.h"
 
 // GenMC headers:
-#include "ADT/value_ptr.hpp"
-#include "ExecutionGraph/EventLabel.hpp"
-#include "ExecutionGraph/LoadAnnotation.hpp"
-#include "Runtime/InterpreterEnumAPI.hpp"
-#include "Static/ModuleID.hpp"
-#include "Support/ASize.hpp"
-#include "Support/Error.hpp"
-#include "Support/Logger.hpp"
-#include "Support/MemAccess.hpp"
-#include "Support/RMWOps.hpp"
-#include "Support/SAddr.hpp"
-#include "Support/SVal.hpp"
-#include "Support/ThreadInfo.hpp"
-#include "Support/Verbosity.hpp"
-#include "Verification/GenMCDriver.hpp"
-#include "Verification/MemoryModel.hpp"
+#include "genmc/ADT/value_ptr.hpp"
+#include "genmc/Execution/EventLabel.hpp"
+#include "genmc/Execution/LoadAnnotation.hpp"
+#include "genmc/Support/ASize.hpp"
+#include "genmc/Support/ActionEnums.hpp"
+#include "genmc/Support/Error.hpp"
+#include "genmc/Support/Logger.hpp"
+#include "genmc/Support/MemAccess.hpp"
+#include "genmc/Support/ModuleVarID.hpp"
+#include "genmc/Support/RMWOps.hpp"
+#include "genmc/Support/SAddr.hpp"
+#include "genmc/Support/SVal.hpp"
+#include "genmc/Support/ThreadInfo.hpp"
+#include "genmc/Support/Verbosity.hpp"
+#include "genmc/Verification/GenMCDriver.hpp"
+#include "genmc/Verification/MemoryModel.hpp"
 
 // C++ headers:
 #include <cmath>
@@ -47,13 +47,13 @@ auto MiriGenmcShim::schedule_next(
         [](auto&& arg) {
             using T = std::decay_t<decltype(arg)>;
             if constexpr (std::is_same_v<T, int>)
-                return SchedulingResult { ExecutionState::Ok, static_cast<int32_t>(arg) };
+                return SchedulingResult { ExecutionStatus::Ok, static_cast<int32_t>(arg) };
             else if constexpr (std::is_same_v<T, Blocked>)
-                return SchedulingResult { ExecutionState::Blocked, 0 };
+                return SchedulingResult { ExecutionStatus::Blocked, 0 };
             else if constexpr (std::is_same_v<T, Error>)
-                return SchedulingResult { ExecutionState::Error, 0 };
+                return SchedulingResult { ExecutionStatus::Error, 0 };
             else if constexpr (std::is_same_v<T, Finished>)
-                return SchedulingResult { ExecutionState::Finished, 0 };
+                return SchedulingResult { ExecutionStatus::Finished, 0 };
             else
                 static_assert(false, "non-exhaustive visitor!");
         },
@@ -75,39 +75,66 @@ auto MiriGenmcShim::handle_execution_end() -> std::unique_ptr<std::string> {
 /**** Blocking instructions ****/
 
 void MiriGenmcShim::handle_assume_block(ThreadId thread_id, AssumeType assume_type) {
-    BUG_ON(getExec().getGraph().isThreadBlocked(thread_id));
-    GenMCDriver::handleAssume(nullptr, inc_pos(thread_id), assume_type);
+    auto ret = GenMCDriver::handleAssume(nullptr, curr_pos(thread_id), assume_type);
+    inc_pos(thread_id, ret.count);
 }
 
 /**** Memory access handling ****/
 
-[[nodiscard]] auto MiriGenmcShim::handle_load(
+[[nodiscard]] auto MiriGenmcShim::handle_atomic_load(
     ThreadId thread_id,
     uint64_t address,
     uint64_t size,
     MemOrdering ord,
     GenmcScalar old_val
 ) -> LoadResult {
-    // `type` is only used for printing.
-    const auto type = AType::Unsigned;
-    const auto ret = handle_load_reset_if_none<EventLabel::EventLabelKind::Read>(
-        thread_id,
+    const auto ret = GenMCDriver::handleRead(
+        nullptr,
+        curr_pos(thread_id),
         GenmcScalarExt::try_to_sval(old_val),
         ord,
         SAddr(address),
         ASize(size),
-        type
+        nullptr,
+        std::nullopt,
+        EventDeps()
     );
-
-    if (const auto* err = std::get_if<VerificationError>(&ret))
+    inc_pos(thread_id, ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&ret.result))
         return LoadResultExt::from_error(format_error(*err));
-    const auto* ret_val = std::get_if<SVal>(&ret);
-    // FIXME(genmc): handle `HandleResult::{Invalid, Reset}` return values.
-    ERROR_ON(!ret_val, "Unimplemented: load returned unexpected result.");
+    if (std::holds_alternative<Invalid>(ret.result))
+        return LoadResultExt::from_invalid();
+    const auto* ret_val = std::get_if<SVal>(&ret.result);
+    // FIXME(genmc): handle `HandleResult::Reset` return value.
+    ERROR_ON(!ret_val, "Unimplemented: atomic load returned unexpected result.");
     return LoadResultExt::from_value(*ret_val);
 }
 
-[[nodiscard]] auto MiriGenmcShim::handle_store(
+[[nodiscard]] auto
+MiriGenmcShim::handle_non_atomic_load(ThreadId thread_id, uint64_t address, uint64_t size)
+    -> LoadResult {
+    const auto ret = GenMCDriver::handleNALoad(
+        nullptr,
+        curr_pos(thread_id),
+        SAddr(address),
+        ASize(size),
+        EventDeps()
+    );
+    inc_pos(thread_id, ret.count);
+
+    if (const auto* err = std::get_if<VerificationError>(&ret.result))
+        return LoadResultExt::from_error(format_error(*err));
+    if (std::holds_alternative<Invalid>(ret.result))
+        return LoadResultExt::from_invalid();
+    // FIXME(genmc): handle `HandleResult::Reset` return value.
+    ERROR_ON(
+        !std::holds_alternative<std::monostate>(ret.result),
+        "Unimplemented: non-atomic load returned unexpected result."
+    );
+    return LoadResultExt::no_value();
+}
+
+[[nodiscard]] auto MiriGenmcShim::handle_atomic_store(
     ThreadId thread_id,
     uint64_t address,
     uint64_t size,
@@ -115,31 +142,57 @@ void MiriGenmcShim::handle_assume_block(ThreadId thread_id, AssumeType assume_ty
     GenmcScalar old_val,
     MemOrdering ord
 ) -> StoreResult {
-    const auto pos = inc_pos(thread_id);
-    const auto ret = GenMCDriver::handleStore<EventLabel::EventLabelKind::Write>(
+    const auto ret = GenMCDriver::handleWrite(
         nullptr,
-        pos,
+        curr_pos(thread_id),
         GenmcScalarExt::try_to_sval(old_val),
         ord,
         SAddr(address),
         ASize(size),
-        /* type */ AType::Unsigned, // `type` is only used for printing.
         GenmcScalarExt::to_sval(value),
+        WriteAttr(),
         EventDeps()
     );
 
-    if (const auto* err = std::get_if<VerificationError>(&ret))
+    inc_pos(thread_id, ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&ret.result))
         return StoreResultExt::from_error(format_error(*err));
+    if (std::holds_alternative<Invalid>(ret.result))
+        return StoreResultExt::from_invalid();
 
-    const auto* is_co_max = std::get_if<bool>(&ret);
-    // FIXME(genmc): handle `HandleResult::{Invalid, Reset}` return values.
-    ERROR_ON(!is_co_max, "Unimplemented: Store returned unexpected result.");
+    const auto* is_co_max = std::get_if<bool>(&ret.result);
+    // FIXME(genmc): handle `HandleResult::Reset` return value.
+    ERROR_ON(!is_co_max, "Unimplemented: atomic store returned unexpected result.");
     return StoreResultExt::ok(*is_co_max);
 }
 
+[[nodiscard]] auto
+MiriGenmcShim::handle_non_atomic_store(ThreadId thread_id, uint64_t address, uint64_t size)
+    -> StoreResult {
+    const auto ret = GenMCDriver::handleNAStore(
+        nullptr,
+        curr_pos(thread_id),
+        SAddr(address),
+        ASize(size),
+        EventDeps()
+    );
+    inc_pos(thread_id, ret.count);
+
+    if (const auto* err = std::get_if<VerificationError>(&ret.result))
+        return StoreResultExt::from_error(format_error(*err));
+    if (std::holds_alternative<Invalid>(ret.result))
+        return StoreResultExt::from_invalid();
+    // FIXME(genmc): handle `HandleResult::Reset` return value.
+    ERROR_ON(
+        !std::holds_alternative<std::monostate>(ret.result),
+        "Unimplemented: non-atomic store returned unexpected result."
+    );
+    return StoreResultExt::ok(true);
+}
+
 void MiriGenmcShim::handle_fence(ThreadId thread_id, MemOrdering ord) {
-    const auto pos = inc_pos(thread_id);
-    GenMCDriver::handleFence(nullptr, pos, ord, EventDeps());
+    auto ret = GenMCDriver::handleFence(nullptr, curr_pos(thread_id), ord, EventDeps());
+    inc_pos(thread_id, ret.count);
 }
 
 [[nodiscard]] auto MiriGenmcShim::handle_read_modify_write(
@@ -155,45 +208,52 @@ void MiriGenmcShim::handle_fence(ThreadId thread_id, MemOrdering ord) {
     // into a load and a store component. This means we can have for example `AcqRel` loads and
     // stores, but this is intended for RMW operations.
 
-    // Somewhat confusingly, the GenMC term for RMW read/write labels is
-    // `FaiRead` and `FaiWrite`.
-    const auto load_ret = handle_load_reset_if_none<EventLabel::EventLabelKind::FaiRead>(
-        thread_id,
+    const auto load_ret = GenMCDriver::handleFaiRead(
+        nullptr,
+        curr_pos(thread_id),
         GenmcScalarExt::try_to_sval(old_val),
         ordering,
         SAddr(address),
         ASize(size),
-        AType::Unsigned, // The type is only used for printing.
         rmw_op,
         GenmcScalarExt::to_sval(rhs_value),
+        WriteAttr(),
+        nullptr,
+        std::nullopt,
         EventDeps()
     );
-    if (const auto* err = std::get_if<VerificationError>(&load_ret))
+    inc_pos(thread_id, load_ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&load_ret.result))
         return ReadModifyWriteResultExt::from_error(format_error(*err));
+    if (std::holds_alternative<GenMCDriver::Invalid>(load_ret.result))
+        return ReadModifyWriteResultExt::from_invalid();
 
-    const auto* ret_val = std::get_if<SVal>(&load_ret);
-    // FIXME(genmc): handle `HandleResult::{Invalid, Reset}` return values.
+    const auto* ret_val = std::get_if<SVal>(&load_ret.result);
+    // FIXME(genmc): handle `HandleResult::Reset` return values.
     ERROR_ON(!ret_val, "Unimplemented: read-modify-write returned unexpected result.");
     const auto read_old_val = *ret_val;
     const auto new_value =
         executeRMWBinOp(read_old_val, GenmcScalarExt::to_sval(rhs_value), size, rmw_op);
 
-    const auto storePos = inc_pos(thread_id);
-    const auto store_ret = GenMCDriver::handleStore<EventLabel::EventLabelKind::FaiWrite>(
+    const auto store_ret = GenMCDriver::handleFaiWrite(
         nullptr,
-        storePos,
+        curr_pos(thread_id),
         GenmcScalarExt::try_to_sval(old_val),
         ordering,
         SAddr(address),
         ASize(size),
-        AType::Unsigned, // The type is only used for printing.
-        new_value
+        new_value,
+        WriteAttr(),
+        EventDeps()
     );
-    if (const auto* err = std::get_if<VerificationError>(&store_ret))
+    inc_pos(thread_id, store_ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&store_ret.result))
         return ReadModifyWriteResultExt::from_error(format_error(*err));
+    if (std::holds_alternative<GenMCDriver::Invalid>(store_ret.result))
+        return ReadModifyWriteResultExt::from_invalid();
 
-    const auto* is_co_max = std::get_if<bool>(&store_ret);
-    // FIXME(genmc): handle `HandleResult::{Invalid, Reset}` return values.
+    const auto* is_co_max = std::get_if<bool>(&store_ret.result);
+    // FIXME(genmc): handle `HandleResult::Reset` return values.
     ERROR_ON(!is_co_max, "Unimplemented: RMW store returned unexpected result.");
     return ReadModifyWriteResultExt::ok(
         /* old_value: */ read_old_val,
@@ -222,20 +282,28 @@ void MiriGenmcShim::handle_fence(ThreadId thread_id, MemOrdering ord) {
     auto expectedVal = GenmcScalarExt::to_sval(expected_value);
     auto new_val = GenmcScalarExt::to_sval(new_value);
 
-    const auto load_ret = handle_load_reset_if_none<EventLabel::EventLabelKind::CasRead>(
-        thread_id,
+    const auto load_ret = GenMCDriver::handleCasRead(
+        nullptr,
+        curr_pos(thread_id),
         GenmcScalarExt::try_to_sval(old_val),
         success_ordering,
         SAddr(address),
         ASize(size),
-        AType::Unsigned, // The type is only used for printing.
         expectedVal,
-        new_val
+        new_val,
+        WriteAttr(),
+        nullptr,
+        std::nullopt,
+        EventDeps()
     );
-    if (const auto* err = std::get_if<VerificationError>(&load_ret))
+    inc_pos(thread_id, load_ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&load_ret.result))
         return CompareExchangeResultExt::from_error(format_error(*err));
-    const auto* ret_val = std::get_if<SVal>(&load_ret);
-    // FIXME(genmc): handle `HandleResult::{Invalid, Reset}` return values.
+    if (std::holds_alternative<GenMCDriver::Invalid>(load_ret.result))
+        return CompareExchangeResultExt::from_invalid();
+
+    const auto* ret_val = std::get_if<SVal>(&load_ret.result);
+    // FIXME(genmc): handle `HandleResult::Reset` return values.
     ERROR_ON(nullptr == ret_val, "Unimplemented: load returned unexpected result.");
     const auto read_old_val = *ret_val;
     if (read_old_val != expectedVal)
@@ -243,21 +311,25 @@ void MiriGenmcShim::handle_fence(ThreadId thread_id, MemOrdering ord) {
 
     // FIXME(GenMC): Add support for modelling spurious failures.
 
-    const auto storePos = inc_pos(thread_id);
-    const auto store_ret = GenMCDriver::handleStore<EventLabel::EventLabelKind::CasWrite>(
+    const auto store_ret = GenMCDriver::handleCasWrite(
         nullptr,
-        storePos,
+        curr_pos(thread_id),
         GenmcScalarExt::try_to_sval(old_val),
         success_ordering,
         SAddr(address),
         ASize(size),
-        AType::Unsigned, // The type is only used for printing.
-        new_val
+        new_val,
+        WriteAttr(),
+        EventDeps()
     );
-    if (const auto* err = std::get_if<VerificationError>(&store_ret))
+    inc_pos(thread_id, store_ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&store_ret.result))
         return CompareExchangeResultExt::from_error(format_error(*err));
-    const auto* is_co_max = std::get_if<bool>(&store_ret);
-    // FIXME(genmc): handle `HandleResult::{Invalid, Reset}` return values.
+    if (std::holds_alternative<GenMCDriver::Invalid>(store_ret.result))
+        return CompareExchangeResultExt::from_invalid();
+
+    const auto* is_co_max = std::get_if<bool>(&store_ret.result);
+    // FIXME(genmc): handle `HandleResult::Reset` return values.
     ERROR_ON(!is_co_max, "Unimplemented: compare-exchange store returned unexpected result.");
     return CompareExchangeResultExt::success(read_old_val, *is_co_max);
 }
@@ -265,33 +337,45 @@ void MiriGenmcShim::handle_fence(ThreadId thread_id, MemOrdering ord) {
 /**** Memory (de)allocation ****/
 
 auto MiriGenmcShim::handle_malloc(ThreadId thread_id, uint64_t size, uint64_t alignment)
-    -> uint64_t {
-    const auto pos = inc_pos(thread_id);
-
+    -> MallocResult {
     // These are only used for printing and features Miri-GenMC doesn't support (yet).
     const auto storage_duration = StorageDuration::SD_Heap;
     // Volatile, as opposed to "persistent" (i.e., non-volatile memory that persists over reboots)
     const auto storage_type = StorageType::ST_Volatile;
     const auto address_space = AddressSpace::AS_User;
 
-    const SVal ret_val = GenMCDriver::handleMalloc(
+    const auto ret = GenMCDriver::handleMalloc(
         nullptr,
-        pos,
+        curr_pos(thread_id),
         size,
         alignment,
         storage_duration,
         storage_type,
         address_space,
+        nullptr,
+        "",
         EventDeps()
     );
-    return ret_val.get();
+    inc_pos(thread_id, ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&ret.result))
+        return MallocResultExt::from_error(format_error(*err));
+    const auto* addr = std::get_if<SVal>(&ret.result);
+    ERROR_ON(!addr, "Unimplemented: malloc returned unexpected result.");
+    return MallocResultExt::ok(*addr);
 }
 
 auto MiriGenmcShim::handle_free(ThreadId thread_id, uint64_t address)
     -> std::unique_ptr<std::string> {
-    auto pos = inc_pos(thread_id);
-    auto ret = GenMCDriver::handleFree(nullptr, pos, SAddr(address), EventDeps());
-    return ret.has_value() ? format_error(*ret) : nullptr;
+    auto ret = GenMCDriver::handleFree(nullptr, curr_pos(thread_id), SAddr(address), EventDeps());
+    inc_pos(thread_id, ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&ret.result))
+        return format_error(*err);
+
+    ERROR_ON(
+        !std::holds_alternative<std::monostate>(ret.result),
+        "Unimplemented: free returned unexpected result."
+    );
+    return nullptr;
 }
 
 /**** Estimation mode result ****/
@@ -325,12 +409,12 @@ auto MiriGenmcShim::handle_mutex_lock(ThreadId thread_id, uint64_t address, uint
     const auto annot = std::move(Annotation(
         AssumeType::Spinloop,
         Annotation::ExprVP(
-            NeExpr<ModuleID::ID>::create(
+            NeExpr<ModuleVarID>::create(
                 // `RegisterExpr` marks the value of the current expression, i.e., the loaded value.
                 // The `id` is ignored by GenMC; it is only used by the LLI frontend to substitute
                 // other variables from previous expressions that may be used here.
-                RegisterExpr<ModuleID::ID>::create(size_bits, /* id */ 0),
-                ConcreteExpr<ModuleID::ID>::create(size_bits, MutexState::LOCKED)
+                RegisterExpr<ModuleVarID>::create(size_bits, /* id */ 0),
+                ConcreteExpr<ModuleVarID>::create(size_bits, MutexState::LOCKED)
             )
                 .release()
         )
@@ -340,26 +424,34 @@ auto MiriGenmcShim::handle_mutex_lock(ThreadId thread_id, uint64_t address, uint
     // access, if there previously was a non-atomic initializing access. We set the initial state of
     // a mutex to be "unlocked".
     const auto old_val = MutexState::UNLOCKED;
-    const auto load_ret = handle_load_reset_if_none<EventLabel::EventLabelKind::LockCasRead>(
-        thread_id,
+    const auto load_ret = GenMCDriver::handleLockCasRead(
+        nullptr,
+        curr_pos(thread_id),
         old_val,
         address,
         size,
         annot,
         EventDeps()
     );
-    if (const auto* err = std::get_if<VerificationError>(&load_ret))
+    inc_pos(thread_id, load_ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&load_ret.result))
         return MutexLockResultExt::from_error(format_error(*err));
+    if (std::holds_alternative<GenMCDriver::Invalid>(load_ret.result))
+        return MutexLockResultExt::from_invalid();
     // If we get a `Reset`, GenMC decided that this lock operation should not yet run, since it
     // would not acquire the mutex. Like the handling of the case further down where we read a `1`
     // ("Mutex already locked"), Miri should call the handle function again once the current thread
     // is scheduled by GenMC the next time.
-    if (std::holds_alternative<Reset>(load_ret))
+    if (std::holds_alternative<Reset>(load_ret.result))
         return MutexLockResultExt::reset();
 
-    const auto* ret_val = std::get_if<SVal>(&load_ret);
+    const auto* ret_val = std::get_if<SVal>(&load_ret.result);
     ERROR_ON(!ret_val, "Unimplemented: mutex lock returned unexpected result.");
-    ERROR_ON(!MutexState::isValid(*ret_val), "Mutex read value was neither 0 nor 1");
+    ERROR_ON(
+        !MutexState::isValid(*ret_val),
+        "Mutex read value was neither 0 nor 1 ({})",
+        std::to_string(ret_val->get())
+    );
     if (*ret_val == MutexState::LOCKED) {
         // We did not acquire the mutex, so we tell GenMC to block the thread until we can acquire
         // it. GenMC determines this based on the annotation we pass with the load further up in
@@ -368,69 +460,72 @@ auto MiriGenmcShim::handle_mutex_lock(ThreadId thread_id, uint64_t address, uint
         return MutexLockResultExt::ok(false);
     }
 
-    const auto store_ret = GenMCDriver::handleStore<EventLabel::EventLabelKind::LockCasWrite>(
-        nullptr,
-        inc_pos(thread_id),
-        old_val,
-        address,
-        size,
-        EventDeps()
-    );
-    if (const auto* err = std::get_if<VerificationError>(&store_ret))
+    const auto store_ret =
+        GenMCDriver::handleLockCasWrite(nullptr, curr_pos(thread_id), address, size, EventDeps());
+    inc_pos(thread_id, store_ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&store_ret.result))
         return MutexLockResultExt::from_error(format_error(*err));
+    if (std::holds_alternative<GenMCDriver::Invalid>(store_ret.result))
+        return MutexLockResultExt::from_invalid();
     // We don't update Miri's memory for this operation so we don't need to know if the store
     // was the co-maximal store, but we still check that we at least get a boolean as the result
     // of the store.
-    const auto* is_co_max = std::get_if<bool>(&store_ret);
+    const auto* is_co_max = std::get_if<bool>(&store_ret.result);
     ERROR_ON(!is_co_max, "Unimplemented: mutex_try_lock store returned unexpected result.");
     return MutexLockResultExt::ok(true);
 }
 
 auto MiriGenmcShim::handle_mutex_try_lock(ThreadId thread_id, uint64_t address, uint64_t size)
     -> MutexLockResult {
-    auto& currPos = threads_action_[thread_id].event;
     // As usual, we need to tell GenMC which value was stored at this location before this atomic
     // access, if there previously was a non-atomic initializing access. We set the initial state of
     // a mutex to be "unlocked".
     const auto old_val = MutexState::UNLOCKED;
-    const auto load_ret = GenMCDriver::handleLoad<EventLabel::EventLabelKind::TrylockCasRead>(
+    const auto load_ret = GenMCDriver::handleTrylockCasRead(
         nullptr,
-        ++currPos,
+        curr_pos(thread_id),
         old_val,
         SAddr(address),
-        ASize(size)
+        ASize(size),
+        std::nullopt,
+        EventDeps()
     );
-    if (const auto* err = std::get_if<VerificationError>(&load_ret))
+    inc_pos(thread_id, load_ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&load_ret.result))
         return MutexLockResultExt::from_error(format_error(*err));
-    const auto* ret_val = std::get_if<SVal>(&load_ret);
+    if (std::holds_alternative<GenMCDriver::Invalid>(load_ret.result))
+        return MutexLockResultExt::from_invalid();
+    const auto* ret_val = std::get_if<SVal>(&load_ret.result);
     ERROR_ON(!ret_val, "Unimplemented: mutex trylock load returned unexpected result.");
 
     ERROR_ON(!MutexState::isValid(*ret_val), "Mutex read value was neither 0 nor 1");
     if (*ret_val == MutexState::LOCKED)
         return MutexLockResultExt::ok(false); /* Lock already held. */
 
-    const auto store_ret = GenMCDriver::handleStore<EventLabel::EventLabelKind::TrylockCasWrite>(
+    const auto store_ret = GenMCDriver::handleTrylockCasWrite(
         nullptr,
-        ++currPos,
-        old_val,
+        curr_pos(thread_id),
         SAddr(address),
-        ASize(size)
+        ASize(size),
+        EventDeps()
     );
-    if (const auto* err = std::get_if<VerificationError>(&store_ret))
+    inc_pos(thread_id, store_ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&store_ret.result))
         return MutexLockResultExt::from_error(format_error(*err));
+    if (std::holds_alternative<GenMCDriver::Invalid>(store_ret.result))
+        return MutexLockResultExt::from_invalid();
     // We don't update Miri's memory for this operation so we don't need to know if the store was
     // co-maximal, but we still check that we get a boolean result.
-    const auto* is_co_max = std::get_if<bool>(&store_ret);
+    const auto* is_co_max = std::get_if<bool>(&store_ret.result);
     ERROR_ON(!is_co_max, "Unimplemented: store part of mutex try_lock returned unexpected result.");
     return MutexLockResultExt::ok(true);
 }
 
 auto MiriGenmcShim::handle_mutex_unlock(ThreadId thread_id, uint64_t address, uint64_t size)
     -> StoreResult {
-    const auto pos = inc_pos(thread_id);
-    const auto ret = GenMCDriver::handleStore<EventLabel::EventLabelKind::UnlockWrite>(
+    const auto ret = GenMCDriver::handleUnlockWrite(
         nullptr,
-        pos,
+        curr_pos(thread_id),
         // As usual, we need to tell GenMC which value was stored at this location before this
         // atomic access, if there previously was a non-atomic initializing access. We set the
         // initial state of a mutex to be "unlocked".
@@ -438,13 +533,16 @@ auto MiriGenmcShim::handle_mutex_unlock(ThreadId thread_id, uint64_t address, ui
         MemOrdering::Release,
         SAddr(address),
         ASize(size),
-        AType::Signed,
         /* store_value */ MutexState::UNLOCKED,
+        WriteAttr(),
         EventDeps()
     );
-    if (const auto* err = std::get_if<VerificationError>(&ret))
+    inc_pos(thread_id, ret.count);
+    if (const auto* err = std::get_if<VerificationError>(&ret.result))
         return StoreResultExt::from_error(format_error(*err));
-    const auto* is_co_max = std::get_if<bool>(&ret);
+    if (std::holds_alternative<GenMCDriver::Invalid>(ret.result))
+        return StoreResultExt::from_invalid();
+    const auto* is_co_max = std::get_if<bool>(&ret.result);
     ERROR_ON(!is_co_max, "Unimplemented: store part of mutex unlock returned unexpected result.");
     return StoreResultExt::ok(*is_co_max);
 }
@@ -452,40 +550,62 @@ auto MiriGenmcShim::handle_mutex_unlock(ThreadId thread_id, uint64_t address, ui
 /** Thread creation/joining */
 
 void MiriGenmcShim::handle_thread_create(ThreadId thread_id, ThreadId parent_id) {
-    // NOTE: The threadCreate event happens in the parent:
-    const auto pos = inc_pos(parent_id);
     // FIXME(genmc): for supporting symmetry reduction, these will need to be properly set:
     const unsigned fun_id = 0;
     const SVal arg = SVal(0);
     const ThreadInfo child_info =
         ThreadInfo { thread_id, parent_id, fun_id, arg, "unknown thread" };
 
-    const auto child_tid = GenMCDriver::handleThreadCreate(nullptr, pos, child_info, EventDeps());
+    // NOTE: The threadCreate event happens in the parent:
+    const auto ret =
+        GenMCDriver::handleThreadCreate(nullptr, curr_pos(parent_id), child_info, EventDeps());
+    inc_pos(parent_id, ret.count);
+    ERROR_ON(
+        !std::holds_alternative<int>(ret.result),
+        "Unimplemented: unexpected return value for thread create"
+    );
+    auto child_tid = std::get<int>(ret.result);
+
     // Sanity check the thread id, which is the index in the `threads_action_` array.
-    BUG_ON(child_tid != thread_id || child_tid <= 0 || child_tid != threads_action_.size());
+    VERIFY(child_tid == thread_id && child_tid > 0 && child_tid == threads_action_.size());
     threads_action_.push_back(Action(ActionKind::Load, Event(child_tid, 0)));
 }
 
 void MiriGenmcShim::handle_thread_join(ThreadId thread_id, ThreadId child_id) {
     // The thread join event happens in the parent.
-    const auto pos = inc_pos(thread_id);
-
-    const auto ret = GenMCDriver::handleThreadJoin(nullptr, pos, child_id, EventDeps());
-    // If the join failed, decrease the event index again:
-    if (!std::holds_alternative<SVal>(ret)) {
-        dec_pos(thread_id);
-    }
-    // FIXME(genmc): handle `HandleResult::{Invalid, Reset, VerificationError}` return values.
+    const auto ret =
+        GenMCDriver::handleThreadJoin(nullptr, curr_pos(thread_id), child_id, EventDeps());
+    inc_pos(thread_id, ret.count);
+    // FIXME(genmc): handle `HandleResult::{Invalid, VerificationError}` return values.
+    ERROR_ON(
+        !std::holds_alternative<SVal>(ret.result) && !std::holds_alternative<Reset>(ret.result),
+        "Unimplemented: unexpected return value for thread join"
+    );
+    // FIXME(genmc): Here Reset{} is silently accepted. Double-check why that is.
+    // The reason is likely that, although GenMC wants to re-run the join instruction,
+    // when GenMC deems that the join has executed, it will also deem it successful,
+    // i.e., the return value is guaranteed to be 0 (or at least we assume that).
+    // In this case, it doesn't matter that we don't re-run the instruction, since
+    // Miri sets the correct return value, and GenMC will only schedule this thread
+    // when it knows the child has terminated.
 
     // NOTE: Thread return value is ignored, since Miri doesn't need it.
 }
 
 void MiriGenmcShim::handle_thread_finish(ThreadId thread_id, uint64_t ret_val) {
-    const auto pos = inc_pos(thread_id);
-    GenMCDriver::handleThreadFinish(nullptr, pos, SVal(ret_val));
+    auto ret = GenMCDriver::handleThreadFinish(nullptr, curr_pos(thread_id), SVal(ret_val));
+    inc_pos(thread_id, ret.count);
+    ERROR_ON(
+        !std::holds_alternative<std::monostate>(ret.result),
+        "Unimplemented: unexpected return value for thread finish"
+    );
 }
 
 void MiriGenmcShim::handle_thread_kill(ThreadId thread_id) {
-    const auto pos = inc_pos(thread_id);
-    GenMCDriver::handleThreadKill(nullptr, pos);
+    auto ret = GenMCDriver::handleThreadKill(nullptr, curr_pos(thread_id));
+    inc_pos(thread_id, ret.count);
+    ERROR_ON(
+        !std::holds_alternative<std::monostate>(ret.result),
+        "Unimplemented: unexpected return value for thread kill"
+    );
 }

src/tools/miri/genmc-sys/cpp/src/MiriInterface/Setup.cpp

@@ -7,9 +7,8 @@
 #include "genmc-sys/src/lib.rs.h"
 
 // GenMC headers:
-#include "Support/Error.hpp"
-#include "Support/Verbosity.hpp"
-#include "Verification/InterpreterCallbacks.hpp"
+#include "genmc/Support/Error.hpp"
+#include "genmc/Support/Verbosity.hpp"
 
 // C++ headers:
 #include <cstdint>
@@ -113,6 +112,13 @@ static auto to_genmc_verbosity_level(const LogLevel log_level) -> VerbosityLevel
     // value written by the skipped thread.
     conf->replayCompletedThreads = true;
 
+    // Initialization checking is done by Miri; GenMC's checks are incorrect for Rust.
+    conf->disableInitializationChecks = true;
+
+    // Don't check static-address validity as it's incompatible with Miri's
+    // dynamic discovery of static variables.
+    conf->disableStaticValidityChecks = true;
+
     // FIXME(genmc): implement symmetry reduction.
     ERROR_ON(
         params.do_symmetry_reduction,
@@ -160,45 +166,5 @@ static auto to_genmc_verbosity_level(const LogLevel log_level) -> VerbosityLevel
 
     // Create the actual driver and Miri-GenMC communication shim.
     auto driver = std::make_unique<MiriGenmcShim>(std::move(conf), mode);
-
-    // FIXME(genmc,HACK): Until a proper solution is implemented in GenMC, these callbacks will
-    // allow Miri to return information about global allocations and override uninitialized memory
-    // checks for non-atomic loads (Miri handles those without GenMC, so the error would be wrong).
-    auto interpreter_callbacks = InterpreterCallbacks {
-        // Miri already ensures that memory accesses are valid, so this check doesn't matter.
-        // We check that the address is static, but skip checking if it is part of an actual
-        // allocation.
-        .isStaticallyAllocated = [](SAddr addr) { return addr.isStatic(); },
-        // FIXME(genmc,error reporting): Once a proper a proper API for passing such information is
-        // implemented in GenMC, Miri should use it to improve the produced error messages.
-        .getStaticName = [](SAddr addr) { return "[UNKNOWN STATIC]"; },
-        // This function is called to get the initial value stored at the given address.
-        //
-        // From a Miri perspective, this API doesn't work very well: most memory starts out
-        // "uninitialized";
-        // only statics have an initial value. And their initial value is just a sequence of bytes,
-        // but GenMC expect this to be already split into separate atomic variables. So we return a
-        // dummy value.
-        // This value should never be visible to the interpreted program.
-        // GenMC does not understand uninitialized memory the same way Miri does, which may cause
-        // this function to be called. The returned value can be visible to Miri or the user:
-        // - Printing the execution graph may contain this value in place of uninitialized values.
-        //   FIXME(genmc): NOTE: printing the execution graph is not yet implemented.
-        // - Non-atomic loads may return this value, but Miri ignores values of non-atomic loads.
-        // - Atomic loads will *not* see this value once mixed atomic-non-atomic support is added.
-        //   Currently, atomic loads can see this value, unless initialized by an *atomic* store.
-        //   FIXME(genmc): update this comment once mixed atomic-non-atomic support is added.
-        //
-        // FIXME(genmc): implement proper support for uninitialized memory in GenMC.
-        // Ideally, the initial value getter would return an `optional<SVal>`, since the memory
-        // location may be uninitialized.
-        .initValGetter = [](const AAccess& a) { return SVal(0xDEAD); },
-        // Miri serves non-atomic loads from its own memory and these GenMC checks are wrong in that
-        // case. This should no longer be required with proper mixed-size access support.
-        .skipUninitLoadChecks = [](const MemAccessLabel* access_label
-                                ) { return access_label->getOrdering() == MemOrdering::NotAtomic; },
-    };
-    driver->setInterpCallbacks(std::move(interpreter_callbacks));
-
     return driver;
 }

src/tools/miri/genmc-sys/src/lib.rs

@@ -140,15 +140,15 @@ enum LogLevel {
         /// Log errors, warnings and tips.
         Tip,
         /// Debug print considered revisits.
-        /// Downgraded to `Tip` if `GENMC_DEBUG` is not enabled.
+        /// Downgraded to `Tip` if `ENABLE_GENMC_DEBUG` is not enabled.
         Debug1Revisits,
         /// Print the execution graph after every memory access.
         /// Also includes the previous debug log level.
-        /// Downgraded to `Tip` if `GENMC_DEBUG` is not enabled.
+        /// Downgraded to `Tip` if `ENABLE_GENMC_DEBUG` is not enabled.
         Debug2MemoryAccesses,
         /// Print reads-from values considered by GenMC.
         /// Also includes the previous debug log level.
-        /// Downgraded to `Tip` if `GENMC_DEBUG` is not enabled.
+        /// Downgraded to `Tip` if `ENABLE_GENMC_DEBUG` is not enabled.
         Debug3ReadsFrom,
     }
 
@@ -182,7 +182,7 @@ struct GenmcScalar {
 
     #[must_use]
     #[derive(Debug, Clone, Copy)]
-    enum ExecutionState {
+    enum ExecutionStatus {
         Ok,
         Error,
         Blocked,
@@ -192,7 +192,7 @@ enum ExecutionState {
     #[must_use]
     #[derive(Debug)]
     struct SchedulingResult {
-        exec_state: ExecutionState,
+        exec_status: ExecutionStatus,
         next_thread: i32,
     }
 
@@ -212,10 +212,10 @@ struct EstimationResult {
     #[must_use]
     #[derive(Debug)]
     struct LoadResult {
+        /// If `true`, exploration should be dropped, **and all other fields are invalid**.
+        invalid: bool,
         /// If not null, contains the error encountered during the handling of the load.
         error: UniquePtr<CxxString>,
-        /// Indicates whether a value was read or not.
-        has_value: bool,
         /// The value that was read. Should not be used if `has_value` is `false`.
         read_value: GenmcScalar,
     }
@@ -223,6 +223,8 @@ struct LoadResult {
     #[must_use]
     #[derive(Debug)]
     struct StoreResult {
+        /// If `true`, exploration should be dropped, **and all other fields are invalid**.
+        invalid: bool,
         /// If not null, contains the error encountered during the handling of the store.
         error: UniquePtr<CxxString>,
         /// `true` if the write should also be reflected in Miri's memory representation.
@@ -232,6 +234,8 @@ struct StoreResult {
     #[must_use]
     #[derive(Debug)]
     struct ReadModifyWriteResult {
+        /// If `true`, exploration should be dropped, **and all other fields are invalid**.
+        invalid: bool,
         /// If there was an error, it will be stored in `error`, otherwise it is `None`.
         error: UniquePtr<CxxString>,
         /// The value that was read by the RMW operation as the left operand.
@@ -245,6 +249,8 @@ struct ReadModifyWriteResult {
     #[must_use]
     #[derive(Debug)]
     struct CompareExchangeResult {
+        /// If `true`, exploration should be dropped, **and all other fields are invalid**.
+        invalid: bool,
         /// If there was an error, it will be stored in `error`, otherwise it is `None`.
         error: UniquePtr<CxxString>,
         /// The value that was read by the compare-exchange.
@@ -258,6 +264,8 @@ struct CompareExchangeResult {
     #[must_use]
     #[derive(Debug)]
     struct MutexLockResult {
+        /// If `true`, exploration should be dropped, **and all other fields are invalid**.
+        invalid: bool,
         /// If there was an error, it will be stored in `error`, otherwise it is `None`.
         error: UniquePtr<CxxString>,
         /// If true, GenMC determined that we should retry the mutex lock operation once the thread attempting to lock is scheduled again.
@@ -266,6 +274,15 @@ struct MutexLockResult {
         is_lock_acquired: bool,
     }
 
+    #[must_use]
+    #[derive(Debug)]
+    struct MallocResult {
+        /// If not null, contains the error encountered during the handling of malloc.
+        error: UniquePtr<CxxString>,
+        /// The allocated address.
+        address: u64,
+    }
+
     /**** These are GenMC types that we have to copy-paste here since cxx does not support
     "importing" externally defined C++ types. ****/
 
@@ -385,7 +402,7 @@ unsafe fn create_handle(
         /***** Functions for handling events encountered during program execution. *****/
 
         /**** Memory access handling ****/
-        fn handle_load(
+        fn handle_atomic_load(
             self: Pin<&mut MiriGenmcShim>,
             thread_id: i32,
             address: u64,
@@ -393,6 +410,12 @@ fn handle_load(
             memory_ordering: MemOrdering,
             old_value: GenmcScalar,
         ) -> LoadResult;
+        fn handle_non_atomic_load(
+            self: Pin<&mut MiriGenmcShim>,
+            thread_id: i32,
+            address: u64,
+            size: u64,
+        ) -> LoadResult;
         fn handle_read_modify_write(
             self: Pin<&mut MiriGenmcShim>,
             thread_id: i32,
@@ -415,7 +438,7 @@ fn handle_compare_exchange(
             fail_load_ordering: MemOrdering,
             can_fail_spuriously: bool,
         ) -> CompareExchangeResult;
-        fn handle_store(
+        fn handle_atomic_store(
             self: Pin<&mut MiriGenmcShim>,
             thread_id: i32,
             address: u64,
@@ -424,6 +447,12 @@ fn handle_store(
             old_value: GenmcScalar,
             memory_ordering: MemOrdering,
         ) -> StoreResult;
+        fn handle_non_atomic_store(
+            self: Pin<&mut MiriGenmcShim>,
+            thread_id: i32,
+            address: u64,
+            size: u64,
+        ) -> StoreResult;
         fn handle_fence(
             self: Pin<&mut MiriGenmcShim>,
             thread_id: i32,
@@ -436,7 +465,7 @@ fn handle_malloc(
             thread_id: i32,
             size: u64,
             alignment: u64,
-        ) -> u64;
+        ) -> MallocResult;
         /// Returns true if an error was found.
         fn handle_free(
             self: Pin<&mut MiriGenmcShim>,

src/tools/miri/rust-version

@@ -1 +1 @@
-4c4205163abcbd08948b3efab796c543ba1ea687
+e22c616e4e87914135c1db261a03e0437255335e

src/tools/miri/src/alloc_addresses/mod.rs

@@ -170,7 +170,9 @@ fn addr_from_alloc_id_uncached(
                     {
                         let fn_sig = this.tcx.instantiate_bound_regions_with_erased(
                             this.tcx
-                                .fn_sig(instance.def_id()).instantiate(*this.tcx, instance.args).skip_norm_wip(),
+                                .fn_sig(instance.def_id())
+                                .instantiate(*this.tcx, instance.args)
+                                .skip_norm_wip(),
                         );
                         let fn_ptr = crate::shims::native_lib::build_libffi_closure(this, fn_sig)?;
 

src/tools/miri/src/concurrency/genmc/helper.rs

@@ -4,7 +4,6 @@
 use rustc_middle::mir;
 use rustc_middle::mir::interpret;
 use rustc_middle::ty::ScalarInt;
-use tracing::debug;
 
 use super::GenmcScalar;
 use crate::alloc_addresses::EvalContextExt as _;
@@ -13,33 +12,6 @@
 
 /// Maximum size memory access in bytes that GenMC supports.
 pub(super) const MAX_ACCESS_SIZE: u64 = 8;
-
-/// This function is used to split up a large memory access into aligned, non-overlapping chunks of a limited size.
-/// Returns an iterator over the chunks, yielding `(base address, size)` of each chunk, ordered by address.
-pub fn split_access(address: Size, size: Size) -> impl Iterator<Item = (u64, u64)> {
-    let start_address = address.bytes();
-    let end_address = start_address + size.bytes();
-
-    let start_address_aligned = start_address.next_multiple_of(MAX_ACCESS_SIZE);
-    let end_address_aligned = (end_address / MAX_ACCESS_SIZE) * MAX_ACCESS_SIZE; // prev_multiple_of
-
-    debug!(
-        "GenMC: splitting NA memory access into {MAX_ACCESS_SIZE} byte chunks: {}B + {} * {MAX_ACCESS_SIZE}B + {}B = {size:?}",
-        start_address_aligned - start_address,
-        (end_address_aligned - start_address_aligned) / MAX_ACCESS_SIZE,
-        end_address - end_address_aligned,
-    );
-
-    // FIXME(genmc): could make remaining accesses powers-of-2, instead of 1 byte.
-    let start_chunks = (start_address..start_address_aligned).map(|address| (address, 1));
-    let aligned_chunks = (start_address_aligned..end_address_aligned)
-        .step_by(MAX_ACCESS_SIZE.try_into().unwrap())
-        .map(|address| (address, MAX_ACCESS_SIZE));
-    let end_chunks = (end_address_aligned..end_address).map(|address| (address, 1));
-
-    start_chunks.chain(aligned_chunks).chain(end_chunks)
-}
-
 /// Inverse function to `scalar_to_genmc_scalar`.
 ///
 /// Convert a Miri `Scalar` to a `GenmcScalar`.

src/tools/miri/src/concurrency/genmc/mod.rs

@@ -19,7 +19,6 @@
 };
 use self::run::GenmcMode;
 use self::thread_id_map::ThreadIdMap;
-use crate::concurrency::genmc::helper::split_access;
 use crate::diagnostics::SpanDedupDiagnostic;
 use crate::intrinsics::AtomicRmwOp;
 use crate::*;
@@ -267,8 +266,13 @@ pub(crate) fn atomic_load<'tcx>(
         } else {
             GenmcScalar::UNINIT
         };
-        let read_value =
-            self.handle_load(&ecx.machine, address, size, ordering.to_genmc(), genmc_old_value)?;
+        let read_value = self.handle_atomic_load(
+            &ecx.machine,
+            address,
+            size,
+            ordering.to_genmc(),
+            genmc_old_value,
+        )?;
         genmc_scalar_to_scalar(ecx, self, read_value, size)
     }
 
@@ -292,7 +296,7 @@ pub(crate) fn atomic_store<'tcx>(
         } else {
             GenmcScalar::UNINIT
         };
-        self.handle_store(
+        self.handle_atomic_store(
             &ecx.machine,
             address,
             size,
@@ -447,6 +451,9 @@ pub(crate) fn atomic_compare_exchange<'tcx>(
             can_fail_spuriously,
         );
 
+        if cas_result.invalid {
+            throw_machine_stop!(TerminationInfo::GenmcSkip);
+        }
         if let Some(error) = cas_result.error.as_ref() {
             // FIXME(genmc): error handling
             throw_ub_format!("{}", error.to_string_lossy());
@@ -488,32 +495,7 @@ pub(crate) fn memory_load<'tcx>(
             return interp_ok(());
         }
 
-        let handle_load = |address, size| {
-            // NOTE: Values loaded non-atomically are still handled by Miri, so we discard whatever we get from GenMC
-            let _read_value = self.handle_load(
-                machine,
-                address,
-                size,
-                MemOrdering::NotAtomic,
-                // This value is used to update the co-maximal store event to the same location.
-                // We don't need to update that store, since if it is ever read by any atomic loads, the value will be updated then.
-                // We use uninit for lack of a better value, since we don't know whether the location we currently load from is initialized or not.
-                GenmcScalar::UNINIT,
-            )?;
-            interp_ok(())
-        };
-
-        // This load is small enough so GenMC can handle it.
-        if size.bytes() <= MAX_ACCESS_SIZE {
-            return handle_load(address, size);
-        }
-
-        // This load is too big to be a single GenMC access, we have to split it.
-        // FIXME(genmc): This will misbehave if there are non-64bit-atomics in there.
-        // Needs proper support on the GenMC side for large and mixed atomic accesses.
-        for (address, size) in split_access(address, size) {
-            handle_load(Size::from_bytes(address), Size::from_bytes(size))?;
-        }
+        self.handle_non_atomic_load(machine, address, size)?;
         interp_ok(())
     }
 
@@ -540,40 +522,7 @@ pub(crate) fn memory_store<'tcx>(
             return interp_ok(());
         }
 
-        let handle_store = |address, size| {
-            // We always write the the stored values to Miri's memory, whether GenMC says the write is co-maximal or not.
-            // The GenMC scheduler ensures that replaying an execution happens in porf-respecting order (po := program order, rf: reads-from order).
-            // This means that for any non-atomic read Miri performs, the corresponding write has already been replayed.
-            let _is_co_max_write = self.handle_store(
-                machine,
-                address,
-                size,
-                // We don't know the value that this store will write, but GenMC expects that we give it an actual value.
-                // Unfortunately, there are situations where this value can actually become visible
-                // to the program: when there is an atomic load reading from a non-atomic store.
-                // FIXME(genmc): update once mixed atomic-non-atomic support is added. Afterwards, this value should never be readable.
-                GenmcScalar::from_u64(0xDEADBEEF),
-                // This value is used to update the co-maximal store event to the same location.
-                // This old value cannot be read anymore by any future loads, since we are doing another non-atomic store to the same location.
-                // Any future load will either see the store we are adding now, or we have a data race (there can only be one possible non-atomic value to read from at any time).
-                // We use uninit for lack of a better value, since we don't know whether the location we currently write to is initialized or not.
-                GenmcScalar::UNINIT,
-                MemOrdering::NotAtomic,
-            )?;
-            interp_ok(())
-        };
-
-        // This store is small enough so GenMC can handle it.
-        if size.bytes() <= MAX_ACCESS_SIZE {
-            return handle_store(address, size);
-        }
-
-        // This store is too big to be a single GenMC access, we have to split it.
-        // FIXME(genmc): This will misbehave if there are non-64bit-atomics in there.
-        // Needs proper support on the GenMC side for large and mixed atomic accesses.
-        for (address, size) in split_access(address, size) {
-            handle_store(Size::from_bytes(address), Size::from_bytes(size))?;
-        }
+        self.handle_non_atomic_store(machine, address, size)?;
         interp_ok(())
     }
 
@@ -599,14 +548,15 @@ pub(crate) fn handle_alloc<'tcx>(
         }
         // GenMC doesn't support ZSTs, so we set the minimum size to 1 byte
         let genmc_size = size.bytes().max(1);
-        let chosen_address = self.handle.borrow_mut().pin_mut().handle_malloc(
+        let malloc_result = self.handle.borrow_mut().pin_mut().handle_malloc(
             self.active_thread_genmc_tid(machine),
             genmc_size,
             alignment.bytes(),
         );
-        if chosen_address == 0 {
+        if let Some(_error) = malloc_result.error.as_ref() {
             throw_exhaust!(AddressSpaceFull);
         }
+        let chosen_address = malloc_result.address;
 
         // Non-global addresses should not be in the global address space.
         assert_eq!(0, chosen_address & GENMC_GLOBAL_ADDRESSES_MASK);
@@ -735,9 +685,9 @@ pub(crate) fn handle_exit<'tcx>(
 }
 
 impl GenmcCtx {
-    /// Inform GenMC about a load (atomic or non-atomic).
+    /// Inform GenMC about an atomic load.
     /// Returns the value that GenMC wants this load to read.
-    fn handle_load<'tcx>(
+    fn handle_atomic_load<'tcx>(
         &self,
         machine: &MiriMachine<'tcx>,
         address: Size,
@@ -758,7 +708,7 @@ fn handle_load<'tcx>(
             "GenMC: load, address: {addr} == {addr:#x}, size: {size:?}, ordering: {memory_ordering:?}, old_value: {genmc_old_value:x?}",
             addr = address.bytes()
         );
-        let load_result = self.handle.borrow_mut().pin_mut().handle_load(
+        let load_result = self.handle.borrow_mut().pin_mut().handle_atomic_load(
             self.active_thread_genmc_tid(machine),
             address.bytes(),
             size.bytes(),
@@ -766,23 +716,51 @@ fn handle_load<'tcx>(
             genmc_old_value,
         );
 
+        if load_result.invalid {
+            throw_machine_stop!(TerminationInfo::GenmcSkip);
+        }
         if let Some(error) = load_result.error.as_ref() {
             // FIXME(genmc): error handling
             throw_ub_format!("{}", error.to_string_lossy());
         }
 
-        if !load_result.has_value {
-            // FIXME(GenMC): Implementing certain GenMC optimizations will lead to this.
-            unimplemented!("GenMC: load returned no value.");
-        }
-
         debug!("GenMC: load returned value: {:?}", load_result.read_value);
         interp_ok(load_result.read_value)
     }
 
-    /// Inform GenMC about a store (atomic or non-atomic).
+    /// Inform GenMC about a non-atomic load.
+    fn handle_non_atomic_load<'tcx>(
+        &self,
+        machine: &MiriMachine<'tcx>,
+        address: Size,
+        size: Size,
+    ) -> InterpResult<'tcx> {
+        assert!(size.bytes() != 0);
+        debug!(
+            "GenMC: NA load, address: {addr} == {addr:#x}, size: {size:?}",
+            addr = address.bytes()
+        );
+        let load_result = self.handle.borrow_mut().pin_mut().handle_non_atomic_load(
+            self.active_thread_genmc_tid(machine),
+            address.bytes(),
+            size.bytes(),
+        );
+
+        if load_result.invalid {
+            throw_machine_stop!(TerminationInfo::GenmcSkip);
+        }
+        if let Some(error) = load_result.error.as_ref() {
+            // FIXME(genmc): error handling
+            throw_ub_format!("{}", error.to_string_lossy());
+        }
+        // `load_result.read_value` is just a dummy for non-atomic loads. And anyway Miri doesn't
+        // give us a chance to change the value here, it'll always use the one from its memory.
+        interp_ok(())
+    }
+
+    /// Inform GenMC about an atomic store.
     /// Returns true if the store is co-maximal, i.e., it should be written to Miri's memory too.
-    fn handle_store<'tcx>(
+    fn handle_atomic_store<'tcx>(
         &self,
         machine: &MiriMachine<'tcx>,
         address: Size,
@@ -804,7 +782,7 @@ fn handle_store<'tcx>(
             "GenMC: store, address: {addr} = {addr:#x}, size: {size:?}, ordering {memory_ordering:?}, value: {genmc_value:?}",
             addr = address.bytes()
         );
-        let store_result = self.handle.borrow_mut().pin_mut().handle_store(
+        let store_result = self.handle.borrow_mut().pin_mut().handle_atomic_store(
             self.active_thread_genmc_tid(machine),
             address.bytes(),
             size.bytes(),
@@ -813,6 +791,9 @@ fn handle_store<'tcx>(
             memory_ordering,
         );
 
+        if store_result.invalid {
+            throw_machine_stop!(TerminationInfo::GenmcSkip);
+        }
         if let Some(error) = store_result.error.as_ref() {
             // FIXME(genmc): error handling
             throw_ub_format!("{}", error.to_string_lossy());
@@ -821,6 +802,36 @@ fn handle_store<'tcx>(
         interp_ok(store_result.is_coherence_order_maximal_write)
     }
 
+    /// Inform GenMC about a non-atomic store.
+    fn handle_non_atomic_store<'tcx>(
+        &self,
+        machine: &MiriMachine<'tcx>,
+        address: Size,
+        size: Size,
+    ) -> InterpResult<'tcx> {
+        assert!(size.bytes() != 0);
+        debug!(
+            "GenMC: NA store, address: {addr} = {addr:#x}, size: {size:?}",
+            addr = address.bytes()
+        );
+        let store_result = self.handle.borrow_mut().pin_mut().handle_non_atomic_store(
+            self.active_thread_genmc_tid(machine),
+            address.bytes(),
+            size.bytes(),
+        );
+
+        if store_result.invalid {
+            throw_machine_stop!(TerminationInfo::GenmcSkip);
+        }
+        if let Some(error) = store_result.error.as_ref() {
+            // FIXME(genmc): error handling
+            throw_ub_format!("{}", error.to_string_lossy());
+        }
+        // Miri will always write non-atomic stores to memory. Make sure GenMC agrees with that.
+        assert!(store_result.is_coherence_order_maximal_write);
+        interp_ok(())
+    }
+
     /// Inform GenMC about an atomic read-modify-write operation.
     /// This includes atomic swap (also often called "exchange"), but does *not*
     /// include compare-exchange (see `RMWBinOp` for full list of operations).
@@ -859,6 +870,9 @@ fn handle_atomic_rmw_op<'tcx>(
             genmc_old_value,
         );
 
+        if rmw_result.invalid {
+            throw_machine_stop!(TerminationInfo::GenmcSkip);
+        }
         if let Some(error) = rmw_result.error.as_ref() {
             // FIXME(genmc): error handling
             throw_ub_format!("{}", error.to_string_lossy());

src/tools/miri/src/concurrency/genmc/scheduling.rs

@@ -1,4 +1,4 @@
-use genmc_sys::{ActionKind, ExecutionState};
+use genmc_sys::{ActionKind, ExecutionStatus};
 use rustc_data_structures::either::Either;
 use rustc_middle::mir::TerminatorKind;
 use rustc_middle::ty::{self, Ty};
@@ -117,9 +117,9 @@ pub(crate) fn schedule_thread<'tcx>(
 
         let result = self.handle.borrow_mut().pin_mut().schedule_next(genmc_tid, atomic_kind);
         // Depending on the exec_state, we either schedule the given thread, or we are finished with this execution.
-        match result.exec_state {
-            ExecutionState::Ok => interp_ok(Some(thread_infos.get_miri_tid(result.next_thread))),
-            ExecutionState::Blocked => {
+        match result.exec_status {
+            ExecutionStatus::Ok => interp_ok(Some(thread_infos.get_miri_tid(result.next_thread))),
+            ExecutionStatus::Blocked => {
                 // This execution doesn't need further exploration. We treat this as "success, no
                 // leak check needed", which makes it a NOP in the big outer loop.
                 throw_machine_stop!(TerminationInfo::Exit {
@@ -127,7 +127,7 @@ pub(crate) fn schedule_thread<'tcx>(
                     leak_check: false,
                 });
             }
-            ExecutionState::Finished => {
+            ExecutionStatus::Finished => {
                 let exit_status = self.exec_state.exit_status.get().expect(
                     "If the execution is finished, we should have a return value from the program.",
                 );
@@ -136,7 +136,7 @@ pub(crate) fn schedule_thread<'tcx>(
                     leak_check: matches!(exit_status.exit_type, super::ExitType::MainThreadFinish),
                 });
             }
-            ExecutionState::Error => {
+            ExecutionStatus::Error => {
                 // GenMC found an error in one of the `handle_*` functions, but didn't return the detected error from the function immediately.
                 // This is still an bug in the user program, so we print the error string.
                 panic!(

src/tools/miri/src/data_structures/mono_hash_map.rs

@@ -96,10 +96,7 @@ fn get_or<E>(&self, k: K, vacant: impl FnOnce() -> Result<V, E>) -> Result<&V, E
 
     /// Read-only lookup (avoid read-acquiring the RefCell).
     fn get(&self, k: K) -> Option<&V> {
-        let val: *const V = match self.0.borrow().get(&k) {
-            Some(v) => &**v,
-            None => return None,
-        };
+        let val: *const V = &**self.0.borrow().get(&k)?;
         // This is safe because `val` points into a `Box`, that we know will not move and
         // will also not be dropped as long as the shared reference `self` is live.
         unsafe { Some(&*val) }

src/tools/miri/src/diagnostics.rs

@@ -18,7 +18,7 @@ pub enum TerminationInfo {
         leak_check: bool,
     },
     Abort(String),
-    /// Miri was interrupted by a Ctrl+C from the user
+    /// Miri was interrupted by a Ctrl+C from the user.
     Interrupted,
     UnsupportedInIsolation(String),
     StackedBorrowsUb {
@@ -32,6 +32,8 @@ pub enum TerminationInfo {
         history: tree_diagnostics::HistoryData,
     },
     Int2PtrWithStrictProvenance,
+    /// GenMC determined that the execution should stop.
+    GenmcSkip,
     /// All threads are blocked.
     GlobalDeadlock,
     /// Some thread discovered a deadlock condition (e.g. in a mutex with reentrancy checking).
@@ -81,6 +83,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
             TreeBorrowsUb { title, .. } => write!(f, "{title}"),
             GlobalDeadlock => write!(f, "the evaluated program deadlocked"),
             LocalDeadlock => write!(f, "a thread deadlocked"),
+            GenmcSkip => write!(f, "GenMC wants to skip this execution"),
             MultipleSymbolDefinitions { link_name, .. } =>
                 write!(f, "multiple definitions of symbol `{link_name}`"),
             SymbolShimClashing { link_name, .. } =>
@@ -240,6 +243,10 @@ pub fn report_result<'tcx>(
                 Some("unsupported operation"),
             StackedBorrowsUb { .. } | TreeBorrowsUb { .. } | DataRace { .. } =>
                 Some("Undefined Behavior"),
+            GenmcSkip => {
+                assert!(ecx.machine.data_race.as_genmc_ref().is_some());
+                return Some((0, false));
+            }
             LocalDeadlock => {
                 labels.push(format!("thread got stuck here"));
                 None

src/tools/miri/src/provenance_gc.rs

@@ -21,6 +21,10 @@ fn visit_provenance(&self, _visit: &mut VisitWith<'_>) {}
 }
 no_provenance!(i8 i16 i32 i64 isize u8 u16 u32 u64 usize bool ThreadId);
 
+impl VisitProvenance for &'static str {
+    fn visit_provenance(&self, _visit: &mut VisitWith<'_>) {}
+}
+
 impl<T: VisitProvenance> VisitProvenance for Option<T> {
     fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
         if let Some(x) = self {

src/tools/miri/src/shims/unix/fd.rs

@@ -62,6 +62,20 @@ fn flock<'tcx>(
         throw_unsup_format!("cannot flock {}", self.name());
     }
 
+    /// Modifies device parameters.
+    /// `op` is the device-dependent operation code. It's either a `c_long` or `c_int`, depending on
+    /// the target and whether it uses glibc or musl.
+    /// `arg` is the optional third argument which exists depending on the operation code. It's either
+    /// an integer or a pointer.
+    fn ioctl<'tcx>(
+        &self,
+        _op: Scalar,
+        _arg: Option<&OpTy<'tcx>>,
+        _ecx: &mut MiriInterpCx<'tcx>,
+    ) -> InterpResult<'tcx, i32> {
+        throw_unsup_format!("cannot use ioctl on {}", self.name());
+    }
+
     /// Return which epoll events are currently active.
     fn epoll_active_events<'tcx>(&self) -> InterpResult<'tcx, EpollEvents> {
         throw_unsup_format!("{}: epoll does not support this file description", self.name());
@@ -129,6 +143,39 @@ fn flock(&mut self, fd_num: i32, op: i32) -> InterpResult<'tcx, Scalar> {
         interp_ok(Scalar::from_i32(this.try_unwrap_io_result(result)?))
     }
 
+    fn ioctl(
+        &mut self,
+        fd: &OpTy<'tcx>,
+        op: &OpTy<'tcx>,
+        varargs: &[OpTy<'tcx>],
+    ) -> InterpResult<'tcx, Scalar> {
+        let this = self.eval_context_mut();
+
+        let fd = this.read_scalar(fd)?.to_i32()?;
+        let op = this.read_scalar(op)?;
+        // There is at most one relevant variadic argument.
+        // It exists depending on the device and the opcode and thus we can't
+        // use `check_min_vararg_count` here.
+        let arg = varargs.first();
+
+        let Some(fd) = this.machine.fds.get(fd) else {
+            return this.set_last_error_and_return_i32(LibcError("EBADF"));
+        };
+
+        // Handle common opcodes.
+        let fioclex = this.eval_libc("FIOCLEX");
+        let fionclex = this.eval_libc("FIONCLEX");
+        if op == fioclex || op == fionclex {
+            // Since we don't support `exec`, those are NOPs.
+            return interp_ok(Scalar::from_i32(0));
+        }
+
+        // Since some ioctl operations use the return value as an output parameter, we cannot strictly use the convention of
+        // zero indicating success and -1 indicating an error.
+        let return_value = fd.as_unix(this).ioctl(op, arg, this)?;
+        interp_ok(Scalar::from_i32(return_value))
+    }
+
     fn fcntl(
         &mut self,
         fd_num: &OpTy<'tcx>,

src/tools/miri/src/shims/unix/foreign_items.rs

@@ -307,6 +307,12 @@ fn emulate_foreign_item_inner(
                 let result = this.flock(fd, op)?;
                 this.write_scalar(result, dest)?;
             }
+            "ioctl" => {
+                let ([fd, op], varargs) =
+                    this.check_shim_sig_variadic_lenient(abi, CanonAbi::C, link_name, args)?;
+                let result = this.ioctl(fd, op, varargs)?;
+                this.write_scalar(result, dest)?;
+            }
 
             // File and file system access
             "open" => {
@@ -658,8 +664,7 @@ fn emulate_foreign_item_inner(
                     abi,
                     args,
                 )?;
-                let result = this.getpeername(socket, address, address_len)?;
-                this.write_scalar(result, dest)?;
+                this.getpeername(socket, address, address_len, dest)?;
             }
 
             // Time

src/tools/miri/src/shims/unix/macos/foreign_items.rs

@@ -80,12 +80,6 @@ fn emulate_foreign_item_inner(
                 let result = this.realpath(path, resolved_path)?;
                 this.write_scalar(result, dest)?;
             }
-            "ioctl" => {
-                let ([fd_num, cmd], varargs) =
-                    this.check_shim_sig_variadic_lenient(abi, CanonAbi::C, link_name, args)?;
-                let result = this.ioctl(fd_num, cmd, varargs)?;
-                this.write_scalar(result, dest)?;
-            }
 
             // Environment related shims
             "_NSGetEnviron" => {
@@ -341,30 +335,4 @@ fn emulate_foreign_item_inner(
 
         interp_ok(EmulateItemResult::NeedsReturn)
     }
-
-    fn ioctl(
-        &mut self,
-        fd_num: &OpTy<'tcx>,
-        cmd: &OpTy<'tcx>,
-        _varargs: &[OpTy<'tcx>],
-    ) -> InterpResult<'tcx, Scalar> {
-        let this = self.eval_context_mut();
-
-        let fioclex = this.eval_libc_u64("FIOCLEX");
-
-        let fd_num = this.read_scalar(fd_num)?.to_i32()?;
-        let cmd = this.read_scalar(cmd)?.to_u64()?;
-
-        if cmd == fioclex {
-            // Since we don't support `exec`, this is a NOP. However, we want to
-            // return EBADF if the FD is invalid.
-            if this.machine.fds.is_fd_num(fd_num) {
-                interp_ok(Scalar::from_i32(0))
-            } else {
-                this.set_last_error_and_return_i32(LibcError("EBADF"))
-            }
-        } else {
-            throw_unsup_format!("ioctl: unsupported command {cmd:#x}");
-        }
-    }
 }

src/tools/miri/src/shims/unix/socket.rs

@@ -1,6 +1,7 @@
 use std::cell::{Cell, RefCell};
 use std::io::Read;
 use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
+use std::time::Duration;
 use std::{io, iter};
 
 use mio::Interest;
@@ -12,8 +13,10 @@
 use rustc_middle::throw_unsup_format;
 use rustc_target::spec::Os;
 
+use crate::concurrency::blocking_io::InterestReceiver;
 use crate::shims::files::{EvalContextExt as _, FdId, FileDescription, FileDescriptionRef};
-use crate::{OpTy, Scalar, *};
+use crate::shims::unix::UnixFileDescription;
+use crate::*;
 
 #[derive(Debug, PartialEq)]
 enum SocketFamily {
@@ -23,22 +26,6 @@ enum SocketFamily {
     IPv6,
 }
 
-enum SocketIoError {
-    /// The socket is not yet ready. Either EINPROGRESS or ENOTCONNECTED occurred.
-    NotReady,
-    /// Any other kind of I/O error.
-    Other(io::Error),
-}
-
-impl From<io::Error> for SocketIoError {
-    fn from(value: io::Error) -> Self {
-        match value.kind() {
-            io::ErrorKind::InProgress | io::ErrorKind::NotConnected => Self::NotReady,
-            _ => Self::Other(value),
-        }
-    }
-}
-
 #[derive(Debug)]
 enum SocketState {
     /// No syscall after `socket` has been made.
@@ -61,59 +48,6 @@ enum SocketState {
     Connected(TcpStream),
 }
 
-impl SocketState {
-    /// If the socket is currently in [`SocketState::Connecting`], try to ensure
-    /// that the connection is established by first checking that [`TcpStream::take_error`]
-    /// doesn't return an error and then by checking that [`TcpStream::peer_addr`]
-    /// returns the address of the connected peer.
-    ///
-    /// If the connection is established or the socket is in any other state,
-    /// [`Ok`] is returned.
-    ///
-    /// **Important**: On Windows hosts this function can only be used to ensure a socket is connected
-    /// _after_ a [`Interest::WRITABLE`] event was received.
-    pub fn try_set_connected(&mut self) -> Result<(), SocketIoError> {
-        // Further explanation of the limitation on Windows hosts:
-        // Windows treats sockets which are connecting as connected until either the connection timeout hits
-        // or an error occurs. Thus, the [`TcpStream::peer_addr`] method returns [`Ok`] with the provided peer
-        // address even when the connection might not yet be established.
-
-        let SocketState::Connecting(stream) = self else { return Ok(()) };
-
-        if let Ok(Some(e)) = stream.take_error() {
-            // There was an error whilst connecting.
-            let e = SocketIoError::from(e);
-            // We won't get EINPROGRESS or ENOTCONNECTED here
-            // so we need to reset the state.
-            assert!(matches!(e, SocketIoError::Other(_)));
-            // Go back to initial state as the only way of getting into the
-            // `Connecting` state is from the `Initial` state.
-            *self = SocketState::Initial;
-            return Err(e);
-        }
-
-        if let Err(e) = stream.peer_addr() {
-            let e = SocketIoError::from(e);
-            if let SocketIoError::Other(_) = &e {
-                // All other errors are fatal for a socket and thus the state needs to be reset.
-                *self = SocketState::Initial;
-            }
-            return Err(e);
-        };
-
-        // We just read the peer address without an error so we can be
-        // sure that the connection is established.
-
-        // Temporarily use dummy state to take ownership of the stream.
-        let SocketState::Connecting(stream) = std::mem::replace(self, SocketState::Initial) else {
-            // At the start of the function we ensured that we're currently connecting.
-            unreachable!()
-        };
-        *self = SocketState::Connected(stream);
-        Ok(())
-    }
-}
-
 #[derive(Debug)]
 struct Socket {
     /// Family of the socket, used to ensure socket only binds/connects to address of
@@ -151,17 +85,40 @@ fn read<'tcx>(
     ) -> InterpResult<'tcx> {
         assert!(communicate_allowed, "cannot have `Socket` with isolation enabled!");
 
-        if !matches!(&*self.state.borrow(), SocketState::Connected(_)) {
-            // We can only receive from connected sockets. For all other
-            // states we return a not connected error.
-            return finish.call(ecx, Err(LibcError("ENOTCONN")));
-        }
+        let socket = self;
 
-        // Since `read` is the same as `recv` with no flags, we just treat
-        // the `read` as a `recv` here.
-        ecx.block_for_recv(self, ptr, len, /* should_peek */ false, finish);
+        ecx.ensure_connected(
+            socket.clone(),
+            !socket.is_non_block.get(),
+            "read",
+            callback!(
+                @capture<'tcx> {
+                    socket: FileDescriptionRef<Socket>,
+                    ptr: Pointer,
+                    len: usize,
+                    finish: DynMachineCallback<'tcx, Result<usize, IoError>>,
+                } |this, result: Result<(), ()>| {
+                    if result.is_err() {
+                        return finish.call(this, Err(LibcError("ENOTCONN")))
+                    }
 
-        interp_ok(())
+                    // Since `read` is the same as `recv` with no flags, we just treat
+                    // the `read` as a `recv` here.
+
+                    if socket.is_non_block.get() {
+                        // We have a non-blocking socket and thus don't want to block until
+                        // we can read.
+                        let result = this.try_non_block_recv(&socket, ptr, len, /* should_peek */ false)?;
+                        finish.call(this, result)
+                    } else {
+                        // The socket is in blocking mode and thus the read call should block
+                        // until we can read some bytes from the socket.
+                        this.block_for_recv(socket, ptr, len, /* should_peek */ false, finish);
+                        interp_ok(())
+                    }
+                }
+            ),
+        )
     }
 
     fn write<'tcx>(
@@ -174,17 +131,40 @@ fn write<'tcx>(
     ) -> InterpResult<'tcx> {
         assert!(communicate_allowed, "cannot have `Socket` with isolation enabled!");
 
-        if !matches!(&*self.state.borrow(), SocketState::Connected(_)) {
-            // We can only send with connected sockets. For all other
-            // states we return a not connected error.
-            return finish.call(ecx, Err(LibcError("ENOTCONN")));
-        }
+        let socket = self;
 
-        // Since `write` is the same as `send` with no flags, we just treat
-        // the `write` as a `send` here.
-        ecx.block_for_send(self, ptr, len, finish);
+        ecx.ensure_connected(
+            socket.clone(),
+            !socket.is_non_block.get(),
+            "write",
+            callback!(
+                @capture<'tcx> {
+                    socket: FileDescriptionRef<Socket>,
+                    ptr: Pointer,
+                    len: usize,
+                    finish: DynMachineCallback<'tcx, Result<usize, IoError>>
+                } |this, result: Result<(), ()>| {
+                    if result.is_err() {
+                        return finish.call(this, Err(LibcError("ENOTCONN")))
+                    }
 
-        interp_ok(())
+                    // Since `write` is the same as `send` with no flags, we just treat
+                    // the `write` as a `send` here.
+
+                    if socket.is_non_block.get() {
+                        // We have a non-blocking socket and thus don't want to block until
+                        // we can write.
+                        let result = this.try_non_block_send(&socket, ptr, len)?;
+                        return finish.call(this, result)
+                    } else {
+                        // The socket is in blocking mode and thus the write call should block
+                        // until we can write some bytes into the socket.
+                        this.block_for_send(socket, ptr, len, finish);
+                        interp_ok(())
+                    }
+                }
+            ),
+        )
     }
 
     fn short_fd_operations(&self) -> bool {
@@ -192,6 +172,10 @@ fn short_fd_operations(&self) -> bool {
         true
     }
 
+    fn as_unix<'tcx>(&self, _ecx: &MiriInterpCx<'tcx>) -> &dyn UnixFileDescription {
+        self
+    }
+
     fn get_flags<'tcx>(&self, ecx: &mut MiriInterpCx<'tcx>) -> InterpResult<'tcx, Scalar> {
         let mut flags = ecx.eval_libc_i32("O_RDWR");
 
@@ -204,10 +188,64 @@ fn get_flags<'tcx>(&self, ecx: &mut MiriInterpCx<'tcx>) -> InterpResult<'tcx, Sc
 
     fn set_flags<'tcx>(
         &self,
-        mut _flag: i32,
-        _ecx: &mut MiriInterpCx<'tcx>,
+        mut flag: i32,
+        ecx: &mut MiriInterpCx<'tcx>,
     ) -> InterpResult<'tcx, Scalar> {
-        throw_unsup_format!("fcntl: socket flags aren't supported")
+        let o_nonblock = ecx.eval_libc_i32("O_NONBLOCK");
+
+        // O_NONBLOCK flag can be set / unset by user.
+        if flag & o_nonblock == o_nonblock {
+            self.is_non_block.set(true);
+            flag &= !o_nonblock;
+        } else {
+            self.is_non_block.set(false);
+        }
+
+        // Throw error if there is any unsupported flag.
+        if flag != 0 {
+            throw_unsup_format!("fcntl: only O_NONBLOCK is supported for sockets")
+        }
+
+        interp_ok(Scalar::from_i32(0))
+    }
+}
+
+impl UnixFileDescription for Socket {
+    fn ioctl<'tcx>(
+        &self,
+        op: Scalar,
+        arg: Option<&OpTy<'tcx>>,
+        ecx: &mut MiriInterpCx<'tcx>,
+    ) -> InterpResult<'tcx, i32> {
+        assert!(ecx.machine.communicate(), "cannot have `Socket` with isolation enabled!");
+
+        let fionbio = ecx.eval_libc("FIONBIO");
+
+        if op == fionbio {
+            // On these OSes, Rust uses the ioctl, so we trust that it is reasonable and controls
+            // the same internal flag as fcntl.
+            if !matches!(ecx.tcx.sess.target.os, Os::Linux | Os::Android | Os::MacOs | Os::FreeBsd)
+            {
+                // FIONBIO cannot be used to change the blocking mode of a socket on solarish targets:
+                // <https://github.com/rust-lang/rust/commit/dda5c97675b4f5b1f6fdab64606c8a1f21021b0a>
+                // Since there might be more targets which do weird things with this option, we use
+                // an allowlist instead of just denying solarish targets.
+                throw_unsup_format!(
+                    "ioctl: setting FIONBIO on sockets is unsupported on target {}",
+                    ecx.tcx.sess.target.os
+                );
+            }
+
+            let Some(value_ptr) = arg else {
+                throw_ub_format!("ioctl: setting FIONBIO on sockets requires a third argument");
+            };
+            let value = ecx.deref_pointer_as(value_ptr, ecx.machine.layouts.i32)?;
+            let non_block = ecx.read_scalar(&value)?.to_i32()? != 0;
+            self.is_non_block.set(non_block);
+            return interp_ok(0);
+        }
+
+        throw_unsup_format!("ioctl: unsupported operation {op:#x} on socket");
     }
 }
 
@@ -469,19 +507,35 @@ fn accept4(
         }
 
         if socket.is_non_block.get() {
-            throw_unsup_format!("accept4: non-blocking accept is unsupported")
+            // We have a non-blocking socket and thus don't want to block until
+            // we can accept an incoming connection.
+            match this.try_non_block_accept(
+                &socket,
+                address_ptr,
+                address_len_ptr,
+                is_client_sock_nonblock,
+            )? {
+                Ok(sockfd) => {
+                    // We need to create the scalar using the destination size since
+                    // `syscall(SYS_accept4, ...)` returns a long which doesn't match
+                    // the int returned from the `accept`/`accept4` syscalls.
+                    // See <https://man7.org/linux/man-pages/man2/syscall.2.html>.
+                    this.write_scalar(Scalar::from_int(sockfd, dest.layout.size), dest)
+                }
+                Err(e) => this.set_last_error_and_return(e, dest),
+            }
+        } else {
+            // The socket is in blocking mode and thus the accept call should block
+            // until an incoming connection is ready.
+            this.block_for_accept(
+                socket,
+                address_ptr,
+                address_len_ptr,
+                is_client_sock_nonblock,
+                dest.clone(),
+            );
+            interp_ok(())
         }
-
-        // The socket is in blocking mode and thus the accept call should block
-        // until an incoming connection is ready.
-        this.block_for_accept(
-            address_ptr,
-            address_len_ptr,
-            is_client_sock_nonblock,
-            socket,
-            dest.clone(),
-        );
-        interp_ok(())
     }
 
     fn connect(
@@ -530,22 +584,44 @@ fn connect(
         // Mio returns a potentially unconnected stream.
         // We can be ensured that the connection is established when
         // [`TcpStream::take_err`] and [`TcpStream::peer_addr`] both
-        // don't return errors.
-        // For non-blocking sockets we need to check that for every
-        // [`Interest::WRITEABLE`] event on the stream.
+        // don't return an error after receiving an [`Interest::WRITEABLE`]
+        // event on the stream.
         match TcpStream::connect(address) {
             Ok(stream) => *socket.state.borrow_mut() = SocketState::Connecting(stream),
             Err(e) => return this.set_last_error_and_return(e, dest),
         };
 
         if socket.is_non_block.get() {
-            throw_unsup_format!("connect: non-blocking connect is unsupported");
-        }
+            // We have a non-blocking socket and thus don't want to block until
+            // the connection is established.
 
-        // The socket is in blocking mode and thus the connect call should block
-        // until the connection with the server is established.
-        this.block_for_connect(socket, dest.clone());
-        interp_ok(())
+            // Since the [`TcpStream::connect`] function of mio hides the EINPROGRESS
+            // we just always return EINPROGRESS and check whether the connection succeeded
+            // once we want to use the connected socket.
+            this.set_last_error_and_return(LibcError("EINPROGRESS"), dest)
+        } else {
+            // The socket is in blocking mode and thus the connect call should block
+            // until the connection with the server is established.
+
+            let dest = dest.clone();
+
+            this.ensure_connected(
+                socket,
+                /* should_wait */ true,
+                "connect",
+                callback!(
+                    @capture<'tcx> {
+                        dest: MPlaceTy<'tcx>
+                    } |this, result: Result<(), ()>| {
+                        if result.is_err() {
+                            this.set_last_error_and_return(LibcError("ENOTCONN"), &dest)
+                        } else {
+                            this.write_scalar(Scalar::from_i32(0), &dest)
+                        }
+                    }
+                ),
+            )
+        }
     }
 
     fn send(
@@ -576,12 +652,6 @@ fn send(
             return this.set_last_error_and_return(LibcError("ENOTSOCK"), dest);
         };
 
-        if !matches!(&*socket.state.borrow(), SocketState::Connected(_)) {
-            // We can only send with connected sockets. For all other
-            // states we return a not connected error.
-            return this.set_last_error_and_return(LibcError("ENOTCONN"), dest);
-        }
-
         // Non-deterministically decide to further reduce the length, simulating a partial send.
         // We avoid reducing the write size to 0: the docs seem to be entirely fine with that,
         // but the standard library is not (https://github.com/rust-lang/rust/issues/145959).
@@ -594,50 +664,86 @@ fn send(
             length
         };
 
+        let mut is_op_non_block = false;
+
         // Interpret the flag. Every flag we recognize is "subtracted" from `flags`, so
         // if there is anything left at the end, that's an unsupported flag.
         if matches!(
             this.tcx.sess.target.os,
             Os::Linux | Os::Android | Os::FreeBsd | Os::Solaris | Os::Illumos
         ) {
-            // MSG_NOSIGNAL only exists on Linux, Android, FreeBSD,
+            // MSG_NOSIGNAL and MSG_DONTWAIT only exist on Linux, Android, FreeBSD,
             // Solaris, and Illumos targets.
             let msg_nosignal = this.eval_libc_i32("MSG_NOSIGNAL");
+            let msg_dontwait = this.eval_libc_i32("MSG_DONTWAIT");
             if flags & msg_nosignal == msg_nosignal {
                 // This is only needed to ensure that no EPIPE signal is sent when
                 // trying to send into a stream which is no longer connected.
                 // Since we don't support signals, we can ignore this.
                 flags &= !msg_nosignal;
             }
+            if flags & msg_dontwait == msg_dontwait {
+                flags &= !msg_dontwait;
+                is_op_non_block = true;
+            }
         }
 
         if flags != 0 {
             throw_unsup_format!(
-                "send: flag {flags:#x} is unsupported, only MSG_NOSIGNAL is allowed",
+                "send: flag {flags:#x} is unsupported, only MSG_NOSIGNAL and MSG_DONTWAIT are allowed",
             );
         }
 
+        // If either the operation or the socket is non-blocking, we don't want
+        // to wait until the connection is established.
+        let should_wait = !is_op_non_block && !socket.is_non_block.get();
         let dest = dest.clone();
 
-        this.block_for_send(
-            socket,
-            buffer_ptr,
-            length,
-            callback!(@capture<'tcx> {
-                dest: MPlaceTy<'tcx>
-            } |this, result: Result<usize, IoError>| {
-                match result {
-                    Ok(read_size) => {
-                        let read_size: u64 = read_size.try_into().unwrap();
-                        let ssize_layout = this.libc_ty_layout("ssize_t");
-                        this.write_scalar(Scalar::from_int(read_size, ssize_layout.size), &dest)
+        this.ensure_connected(
+            socket.clone(),
+            should_wait,
+            "send",
+            callback!(
+                @capture<'tcx> {
+                    socket: FileDescriptionRef<Socket>,
+                    flags: i32,
+                    buffer_ptr: Pointer,
+                    length: usize,
+                    is_op_non_block: bool,
+                    dest: MPlaceTy<'tcx>,
+                } |this, result: Result<(), ()>| {
+                    if result.is_err() {
+                        return this.set_last_error_and_return(LibcError("ENOTCONN"), &dest)
                     }
-                    Err(e) => this.set_last_error_and_return(e, &dest)
-                }
-            }),
-        );
 
-        interp_ok(())
+                    if is_op_non_block || socket.is_non_block.get() {
+                        // We have a non-blocking operation or a non-blocking socket and
+                        // thus don't want to block until we can send.
+                        match this.try_non_block_send(&socket, buffer_ptr, length)? {
+                            Ok(size) => this.write_scalar(Scalar::from_target_isize(size.try_into().unwrap(), this), &dest),
+                            Err(e) => this.set_last_error_and_return(e, &dest),
+                        }
+                    } else {
+                        // The socket is in blocking mode and thus the send call should block
+                        // until we can send some bytes into the socket.
+                        this.block_for_send(
+                            socket,
+                            buffer_ptr,
+                            length,
+                            callback!(@capture<'tcx> {
+                                dest: MPlaceTy<'tcx>
+                            } |this, result: Result<usize, IoError>| {
+                                match result {
+                                    Ok(size) => this.write_scalar(Scalar::from_target_isize(size.try_into().unwrap(), this), &dest),
+                                    Err(e) => this.set_last_error_and_return(e, &dest)
+                                }
+                            }),
+                        );
+                        interp_ok(())
+                    }
+                }
+            ),
+        )
     }
 
     fn recv(
@@ -668,12 +774,6 @@ fn recv(
             return this.set_last_error_and_return(LibcError("ENOTSOCK"), dest);
         };
 
-        if !matches!(&*socket.state.borrow(), SocketState::Connected(_)) {
-            // We can only receive from connected sockets. For all other
-            // states we return a not connected error.
-            return this.set_last_error_and_return(LibcError("ENOTCONN"), dest);
-        }
-
         // Non-deterministically decide to further reduce the length, simulating a partial receive.
         // We don't simulate partial receives for lengths < 2 because the man page states that a
         // return value of zero can only be returned in some special cases:
@@ -690,6 +790,7 @@ fn recv(
         };
 
         let mut should_peek = false;
+        let mut is_op_non_block = false;
 
         // Interpret the flag. Every flag we recognize is "subtracted" from `flags`, so
         // if there is anything left at the end, that's an unsupported flag.
@@ -710,35 +811,77 @@ fn recv(
             }
         }
 
+        if matches!(
+            this.tcx.sess.target.os,
+            Os::Linux | Os::Android | Os::FreeBsd | Os::Solaris | Os::Illumos
+        ) {
+            // MSG_DONTWAIT only exists on Linux, Android, FreeBSD,
+            // Solaris, and Illumos targets.
+            let msg_dontwait = this.eval_libc_i32("MSG_DONTWAIT");
+            if flags & msg_dontwait == msg_dontwait {
+                flags &= !msg_dontwait;
+                is_op_non_block = true;
+            }
+        }
+
         if flags != 0 {
             throw_unsup_format!(
-                "recv: flag {flags:#x} is unsupported, only MSG_PEEK \
+                "recv: flag {flags:#x} is unsupported, only MSG_PEEK, MSG_DONTWAIT \
                 and MSG_CMSG_CLOEXEC are allowed",
             );
         }
 
+        // If either the operation or the socket is non-blocking, we don't want
+        // to wait until the connection is established.
+        let should_wait = !is_op_non_block && !socket.is_non_block.get();
         let dest = dest.clone();
 
-        this.block_for_recv(
-            socket,
-            buffer_ptr,
-            length,
-            should_peek,
-            callback!(@capture<'tcx> {
-                dest: MPlaceTy<'tcx>
-            } |this, result: Result<usize, IoError>| {
-                match result {
-                    Ok(read_size) => {
-                        let read_size: u64 = read_size.try_into().unwrap();
-                        let ssize_layout = this.libc_ty_layout("ssize_t");
-                        this.write_scalar(Scalar::from_int(read_size, ssize_layout.size), &dest)
+        this.ensure_connected(
+            socket.clone(),
+            should_wait,
+            "recv",
+            callback!(
+                @capture<'tcx> {
+                    socket: FileDescriptionRef<Socket>,
+                    buffer_ptr: Pointer,
+                    length: usize,
+                    should_peek: bool,
+                    is_op_non_block: bool,
+                    dest: MPlaceTy<'tcx>,
+                } |this, result: Result<(), ()>| {
+                    if result.is_err() {
+                        return this.set_last_error_and_return(LibcError("ENOTCONN"), &dest)
                     }
-                    Err(e) => this.set_last_error_and_return(e, &dest)
-                }
-            }),
-        );
 
-        interp_ok(())
+                    if is_op_non_block || socket.is_non_block.get() {
+                        // We have a non-blocking operation or a non-blocking socket and
+                        // thus don't want to block until we can receive.
+                        match this.try_non_block_recv(&socket, buffer_ptr, length, should_peek)? {
+                            Ok(size) => this.write_scalar(Scalar::from_target_isize(size.try_into().unwrap(), this), &dest),
+                            Err(e) => this.set_last_error_and_return(e, &dest),
+                        }
+                    } else {
+                        // The socket is in blocking mode and thus the receive call should block
+                        // until we can receive some bytes from the socket.
+                        this.block_for_recv(
+                            socket,
+                            buffer_ptr,
+                            length,
+                            should_peek,
+                            callback!(@capture<'tcx> {
+                                dest: MPlaceTy<'tcx>
+                            } |this, result: Result<usize, IoError>| {
+                                match result {
+                                    Ok(size) => this.write_scalar(Scalar::from_target_isize(size.try_into().unwrap(), this), &dest),
+                                    Err(e) => this.set_last_error_and_return(e, &dest)
+                                }
+                            }),
+                        );
+                        interp_ok(())
+                    }
+                }
+            ),
+        )
     }
 
     fn setsockopt(
@@ -871,7 +1014,9 @@ fn getpeername(
         socket: &OpTy<'tcx>,
         address: &OpTy<'tcx>,
         address_len: &OpTy<'tcx>,
-    ) -> InterpResult<'tcx, Scalar> {
+        // Location where the output scalar is written to.
+        dest: &MPlaceTy<'tcx>,
+    ) -> InterpResult<'tcx> {
         let this = self.eval_context_mut();
 
         let socket = this.read_scalar(socket)?.to_i32()?;
@@ -880,32 +1025,56 @@ fn getpeername(
 
         // Get the file handle
         let Some(fd) = this.machine.fds.get(socket) else {
-            return this.set_last_error_and_return_i32(LibcError("EBADF"));
+            return this.set_last_error_and_return(LibcError("EBADF"), dest);
         };
 
         let Some(socket) = fd.downcast::<Socket>() else {
             // Man page specifies to return ENOTSOCK if `fd` is not a socket.
-            return this.set_last_error_and_return_i32(LibcError("ENOTSOCK"));
+            return this.set_last_error_and_return(LibcError("ENOTSOCK"), dest);
         };
 
         assert!(this.machine.communicate(), "cannot have `Socket` with isolation enabled!");
 
-        let state = socket.state.borrow();
+        let dest = dest.clone();
 
-        let SocketState::Connected(stream) = &*state else {
-            // We can only read the peer address of connected sockets.
-            return this.set_last_error_and_return_i32(LibcError("ENOTCONN"));
-        };
+        // It's only safe to call [`TcpStream::peer_addr`] after the socket is connected since
+        // UNIX targets should return ENOTCONN when the connection is not yet established.
+        this.ensure_connected(
+            socket.clone(),
+            /* should_wait */ false,
+            "getpeername",
+            callback!(
+                @capture<'tcx> {
+                    socket: FileDescriptionRef<Socket>,
+                    address_ptr: Pointer,
+                    address_len_ptr: Pointer,
+                    dest: MPlaceTy<'tcx>,
+                } |this, result: Result<(), ()>| {
+                    if result.is_err() {
+                        return this.set_last_error_and_return(LibcError("ENOTCONN"), &dest)
+                    };
 
-        let address = match stream.peer_addr() {
-            Ok(address) => address,
-            Err(e) => return this.set_last_error_and_return_i32(e),
-        };
+                    let SocketState::Connected(stream) = &*socket.state.borrow() else {
+                        unreachable!()
+                    };
 
-        match this.write_socket_address(&address, address_ptr, address_len_ptr, "getpeername")? {
-            Ok(_) => interp_ok(Scalar::from_i32(0)),
-            Err(e) => this.set_last_error_and_return_i32(e),
-        }
+                    let address = match stream.peer_addr() {
+                        Ok(address) => address,
+                        Err(e) => return this.set_last_error_and_return(e, &dest),
+                    };
+
+                    match this.write_socket_address(
+                        &address,
+                        address_ptr,
+                        address_len_ptr,
+                        "getpeername",
+                    )? {
+                        Ok(_) => this.write_scalar(Scalar::from_i32(0), &dest),
+                        Err(e) => this.set_last_error_and_return(e, &dest),
+                    }
+                }
+            ),
+        )
     }
 }
 
@@ -1182,12 +1351,15 @@ fn write_socket_address(
     /// Block the thread until there's an incoming connection or an error occurred.
     ///
     /// This recursively calls itself should the operation still block for some reason.
+    ///
+    /// **Note**: This function is only safe to call when having previously ensured
+    /// that the socket is in [`SocketState::Listening`].
     fn block_for_accept(
         &mut self,
+        socket: FileDescriptionRef<Socket>,
         address_ptr: Pointer,
         address_len_ptr: Pointer,
         is_client_sock_nonblock: bool,
-        socket: FileDescriptionRef<Socket>,
         dest: MPlaceTy<'tcx>,
     ) {
         let this = self.eval_context_mut();
@@ -1204,89 +1376,83 @@ fn block_for_accept(
             } |this, kind: UnblockKind| {
                 assert_eq!(kind, UnblockKind::Ready);
 
-                let state = socket.state.borrow();
-
-                let SocketState::Listening(listener) = &*state else {
-                    // We checked that the socket is in listening state before blocking
-                    // and since there is no outgoing transition from that state this
-                    // should be unreachable.
-                    unreachable!()
-                };
-
-                let (stream, addr) = match listener.accept() {
-                    Ok(peer) => peer,
-                    Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
-                        // We need to block the thread again as it would still block.
-                        drop(state);
-                        this.block_for_accept(address_ptr, address_len_ptr, is_client_sock_nonblock, socket, dest);
-                        return interp_ok(())
+                match this.try_non_block_accept(&socket, address_ptr, address_len_ptr, is_client_sock_nonblock)? {
+                    Ok(sockfd) => {
+                        // We need to create the scalar using the destination size since
+                        // `syscall(SYS_accept4, ...)` returns a long which doesn't match
+                        // the int returned from the `accept`/`accept4` syscalls.
+                        // See <https://man7.org/linux/man-pages/man2/syscall.2.html>.
+                        this.write_scalar(Scalar::from_int(sockfd, dest.layout.size), &dest)
                     },
-                    Err(e) => return this.set_last_error_and_return(e, &dest),
-                };
-
-                let family = match addr {
-                    SocketAddr::V4(_) => SocketFamily::IPv4,
-                    SocketAddr::V6(_) => SocketFamily::IPv6,
-                };
-
-                if address_ptr != Pointer::null() {
-                    // We only attempt a write if the address pointer is not a null pointer.
-                    // If the address pointer is a null pointer the user isn't interested in the
-                    // address and we don't need to write anything.
-                    if let Err(e) = this.write_socket_address(&addr, address_ptr, address_len_ptr, "accept4")? {
-                      return this.set_last_error_and_return(e, &dest);
-                    };
+                    Err(IoError::HostError(e)) if e.kind() == io::ErrorKind::WouldBlock => {
+                        // We need to block the thread again as it would still block.
+                        this.block_for_accept(socket, address_ptr, address_len_ptr, is_client_sock_nonblock, dest);
+                        interp_ok(())
+                    }
+                    Err(e) => this.set_last_error_and_return(e, &dest),
                 }
-
-                let fd = this.machine.fds.new_ref(Socket {
-                    family,
-                    state: RefCell::new(SocketState::Connected(stream)),
-                    is_non_block: Cell::new(is_client_sock_nonblock),
-                });
-                let sockfd = this.machine.fds.insert(fd);
-                // We need to create the scalar using the destination size since
-                // `syscall(SYS_accept4, ...)` returns a long which doesn't match
-                // the int returned from the `accept`/`accept4` syscalls.
-                // See <https://man7.org/linux/man-pages/man2/syscall.2.html>.
-                this.write_scalar(Scalar::from_int(sockfd, dest.layout.size), &dest)
             }),
         );
     }
 
-    /// Block the thread until the stream is connected or an error occurred.
-    fn block_for_connect(&mut self, socket: FileDescriptionRef<Socket>, dest: MPlaceTy<'tcx>) {
+    /// Attempt to accept an incoming connection on the listening socket in a
+    /// non-blocking manner.
+    ///
+    /// **Note**: This function is only safe to call when having previously ensured
+    /// that the socket is in [`SocketState::Listening`].
+    fn try_non_block_accept(
+        &mut self,
+        socket: &FileDescriptionRef<Socket>,
+        address_ptr: Pointer,
+        address_len_ptr: Pointer,
+        is_client_sock_nonblock: bool,
+    ) -> InterpResult<'tcx, Result<i32, IoError>> {
         let this = self.eval_context_mut();
-        this.block_thread_for_io(
-            socket.clone(),
-            Interest::WRITABLE,
-            None,
-            callback!(@capture<'tcx> {
-                socket: FileDescriptionRef<Socket>,
-                dest: MPlaceTy<'tcx>,
-            } |this, kind: UnblockKind| {
-                assert_eq!(kind, UnblockKind::Ready);
 
-                let mut state = socket.state.borrow_mut();
+        let state = socket.state.borrow();
+        let SocketState::Listening(listener) = &*state else {
+            panic!(
+                "try_non_block_accept must only be called when socket is in `SocketState::Listening`"
+            )
+        };
 
-                // We received a "writable" event so `try_set_connected` is safe to call.
-                match state.try_set_connected() {
-                    Ok(_) => this.write_scalar(Scalar::from_i32(0), &dest),
-                     Err(SocketIoError::NotReady) => {
-                        // We need to block the thread again as the connection is still not yet ready.
-                        drop(state);
-                        this.block_for_connect(socket, dest);
-                        return interp_ok(())
-                    },
-                    Err(SocketIoError::Other(e)) => return this.set_last_error_and_return(e, &dest)
-                }
-            }),
-        );
+        let (stream, addr) = match listener.accept() {
+            Ok(peer) => peer,
+            Err(e) => return interp_ok(Err(IoError::HostError(e))),
+        };
+
+        let family = match addr {
+            SocketAddr::V4(_) => SocketFamily::IPv4,
+            SocketAddr::V6(_) => SocketFamily::IPv6,
+        };
+
+        if address_ptr != Pointer::null() {
+            // We only attempt a write if the address pointer is not a null pointer.
+            // If the address pointer is a null pointer the user isn't interested in the
+            // address and we don't need to write anything.
+            if let Err(e) =
+                this.write_socket_address(&addr, address_ptr, address_len_ptr, "accept4")?
+            {
+                return interp_ok(Err(e));
+            };
+        }
+
+        let fd = this.machine.fds.new_ref(Socket {
+            family,
+            state: RefCell::new(SocketState::Connected(stream)),
+            is_non_block: Cell::new(is_client_sock_nonblock),
+        });
+        let sockfd = this.machine.fds.insert(fd);
+        interp_ok(Ok(sockfd))
     }
 
     /// Block the thread until we can send bytes into the connected socket
     /// or an error occurred.
     ///
     /// This recursively calls itself should the operation still block for some reason.
+    ///
+    /// **Note**: This function is only safe to call when having previously ensured
+    /// that the socket is in [`SocketState::Connected`].
     fn block_for_send(
         &mut self,
         socket: FileDescriptionRef<Socket>,
@@ -1307,18 +1473,8 @@ fn block_for_send(
             } |this, kind: UnblockKind| {
                 assert_eq!(kind, UnblockKind::Ready);
 
-                let mut state = socket.state.borrow_mut();
-                let SocketState::Connected(stream) = &mut*state else {
-                    // We ensured that the socket is connected before blocking.
-                    unreachable!()
-                };
-
-                // This is a *non-blocking* write.
-                let result = this.write_to_host(stream, length, buffer_ptr)?;
-                match result {
+                match this.try_non_block_send(&socket, buffer_ptr, length)? {
                     Err(IoError::HostError(e)) if e.kind() == io::ErrorKind::WouldBlock => {
-                        // We need to block the thread again as it would still block.
-                        drop(state);
                         this.block_for_send(socket, buffer_ptr, length, finish);
                         interp_ok(())
                     },
@@ -1328,10 +1484,41 @@ fn block_for_send(
         );
     }
 
+    /// Attempt to send bytes into the connected socket in a non-blocking manner.
+    ///
+    /// **Note**: This function is only safe to call when having previously ensured
+    /// that the socket is in [`SocketState::Connected`].
+    fn try_non_block_send(
+        &mut self,
+        socket: &FileDescriptionRef<Socket>,
+        buffer_ptr: Pointer,
+        length: usize,
+    ) -> InterpResult<'tcx, Result<usize, IoError>> {
+        let this = self.eval_context_mut();
+
+        let SocketState::Connected(stream) = &mut *socket.state.borrow_mut() else {
+            panic!("try_non_block_send must only be called when the socket is connected")
+        };
+
+        // This is a *non-blocking* write.
+        let result = this.write_to_host(stream, length, buffer_ptr)?;
+        match result {
+            Err(IoError::HostError(e)) if e.kind() == io::ErrorKind::NotConnected => {
+                // On Windows hosts, `send` can return WSAENOTCONN where EAGAIN or EWOULDBLOCK
+                // would be returned on UNIX-like systems. We thus remap this error to an EWOULDBLOCK.
+                interp_ok(Err(IoError::HostError(io::ErrorKind::WouldBlock.into())))
+            }
+            result => interp_ok(result),
+        }
+    }
+
     /// Block the thread until we can receive bytes from the connected socket
     /// or an error occurred.
     ///
     /// This recursively calls itself should the operation still block for some reason.
+    ///
+    /// **Note**: This function is only safe to call when having previously ensured
+    /// that the socket is in [`SocketState::Connected`].
     fn block_for_recv(
         &mut self,
         socket: FileDescriptionRef<Socket>,
@@ -1354,24 +1541,9 @@ fn block_for_recv(
             } |this, kind: UnblockKind| {
                 assert_eq!(kind, UnblockKind::Ready);
 
-                let mut state = socket.state.borrow_mut();
-                let SocketState::Connected(stream) = &mut*state else {
-                    // We ensured that the socket is connected before blocking.
-                    unreachable!()
-                };
-
-                // This is a *non-blocking* read/peek.
-                let result = this.read_from_host(|buf| {
-                    if should_peek {
-                        stream.peek(buf)
-                    } else {
-                        stream.read(buf)
-                    }
-                }, length, buffer_ptr)?;
-                match result {
+                match this.try_non_block_recv(&socket, buffer_ptr, length, should_peek)? {
                     Err(IoError::HostError(e)) if e.kind() == io::ErrorKind::WouldBlock => {
                         // We need to block the thread again as it would still block.
-                        drop(state);
                         this.block_for_recv(socket, buffer_ptr, length, should_peek, finish);
                         interp_ok(())
                     },
@@ -1380,6 +1552,178 @@ fn block_for_recv(
             }),
         );
     }
+
+    /// Attempt to receive bytes from the connected socket in a non-blocking manner.
+    ///
+    /// **Note**: This function is only safe to call when having previously ensured
+    /// that the socket is in [`SocketState::Connected`].
+    fn try_non_block_recv(
+        &mut self,
+        socket: &FileDescriptionRef<Socket>,
+        buffer_ptr: Pointer,
+        length: usize,
+        should_peek: bool,
+    ) -> InterpResult<'tcx, Result<usize, IoError>> {
+        let this = self.eval_context_mut();
+
+        let SocketState::Connected(stream) = &mut *socket.state.borrow_mut() else {
+            panic!("try_non_block_recv must only be called when the socket is connected")
+        };
+
+        // This is a *non-blocking* read/peek.
+        let result = this.read_from_host(
+            |buf| {
+                if should_peek { stream.peek(buf) } else { stream.read(buf) }
+            },
+            length,
+            buffer_ptr,
+        )?;
+        match result {
+            Err(IoError::HostError(e)) if e.kind() == io::ErrorKind::NotConnected => {
+                // On Windows hosts, `recv` can return WSAENOTCONN where EAGAIN or EWOULDBLOCK
+                // would be returned on UNIX-like systems. We thus remap this error to an EWOULDBLOCK.
+                interp_ok(Err(IoError::HostError(io::ErrorKind::WouldBlock.into())))
+            }
+            result => interp_ok(result),
+        }
+    }
+
+    // Execute the provided callback function when the socket is either in
+    // [`SocketState::Connected`] or an error occurred.
+    /// If the socket is currently neither in the [`SocketState::Connecting`] nor
+    /// the [`SocketState::Connecting`] state, an ENOTCONN error is returned.
+    /// When the callback function is called with `Ok(_)`, then we're guaranteed
+    /// that the socket is in the [`SocketState::Connected`] state.
+    ///
+    /// This function can optionally also block until either an error occurred or
+    /// the socket reached the [`SocketState::Connected`] state.
+    fn ensure_connected(
+        &mut self,
+        socket: FileDescriptionRef<Socket>,
+        should_wait: bool,
+        foreign_name: &'static str,
+        action: DynMachineCallback<'tcx, Result<(), ()>>,
+    ) -> InterpResult<'tcx> {
+        let this = self.eval_context_mut();
+
+        let state = socket.state.borrow();
+        match &*state {
+            SocketState::Connecting(_) => { /* fall-through to below */ }
+            SocketState::Connected(_) => {
+                drop(state);
+                return action.call(this, Ok(()));
+            }
+            _ => {
+                drop(state);
+                return action.call(this, Err(()));
+            }
+        };
+
+        drop(state);
+
+        // We're currently connecting. Since the underlying mio socket is non-blocking,
+        // the only way to determine whether we are done connecting is by polling.
+        // If we should wait until the connection is established, the timeout is `None`.
+        // Otherwise, we use a zero duration timeout, i.e. we return immediately
+        // (but we still go through the scheduler once -- which is fine).
+        let timeout = if should_wait {
+            None
+        } else {
+            Some((TimeoutClock::Monotonic, TimeoutAnchor::Absolute, Duration::ZERO))
+        };
+
+        this.block_thread_for_io(
+            socket.clone(),
+            Interest::WRITABLE,
+            timeout,
+            callback!(
+                @capture<'tcx> {
+                    socket: FileDescriptionRef<Socket>,
+                    should_wait: bool,
+                    foreign_name: &'static str,
+                    action: DynMachineCallback<'tcx, Result<(), ()>>,
+                } |this, kind: UnblockKind| {
+                    if UnblockKind::TimedOut == kind {
+                        // We can only time out when `should_wait` is false.
+                        // This then means that the socket is not yet connected.
+                        assert!(!should_wait);
+                        this.machine.blocking_io.deregister(socket.id(), InterestReceiver::UnblockThread(this.active_thread()));
+                        return action.call(this, Err(()))
+                    }
+
+                    // The thread woke up because it's ready, indicating a writeable or error event.
+
+                    let mut state = socket.state.borrow_mut();
+                    let stream = match &*state {
+                        SocketState::Connecting(stream) => stream,
+                        SocketState::Connected(_) => {
+                            drop(state);
+                            // This can happen because we blocked the thread:
+                            // maybe another thread "upgraded" the connection in the meantime.
+                            return action.call(this, Ok(()))
+                        },
+                        _ => {
+                            drop(state);
+                            // We ensured that we only block when we're currently connecting.
+                            // Since this thread just got rescheduled, it could be that another
+                            // thread realized that the connection failed and we're thus in
+                            // an "invalid state".
+                            return action.call(this, Err(()))
+                        }
+                    };
+
+                    // Manually check whether there were any errors since calling `connect`.
+                    if let Ok(Some(_)) = stream.take_error() {
+                        // There was an error during connecting and thus we
+                        // return ENOTCONN. It's the program's responsibility
+                        // to read SO_ERROR itself.
+                        //
+                        // Go back to initial state since the only way of getting into the
+                        // `Connecting` state is from the `Initial` state and at this point
+                        // we know that the connection won't be established anymore.
+                        //
+                        // FIXME: We're currently just dropping the error information. Eventually
+                        // we'll have to store it so that it can be recovered by the user.
+                        *state = SocketState::Initial;
+                        drop(state);
+                        return action.call(this, Err(()))
+                    }
+
+                    // There was no error during connecting. We still need to ensure that
+                    // the wakeup wasn't spurious. We do this by attempting to read the
+                    // peer address of the socket (following the advice given by mio):
+                    // <https://docs.rs/mio/latest/mio/net/struct.TcpStream.html#notes>
+
+                    match stream.peer_addr() {
+                        Ok(_) => { /* fall-through to below */},
+                        Err(e) if matches!(e.kind(), io::ErrorKind::NotConnected | io::ErrorKind::InProgress) => {
+                            // We received a spurious wakeup from the OS. This should be considered an OS bug:
+                            // <https://github.com/tokio-rs/mio/issues/1942#issuecomment-4169378308>
+                            panic!("{foreign_name}: received writable event from OS but socket is not yet connected")
+                        },
+                        Err(_) => {
+                            // For all other errors the socket is connected. Since we're not interested in the
+                            // peer address and only want to know whether the socket is connected, we can ignore
+                            // the error and continue.
+                        }
+                    }
+
+                    // The connection is established.
+
+                    // Temporarily use dummy state to take ownership of the stream.
+                    let SocketState::Connecting(stream) = std::mem::replace(&mut*state, SocketState::Initial) else {
+                        // At the start of the function we ensured that we're currently connecting.
+                        unreachable!()
+                    };
+                    *state = SocketState::Connected(stream);
+                    drop(state);
+                    action.call(this, Ok(()))
+                }
+            ),
+        );
+
+        interp_ok(())
+    }
 }
 
 impl VisitProvenance for FileDescriptionRef<Socket> {

src/tools/miri/tests/genmc/fail/atomics/atomic_ptr_double_free.rs

@@ -1,5 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // Test that we can detect a double-free bug across two threads, which only shows up if the second thread reads an atomic pointer at a very specific moment.
 // GenMC can detect this error consistently, without having to run the buggy code with multiple RNG seeds or in a loop.
 

src/tools/miri/tests/genmc/fail/atomics/atomic_ptr_invalid_provenance.rs

@@ -1,5 +1,4 @@
 //@revisions: send make
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
 
 // Test that we can distinguish two pointers with the same address, but different provenance, after they are sent to GenMC and back.
 // We have two variants, one where we send such a pointer to GenMC, and one where we make it on the GenMC side.

src/tools/miri/tests/genmc/fail/data_race/atomic_ptr_alloc_race.dealloc.stderr

@@ -1,5 +1,5 @@
 Running GenMC Verification...
-error: Undefined Behavior: Attempt to access freed memory
+error: Undefined Behavior: Attempt to access non-allocated memory
   --> tests/genmc/fail/data_race/atomic_ptr_alloc_race.rs:LL:CC
    |
 LL |                         dealloc(b as *mut u8, Layout::new::<u64>());

src/tools/miri/tests/genmc/fail/data_race/atomic_ptr_alloc_race.rs

@@ -1,5 +1,5 @@
 //@revisions: write dealloc
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows -Zmiri-ignore-leaks
+//@compile-flags: -Zmiri-ignore-leaks
 
 // Test that we can detect data races between an allocation and an unsynchronized action in another thread.
 // We have two variants, an alloc-dealloc race and an alloc-write race.

src/tools/miri/tests/genmc/fail/data_race/atomic_ptr_dealloc_write_race.rs

@@ -1,5 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // Test that use-after-free bugs involving atomic pointers are detected in GenMC mode.
 
 #![no_main]

src/tools/miri/tests/genmc/fail/data_race/atomic_ptr_write_dealloc_race.rs

@@ -1,5 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // Test that use-after-free bugs involving atomic pointers are detected in GenMC mode.
 // Compared to `atomic_ptr_dealloc_write_race.rs`, this variant checks that the data race is still detected, even if the write happens before the free.
 //

src/tools/miri/tests/genmc/fail/data_race/mpu2_rels_rlx.rs

@@ -1,5 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // Translated from GenMC's test `wrong/racy/MPU2+rels+rlx`.
 // Test if Miri with GenMC can detect the data race on `X`.
 // The data race only occurs if thread 1 finishes, then threads 3 and 4 run, then thread 2.

src/tools/miri/tests/genmc/fail/data_race/weak_orderings.rs

@@ -1,4 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
 //@revisions: rlx_rlx rlx_acq rel_rlx
 
 // Translated from GenMC's test `wrong/racy/MP+rel+rlx`, `MP+rlx+acq` and `MP+rlx+rlx`.

src/tools/miri/tests/genmc/fail/loom/buggy_inc.rs

@@ -1,5 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // SPDX-License-Identifier: MIT
 // SPDX-FileCopyrightText: Copyright (c) 2019 Carl Lerche
 

src/tools/miri/tests/genmc/fail/loom/store_buffering.non_genmc.stderr

@@ -1,12 +0,0 @@
-error: abnormal termination: the program aborted execution
-  --> tests/genmc/fail/loom/store_buffering.rs:LL:CC
-   |
-LL |             std::process::abort();
-   |             ^^^^^^^^^^^^^^^^^^^^^ abnormal termination occurred here
-   |
-   = note: this is on thread `main`
-
-note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
-
-error: aborting due to 1 previous error
-

src/tools/miri/tests/genmc/fail/loom/store_buffering.rs

@@ -1,15 +1,9 @@
-//@ revisions: non_genmc genmc
-//@[genmc] compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // SPDX-License-Identifier: MIT
 // SPDX-FileCopyrightText: Copyright (c) 2019 Carl Lerche
 
 // This is the test `store_buffering` from `loom/test/litmus.rs`, adapted for Miri-GenMC.
 // https://github.com/tokio-rs/loom/blob/dbf32b04bae821c64be44405a0bb72ca08741558/tests/litmus.rs
 
-// This test shows the comparison between running Miri with or without GenMC.
-// Without GenMC, Miri requires multiple iterations of the loop to detect the error.
-
 #![no_main]
 
 #[path = "../../../utils/genmc.rs"]
@@ -23,30 +17,27 @@
 #[unsafe(no_mangle)]
 fn miri_start(_argc: isize, _argv: *const *const u8) -> isize {
     // For normal Miri, we need multiple repetitions, but GenMC should find the bug with only 1.
-    const REPS: usize = if cfg!(non_genmc) { 128 } else { 1 };
-    for _ in 0..REPS {
-        // New atomics every iterations, so they don't influence each other.
-        let x = AtomicUsize::new(0);
-        let y = AtomicUsize::new(0);
 
-        let mut a: usize = 1234;
-        let mut b: usize = 1234;
-        unsafe {
-            let ids = [
-                spawn_pthread_closure(|| {
-                    x.store(1, Relaxed);
-                    a = y.load(Relaxed)
-                }),
-                spawn_pthread_closure(|| {
-                    y.store(1, Relaxed);
-                    b = x.load(Relaxed)
-                }),
-            ];
-            join_pthreads(ids);
-        }
-        if (a, b) == (0, 0) {
-            std::process::abort(); //~ ERROR: abnormal termination
-        }
+    let x = AtomicUsize::new(0);
+    let y = AtomicUsize::new(0);
+
+    let mut a: usize = 1234;
+    let mut b: usize = 1234;
+    unsafe {
+        let ids = [
+            spawn_pthread_closure(|| {
+                x.store(1, Relaxed);
+                a = y.load(Relaxed)
+            }),
+            spawn_pthread_closure(|| {
+                y.store(1, Relaxed);
+                b = x.load(Relaxed)
+            }),
+        ];
+        join_pthreads(ids);
+    }
+    if (a, b) == (0, 0) {
+        std::process::abort(); //~ ERROR: abnormal termination
     }
 
     0

src/tools/miri/tests/genmc/fail/loom/store_buffering.stderr

@@ -2,8 +2,8 @@ Running GenMC Verification...
 error: abnormal termination: the program aborted execution
   --> tests/genmc/fail/loom/store_buffering.rs:LL:CC
    |
-LL |             std::process::abort();
-   |             ^^^^^^^^^^^^^^^^^^^^^ abnormal termination occurred here
+LL |         std::process::abort();
+   |         ^^^^^^^^^^^^^^^^^^^^^ abnormal termination occurred here
    |
    = note: this is on thread `main`
 

src/tools/miri/tests/genmc/fail/shims/exit.rs

@@ -1,5 +1,3 @@
-//@ compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 fn main() {
     std::thread::spawn(|| {
         unsafe { std::hint::unreachable_unchecked() }; //~ERROR: entering unreachable code

src/tools/miri/tests/genmc/fail/shims/mutex_diff_thread_unlock.rs

@@ -1,4 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
 //@error-in-other-file: Undefined Behavior
 
 // Test that GenMC throws an error if a `std::sync::Mutex` is unlocked from a different thread than the one that locked it.

src/tools/miri/tests/genmc/fail/shims/mutex_double_unlock.rs

@@ -1,4 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
 //@error-in-other-file: Undefined Behavior
 
 // Test that GenMC can detect a double unlock of a mutex.

src/tools/miri/tests/genmc/fail/simple/2w2w_weak.rs

@@ -1,4 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
 //@revisions: sc3_rel1 release4 relaxed4
 
 // The pass tests "2w2w_3sc_1rel.rs", "2w2w_4rel" and "2w2w_4sc" and the fail test "2w2w_weak.rs" are related.

src/tools/miri/tests/genmc/fail/simple/alloc_large.rs

@@ -1,5 +1,4 @@
 //@revisions: single multiple
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
 //@error-in-other-file: resource exhaustion
 
 // Ensure that we emit a proper error if GenMC fails to fulfill an allocation.

src/tools/miri/tests/genmc/pass/atomics/atomic_ptr_ops.rs

@@ -1,5 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // Test several operations on atomic pointers.
 
 #![no_main]

src/tools/miri/tests/genmc/pass/atomics/atomic_ptr_roundtrip.rs

@@ -1,5 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // Test that we can send pointers with any alignment to GenMC and back, even across threads.
 // After a round-trip, the pointers should still work properly (no missing provenance).
 

src/tools/miri/tests/genmc/pass/atomics/cas_failure_ord_racy_key_init.rs

@@ -1,4 +1,4 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows -Zmiri-ignore-leaks
+//@compile-flags: -Zmiri-ignore-leaks
 
 // Adapted from: `impl LazyKey`, `fn lazy_init`: rust/library/std/src/sys/thread_local/key/racy.rs
 // Two threads race to initialize a key, which is just an index into an array in this test.

src/tools/miri/tests/genmc/pass/atomics/cas_simple.rs

@@ -1,5 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // Test the basic functionality of compare_exchange.
 
 #![no_main]

src/tools/miri/tests/genmc/pass/atomics/mixed_atomic_non_atomic.rs

@@ -1,5 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // Test that we can read the value of a non-atomic store atomically and an of an atomic value non-atomically.
 
 #![no_main]

src/tools/miri/tests/genmc/pass/atomics/read_initial_value.rs

@@ -1,5 +1,3 @@
-//@compile-flags: -Zmiri-genmc -Zmiri-disable-stacked-borrows
-
 // Test that we can read the initial value of global, heap and stack allocations in GenMC mode.
 
 #![no_main]