Merge from rustc

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ concurrency:
   # For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
   # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
   # are all triggered on the same branch, but which should be able to run concurrently.
-  group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
+  group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/automation/bors/try') && github.sha) || github.ref }}
   cancel-in-progress: true
 env:
   TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
@@ -46,7 +46,7 @@ jobs:
   # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
   calculate_matrix:
     name: Calculate job matrix
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-24.04-arm
     outputs:
       jobs: ${{ steps.jobs.outputs.jobs }}
       run_type: ${{ steps.jobs.outputs.run_type }}
@@ -80,7 +80,7 @@ jobs:
     # access the environment.
     #
     # We only enable the environment for the rust-lang/rust repository, so that CI works on forks.
-    environment: ${{ ((github.repository == 'rust-lang/rust' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/auto')) && 'bors') || '' }}
+    environment: ${{ ((github.repository == 'rust-lang/rust' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/automation/bors/try' || github.ref == 'refs/heads/auto')) && 'bors') || '' }}
     env:
       CI_JOB_NAME: ${{ matrix.name }}
       CI_JOB_DOC_URL: ${{ matrix.doc_url }}

Cargo.lock

@@ -2171,6 +2171,7 @@ dependencies = [
 name = "lint-docs"
 version = "0.1.0"
 dependencies = [
+ "rustc-literal-escaper",
  "serde_json",
  "tempfile",
  "walkdir",
@@ -3445,6 +3446,7 @@ dependencies = [
  "rustc_macros",
  "rustc_parse",
  "rustc_parse_format",
+ "rustc_proc_macro",
  "rustc_session",
  "rustc_span",
  "rustc_target",
@@ -3734,6 +3736,7 @@ dependencies = [
  "rustc_lint_defs",
  "rustc_macros",
  "rustc_parse",
+ "rustc_proc_macro",
  "rustc_serialize",
  "rustc_session",
  "rustc_span",
@@ -4082,6 +4085,7 @@ dependencies = [
  "rustc_index",
  "rustc_macros",
  "rustc_middle",
+ "rustc_proc_macro",
  "rustc_serialize",
  "rustc_session",
  "rustc_span",
@@ -4338,6 +4342,13 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "rustc_proc_macro"
+version = "0.0.0"
+dependencies = [
+ "rustc-literal-escaper",
+]
+
 [[package]]
 name = "rustc_query_impl"
 version = "0.0.0"

compiler/rustc_ast/src/ast.rs

@@ -32,7 +32,7 @@
 use rustc_macros::{Decodable, Encodable, HashStable_Generic};
 pub use rustc_span::AttrId;
 use rustc_span::source_map::{Spanned, respan};
-use rustc_span::{ErrorGuaranteed, Ident, Span, Symbol, kw, sym};
+use rustc_span::{DUMMY_SP, ErrorGuaranteed, Ident, Span, Symbol, kw, sym};
 use thin_vec::{ThinVec, thin_vec};
 
 pub use crate::format::*;
@@ -1526,6 +1526,19 @@ pub fn is_approximately_pattern(&self) -> bool {
                 | ExprKind::Struct(_)
         )
     }
+
+    /// Creates a dummy `P<Expr>`.
+    ///
+    /// Should only be used when it will be replaced afterwards or as a return value when an error was encountered.
+    pub fn dummy() -> P<Expr> {
+        P(Expr {
+            id: DUMMY_NODE_ID,
+            kind: ExprKind::Dummy,
+            span: DUMMY_SP,
+            attrs: ThinVec::new(),
+            tokens: None,
+        })
+    }
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]

compiler/rustc_ast/src/ast_traits.rs

@@ -304,6 +304,7 @@ fn visit_attrs(&mut self, f: impl FnOnce(&mut AttrVec)) {
 }
 
 /// A newtype around an AST node that implements the traits above if the node implements them.
+#[repr(transparent)]
 pub struct AstNodeWrapper<Wrapped, Tag> {
     pub wrapped: Wrapped,
     pub tag: PhantomData<Tag>,
@@ -313,6 +314,11 @@ impl<Wrapped, Tag> AstNodeWrapper<Wrapped, Tag> {
     pub fn new(wrapped: Wrapped, _tag: Tag) -> AstNodeWrapper<Wrapped, Tag> {
         AstNodeWrapper { wrapped, tag: Default::default() }
     }
+
+    pub fn from_mut(wrapped: &mut Wrapped, _tag: Tag) -> &mut AstNodeWrapper<Wrapped, Tag> {
+        // SAFETY: `AstNodeWrapper` is `repr(transparent)` w.r.t `Wrapped`
+        unsafe { &mut *<*mut Wrapped>::cast(wrapped) }
+    }
 }
 
 impl<Wrapped: HasNodeId, Tag> HasNodeId for AstNodeWrapper<Wrapped, Tag> {

compiler/rustc_ast/src/mut_visit.rs

@@ -77,7 +77,7 @@ fn visit_use_tree(&mut self, use_tree: &mut UseTree) {
         walk_use_tree(self, use_tree);
     }
 
-    fn visit_foreign_item(&mut self, ni: &mut P<ForeignItem>) {
+    fn visit_foreign_item(&mut self, ni: &mut ForeignItem) {
         walk_item(self, ni);
     }
 
@@ -85,7 +85,7 @@ fn visit_foreign_item(&mut self, ni: &mut P<ForeignItem>) {
         walk_flat_map_foreign_item(self, ni)
     }
 
-    fn visit_item(&mut self, i: &mut P<Item>) {
+    fn visit_item(&mut self, i: &mut Item) {
         walk_item(self, i);
     }
 
@@ -105,7 +105,7 @@ fn visit_field_def(&mut self, fd: &mut FieldDef) {
         walk_flat_map_field_def(self, fd)
     }
 
-    fn visit_assoc_item(&mut self, i: &mut P<AssocItem>, ctxt: AssocCtxt) {
+    fn visit_assoc_item(&mut self, i: &mut AssocItem, ctxt: AssocCtxt) {
         walk_assoc_item(self, i, ctxt)
     }
 
@@ -117,11 +117,11 @@ fn flat_map_assoc_item(
         walk_flat_map_assoc_item(self, i, ctxt)
     }
 
-    fn visit_contract(&mut self, c: &mut P<FnContract>) {
+    fn visit_contract(&mut self, c: &mut FnContract) {
         walk_contract(self, c);
     }
 
-    fn visit_fn_decl(&mut self, d: &mut P<FnDecl>) {
+    fn visit_fn_decl(&mut self, d: &mut FnDecl) {
         walk_fn_decl(self, d);
     }
 
@@ -138,7 +138,7 @@ fn visit_closure_binder(&mut self, b: &mut ClosureBinder) {
         walk_closure_binder(self, b);
     }
 
-    fn visit_block(&mut self, b: &mut P<Block>) {
+    fn visit_block(&mut self, b: &mut Block) {
         walk_block(self, b);
     }
 
@@ -184,7 +184,7 @@ fn visit_ty(&mut self, t: &mut P<Ty>) {
         walk_ty(self, t);
     }
 
-    fn visit_ty_pat(&mut self, t: &mut P<TyPat>) {
+    fn visit_ty_pat(&mut self, t: &mut TyPat) {
         walk_ty_pat(self, t);
     }
 
@@ -240,7 +240,7 @@ fn visit_parenthesized_parameter_data(&mut self, p: &mut ParenthesizedArgs) {
         walk_parenthesized_parameter_data(self, p);
     }
 
-    fn visit_local(&mut self, l: &mut P<Local>) {
+    fn visit_local(&mut self, l: &mut Local) {
         walk_local(self, l);
     }
 
@@ -368,14 +368,6 @@ fn visit_fn_ret_ty(&mut self, fn_ret_ty: &mut FnRetTy) {
 
 super::common_visitor_and_walkers!((mut) MutVisitor);
 
-/// Use a map-style function (`FnOnce(T) -> T`) to overwrite a `&mut T`. Useful
-/// when using a `flat_map_*` or `filter_map_*` method within a `visit_`
-/// method.
-pub fn visit_clobber<T: DummyAstNode>(t: &mut T, f: impl FnOnce(T) -> T) {
-    let old_t = std::mem::replace(t, T::dummy());
-    *t = f(old_t);
-}
-
 #[inline]
 fn visit_vec<T, F>(elems: &mut Vec<T>, mut visit_elem: F)
 where
@@ -507,8 +499,8 @@ fn walk_assoc_item_constraint<T: MutVisitor>(
     vis.visit_span(span);
 }
 
-pub fn walk_ty<T: MutVisitor>(vis: &mut T, ty: &mut P<Ty>) {
-    let Ty { id, kind, span, tokens: _ } = ty.deref_mut();
+pub fn walk_ty<T: MutVisitor>(vis: &mut T, ty: &mut Ty) {
+    let Ty { id, kind, span, tokens: _ } = ty;
     vis.visit_id(id);
     match kind {
         TyKind::Err(_guar) => {}
@@ -559,8 +551,8 @@ pub fn walk_ty<T: MutVisitor>(vis: &mut T, ty: &mut P<Ty>) {
     vis.visit_span(span);
 }
 
-pub fn walk_ty_pat<T: MutVisitor>(vis: &mut T, ty: &mut P<TyPat>) {
-    let TyPat { id, kind, span, tokens: _ } = ty.deref_mut();
+pub fn walk_ty_pat<T: MutVisitor>(vis: &mut T, ty: &mut TyPat) {
+    let TyPat { id, kind, span, tokens: _ } = ty;
     vis.visit_id(id);
     match kind {
         TyPatKind::Range(start, end, _include_end) => {
@@ -651,8 +643,8 @@ fn walk_parenthesized_parameter_data<T: MutVisitor>(vis: &mut T, args: &mut Pare
     vis.visit_span(inputs_span);
 }
 
-fn walk_local<T: MutVisitor>(vis: &mut T, local: &mut P<Local>) {
-    let Local { id, super_, pat, ty, kind, span, colon_sp, attrs, tokens: _ } = local.deref_mut();
+fn walk_local<T: MutVisitor>(vis: &mut T, local: &mut Local) {
+    let Local { id, super_, pat, ty, kind, span, colon_sp, attrs, tokens: _ } = local;
     visit_opt(super_, |sp| vis.visit_span(sp));
     vis.visit_id(id);
     visit_attrs(vis, attrs);
@@ -789,8 +781,8 @@ fn walk_fn<T: MutVisitor>(vis: &mut T, kind: FnKind<'_>) {
     }
 }
 
-fn walk_contract<T: MutVisitor>(vis: &mut T, contract: &mut P<FnContract>) {
-    let FnContract { requires, ensures } = contract.deref_mut();
+fn walk_contract<T: MutVisitor>(vis: &mut T, contract: &mut FnContract) {
+    let FnContract { requires, ensures } = contract;
     if let Some(pred) = requires {
         vis.visit_expr(pred);
     }
@@ -799,8 +791,8 @@ fn walk_contract<T: MutVisitor>(vis: &mut T, contract: &mut P<FnContract>) {
     }
 }
 
-fn walk_fn_decl<T: MutVisitor>(vis: &mut T, decl: &mut P<FnDecl>) {
-    let FnDecl { inputs, output } = decl.deref_mut();
+fn walk_fn_decl<T: MutVisitor>(vis: &mut T, decl: &mut FnDecl) {
+    let FnDecl { inputs, output } = decl;
     inputs.flat_map_in_place(|param| vis.flat_map_param(param));
     vis.visit_fn_ret_ty(output);
 }
@@ -999,8 +991,8 @@ pub fn walk_flat_map_expr_field<T: MutVisitor>(
     smallvec![f]
 }
 
-pub fn walk_block<T: MutVisitor>(vis: &mut T, block: &mut P<Block>) {
-    let Block { id, stmts, rules: _, span, tokens: _ } = block.deref_mut();
+pub fn walk_block<T: MutVisitor>(vis: &mut T, block: &mut Block) {
+    let Block { id, stmts, rules: _, span, tokens: _ } = block;
     vis.visit_id(id);
     stmts.flat_map_in_place(|stmt| vis.flat_map_stmt(stmt));
     vis.visit_span(span);
@@ -1049,8 +1041,8 @@ pub fn walk_flat_map_assoc_item(
     smallvec![item]
 }
 
-pub fn walk_pat<T: MutVisitor>(vis: &mut T, pat: &mut P<Pat>) {
-    let Pat { id, kind, span, tokens: _ } = pat.deref_mut();
+pub fn walk_pat<T: MutVisitor>(vis: &mut T, pat: &mut Pat) {
+    let Pat { id, kind, span, tokens: _ } = pat;
     vis.visit_id(id);
     match kind {
         PatKind::Err(_guar) => {}
@@ -1417,101 +1409,6 @@ fn walk_capture_by<T: MutVisitor>(vis: &mut T, capture_by: &mut CaptureBy) {
     }
 }
 
-/// Some value for the AST node that is valid but possibly meaningless. Similar
-/// to `Default` but not intended for wide use. The value will never be used
-/// meaningfully, it exists just to support unwinding in `visit_clobber` in the
-/// case where its closure panics.
-pub trait DummyAstNode {
-    fn dummy() -> Self;
-}
-
-impl<T> DummyAstNode for Option<T> {
-    fn dummy() -> Self {
-        Default::default()
-    }
-}
-
-impl<T: DummyAstNode + 'static> DummyAstNode for P<T> {
-    fn dummy() -> Self {
-        P(DummyAstNode::dummy())
-    }
-}
-
-impl DummyAstNode for Item {
-    fn dummy() -> Self {
-        Item {
-            attrs: Default::default(),
-            id: DUMMY_NODE_ID,
-            span: Default::default(),
-            vis: Visibility {
-                kind: VisibilityKind::Public,
-                span: Default::default(),
-                tokens: Default::default(),
-            },
-            kind: ItemKind::ExternCrate(None, Ident::dummy()),
-            tokens: Default::default(),
-        }
-    }
-}
-
-impl DummyAstNode for Expr {
-    fn dummy() -> Self {
-        Expr {
-            id: DUMMY_NODE_ID,
-            kind: ExprKind::Dummy,
-            span: Default::default(),
-            attrs: Default::default(),
-            tokens: Default::default(),
-        }
-    }
-}
-
-impl DummyAstNode for Ty {
-    fn dummy() -> Self {
-        Ty {
-            id: DUMMY_NODE_ID,
-            kind: TyKind::Dummy,
-            span: Default::default(),
-            tokens: Default::default(),
-        }
-    }
-}
-
-impl DummyAstNode for Pat {
-    fn dummy() -> Self {
-        Pat {
-            id: DUMMY_NODE_ID,
-            kind: PatKind::Wild,
-            span: Default::default(),
-            tokens: Default::default(),
-        }
-    }
-}
-
-impl DummyAstNode for Stmt {
-    fn dummy() -> Self {
-        Stmt { id: DUMMY_NODE_ID, kind: StmtKind::Empty, span: Default::default() }
-    }
-}
-
-impl DummyAstNode for Crate {
-    fn dummy() -> Self {
-        Crate {
-            attrs: Default::default(),
-            items: Default::default(),
-            spans: Default::default(),
-            id: DUMMY_NODE_ID,
-            is_placeholder: Default::default(),
-        }
-    }
-}
-
-impl<N: DummyAstNode, T: DummyAstNode> DummyAstNode for crate::ast_traits::AstNodeWrapper<N, T> {
-    fn dummy() -> Self {
-        crate::ast_traits::AstNodeWrapper::new(N::dummy(), T::dummy())
-    }
-}
-
 #[derive(Debug)]
 pub enum FnKind<'a> {
     /// E.g., `fn foo()`, `fn foo(&self)`, or `extern "Abi" fn foo()`.

compiler/rustc_ast/src/visit.rs

@@ -404,10 +404,10 @@ pub fn walk_lifetime<$($lt,)? V: $Visitor$(<$lt>)?>(visitor: &mut V, Lifetime {
 
         fn walk_item_ctxt<$($lt,)? V: $Visitor$(<$lt>)?, K: WalkItemKind>(
             visitor: &mut V,
-            item: &$($mut P<Item<K>>)? $($lt Item<K>)?,
+            item: &$($mut)? $($lt)? Item<K>,
             ctxt: K::Ctxt,
         ) $(-> <V as Visitor<$lt>>::Result)? {
-            let Item { attrs, id, kind, vis, span, tokens: _ } = &$($mut *)? *item;
+            let Item { attrs, id, kind, vis, span, tokens: _ } = item;
             try_visit!(visit_id(visitor, id));
             walk_list!(visitor, visit_attribute, attrs);
             try_visit!(visitor.visit_vis(vis));
@@ -417,14 +417,14 @@ fn walk_item_ctxt<$($lt,)? V: $Visitor$(<$lt>)?, K: WalkItemKind>(
 
         pub fn walk_item<$($lt,)? V: $Visitor$(<$lt>)?, K: WalkItemKind<Ctxt = ()>>(
             visitor: &mut V,
-            item: &$($mut P<Item<K>>)? $($lt Item<K>)?,
+            item: &$($mut)? $($lt)? Item<K>,
         ) $(-> <V as Visitor<$lt>>::Result)? {
             walk_item_ctxt(visitor, item, ())
         }
 
         pub fn walk_assoc_item<$($lt,)? V: $Visitor$(<$lt>)?>(
             visitor: &mut V,
-            item: &$($mut P<AssocItem>)? $($lt AssocItem)?,
+            item: &$($mut)? $($lt)? AssocItem,
             ctxt: AssocCtxt,
         ) $(-> <V as Visitor<$lt>>::Result)? {
             walk_item_ctxt(visitor, item, ctxt)

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -474,17 +474,18 @@ fn relate_type_and_user_type(
             let projected_ty = curr_projected_ty.projection_ty_core(
                 tcx,
                 proj,
-                |this, field, ()| {
-                    let ty = this.field_ty(tcx, field);
-                    self.structurally_resolve(ty, locations)
-                },
-                |_, _| unreachable!(),
+                |ty| self.structurally_resolve(ty, locations),
+                |ty, variant_index, field, ()| PlaceTy::field_ty(tcx, ty, variant_index, field),
+                |_| unreachable!(),
             );
             curr_projected_ty = projected_ty;
         }
         trace!(?curr_projected_ty);
 
-        let ty = curr_projected_ty.ty;
+        // Need to renormalize `a` as typecheck may have failed to normalize
+        // higher-ranked aliases if normalization was ambiguous due to inference.
+        let a = self.normalize(a, locations);
+        let ty = self.normalize(curr_projected_ty.ty, locations);
         self.relate_types(ty, v.xform(ty::Contravariant), a, locations, category)?;
 
         Ok(())
@@ -1852,7 +1853,7 @@ fn visit_projection_elem(
             | ProjectionElem::Downcast(..) => {}
             ProjectionElem::Field(field, fty) => {
                 let fty = self.normalize(fty, location);
-                let ty = base_ty.field_ty(tcx, field);
+                let ty = PlaceTy::field_ty(tcx, base_ty.ty, base_ty.variant_index, field);
                 let ty = self.normalize(ty, location);
                 debug!(?fty, ?ty);
 

compiler/rustc_builtin_macros/Cargo.toml

@@ -24,6 +24,9 @@ rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_parse_format = { path = "../rustc_parse_format" }
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+rustc_proc_macro = { path = "../rustc_proc_macro" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }

compiler/rustc_builtin_macros/src/lib.rs

@@ -20,8 +20,6 @@
 #![recursion_limit = "256"]
 // tidy-alphabetical-end
 
-extern crate proc_macro;
-
 use std::sync::Arc;
 
 use rustc_expand::base::{MacroExpanderFn, ResolverExpand, SyntaxExtensionKind};
@@ -140,7 +138,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
         CoercePointee: coerce_pointee::expand_deriving_coerce_pointee,
     }
 
-    let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote);
+    let client = rustc_proc_macro::bridge::client::Client::expand1(rustc_proc_macro::quote);
     register(sym::quote, SyntaxExtensionKind::Bang(Arc::new(BangProcMacro { client })));
     let requires = SyntaxExtensionKind::Attr(Arc::new(contracts::ExpandRequires));
     register(sym::contracts_requires, requires);

compiler/rustc_builtin_macros/src/test_harness.rs

@@ -128,9 +128,7 @@ fn visit_crate(&mut self, c: &mut ast::Crate) {
         c.items.push(mk_main(&mut self.cx));
     }
 
-    fn visit_item(&mut self, item: &mut P<ast::Item>) {
-        let item = &mut **item;
-
+    fn visit_item(&mut self, item: &mut ast::Item) {
         if let Some(name) = get_test_name(&item) {
             debug!("this is a test item");
 
@@ -193,7 +191,7 @@ struct EntryPointCleaner<'a> {
 }
 
 impl<'a> MutVisitor for EntryPointCleaner<'a> {
-    fn visit_item(&mut self, item: &mut P<ast::Item>) {
+    fn visit_item(&mut self, item: &mut ast::Item) {
         self.depth += 1;
         ast::mut_visit::walk_item(self, item);
         self.depth -= 1;

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -870,11 +870,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
             // FIXME use a compiler fence once Cranelift supports it
             fx.bcx.ins().fence();
         }
-        _ if intrinsic.as_str().starts_with("atomic_load") => {
+        sym::atomic_load => {
             intrinsic_args!(fx, args => (ptr); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
             let ty = generic_args.type_at(0);
+            let _ord = generic_args.const_at(1).to_value(); // FIXME: forward this to cranelift once they support that
             match ty.kind() {
                 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
                     // FIXME implement 128bit atomics

compiler/rustc_codegen_gcc/src/abi.rs

@@ -15,7 +15,6 @@
 
 use crate::builder::Builder;
 use crate::context::CodegenCx;
-use crate::intrinsic::ArgAbiExt;
 use crate::type_of::LayoutGccExt;
 
 impl AbiBuilderMethods for Builder<'_, '_, '_> {
@@ -125,7 +124,7 @@ fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> FnAbiGcc<'gcc> {
             PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_gcc_type(cx),
             PassMode::Cast { ref cast, .. } => cast.gcc_type(cx),
             PassMode::Indirect { .. } => {
-                argument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx)));
+                argument_tys.push(cx.type_ptr_to(self.ret.layout.gcc_type(cx)));
                 cx.type_void()
             }
         };
@@ -176,13 +175,13 @@ fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> FnAbiGcc<'gcc> {
                 PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: true } => {
                     // This is a "byval" argument, so we don't apply the `restrict` attribute on it.
                     on_stack_param_indices.insert(argument_tys.len());
-                    arg.memory_ty(cx)
+                    arg.layout.gcc_type(cx)
                 }
                 PassMode::Direct(attrs) => {
                     apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs, argument_tys.len())
                 }
                 PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
-                    apply_attrs(cx.type_ptr_to(arg.memory_ty(cx)), &attrs, argument_tys.len())
+                    apply_attrs(cx.type_ptr_to(arg.layout.gcc_type(cx)), &attrs, argument_tys.len())
                 }
                 PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
                     assert!(!on_stack);

compiler/rustc_codegen_gcc/src/base.rs

@@ -219,17 +219,22 @@ fn module_codegen(
 
             let mono_items = cgu.items_in_deterministic_order(tcx);
             for &(mono_item, data) in &mono_items {
-                mono_item.predefine::<Builder<'_, '_, '_>>(&cx, data.linkage, data.visibility);
+                mono_item.predefine::<Builder<'_, '_, '_>>(
+                    &mut cx,
+                    cgu_name.as_str(),
+                    data.linkage,
+                    data.visibility,
+                );
             }
 
             // ... and now that we have everything pre-defined, fill out those definitions.
             for &(mono_item, item_data) in &mono_items {
-                mono_item.define::<Builder<'_, '_, '_>>(&mut cx, item_data);
+                mono_item.define::<Builder<'_, '_, '_>>(&mut cx, cgu_name.as_str(), item_data);
             }
 
             // If this codegen unit contains the main function, also create the
             // wrapper here
-            maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx);
+            maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit);
 
             // Finalize debuginfo
             if cx.sess().opts.debuginfo != DebugInfo::None {

compiler/rustc_codegen_gcc/src/builder.rs

@@ -12,7 +12,7 @@
 use rustc_apfloat::{Float, Round, Status, ieee};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::{
-    AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind,
+    AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind,
 };
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
@@ -26,7 +26,7 @@
 use rustc_middle::ty::layout::{
     FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOfHelpers,
 };
-use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
+use rustc_middle::ty::{self, AtomicOrdering, Instance, Ty, TyCtxt};
 use rustc_span::Span;
 use rustc_span::def_id::DefId;
 use rustc_target::callconv::FnAbi;
@@ -75,7 +75,7 @@ fn atomic_extremum(
 
         let load_ordering = match order {
             // TODO(antoyo): does this make sense?
-            AtomicOrdering::AcquireRelease | AtomicOrdering::Release => AtomicOrdering::Acquire,
+            AtomicOrdering::AcqRel | AtomicOrdering::Release => AtomicOrdering::Acquire,
             _ => order,
         };
         let previous_value =
@@ -2474,8 +2474,8 @@ fn to_gcc(self) -> i32 {
             AtomicOrdering::Relaxed => __ATOMIC_RELAXED, // TODO(antoyo): check if that's the same.
             AtomicOrdering::Acquire => __ATOMIC_ACQUIRE,
             AtomicOrdering::Release => __ATOMIC_RELEASE,
-            AtomicOrdering::AcquireRelease => __ATOMIC_ACQ_REL,
-            AtomicOrdering::SequentiallyConsistent => __ATOMIC_SEQ_CST,
+            AtomicOrdering::AcqRel => __ATOMIC_ACQ_REL,
+            AtomicOrdering::SeqCst => __ATOMIC_SEQ_CST,
         };
         ordering as i32
     }

compiler/rustc_codegen_gcc/src/consts.rs

@@ -67,7 +67,7 @@ fn static_addr_of(&self, cv: RValue<'gcc>, align: Align, kind: Option<&str>) ->
     }
 
     #[cfg_attr(not(feature = "master"), allow(unused_mut))]
-    fn codegen_static(&self, def_id: DefId) {
+    fn codegen_static(&mut self, def_id: DefId) {
         let attrs = self.tcx.codegen_fn_attrs(def_id);
 
         let Ok((value, alloc)) = codegen_static_initializer(self, def_id) else {
@@ -160,19 +160,14 @@ fn codegen_static(&self, def_id: DefId) {
             self.add_used_global(global.to_rvalue());
         }
     }
-
-    /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*.
-    fn add_used_global(&self, _global: RValue<'gcc>) {
-        // TODO(antoyo)
-    }
-
-    fn add_compiler_used_global(&self, global: RValue<'gcc>) {
-        // NOTE: seems like GCC does not make the distinction between compiler.used and used.
-        self.add_used_global(global);
-    }
 }
 
 impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
+    /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*.
+    pub fn add_used_global(&mut self, _global: RValue<'gcc>) {
+        // TODO(antoyo)
+    }
+
     #[cfg_attr(not(feature = "master"), allow(unused_variables))]
     pub fn add_used_function(&self, function: Function<'gcc>) {
         #[cfg(feature = "master")]

compiler/rustc_codegen_gcc/src/context.rs

@@ -470,10 +470,6 @@ fn sess(&self) -> &Session {
         self.tcx.sess
     }
 
-    fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx> {
-        self.codegen_unit
-    }
-
     fn set_frame_pointer_type(&self, _llfn: RValue<'gcc>) {
         // TODO(antoyo)
     }

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -574,14 +574,9 @@ fn store_arg(
     ) {
         arg_abi.store(self, val, dst)
     }
-
-    fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {
-        arg_abi.memory_ty(self)
-    }
 }
 
 pub trait ArgAbiExt<'gcc, 'tcx> {
-    fn memory_ty(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc>;
     fn store(
         &self,
         bx: &mut Builder<'_, 'gcc, 'tcx>,
@@ -597,12 +592,6 @@ fn store_fn_arg(
 }
 
 impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
-    /// Gets the LLVM type for a place of the original Rust type of
-    /// this argument/return, i.e., the result of `type_of::type_of`.
-    fn memory_ty(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc> {
-        self.layout.gcc_type(cx)
-    }
-
     /// Stores a direct/indirect value described by this ArgAbi into a
     /// place for the original Rust type of this argument/return.
     /// Can be used for both storing formal arguments into Rust variables

compiler/rustc_codegen_gcc/src/lib.rs

@@ -391,7 +391,7 @@ fn print_statistics(&self) {
         unimplemented!()
     }
 
-    unsafe fn optimize(
+    fn optimize(
         _cgcx: &CodegenContext<Self>,
         _dcx: DiagCtxtHandle<'_>,
         module: &mut ModuleCodegen<Self::Module>,
@@ -409,14 +409,14 @@ fn optimize_fat(
         Ok(())
     }
 
-    unsafe fn optimize_thin(
+    fn optimize_thin(
         cgcx: &CodegenContext<Self>,
         thin: ThinModule<Self>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
         back::lto::optimize_thin_module(thin, cgcx)
     }
 
-    unsafe fn codegen(
+    fn codegen(
         cgcx: &CodegenContext<Self>,
         dcx: DiagCtxtHandle<'_>,
         module: ModuleCodegen<Self::Module>,

compiler/rustc_codegen_gcc/src/mono_item.rs

@@ -16,7 +16,7 @@
 impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     #[cfg_attr(not(feature = "master"), allow(unused_variables))]
     fn predefine_static(
-        &self,
+        &mut self,
         def_id: DefId,
         _linkage: Linkage,
         visibility: Visibility,
@@ -42,7 +42,7 @@ fn predefine_static(
 
     #[cfg_attr(not(feature = "master"), allow(unused_variables))]
     fn predefine_fn(
-        &self,
+        &mut self,
         instance: Instance<'tcx>,
         linkage: Linkage,
         visibility: Visibility,

compiler/rustc_codegen_llvm/src/abi.rs

@@ -172,7 +172,6 @@ fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type {
 }
 
 trait ArgAbiExt<'ll, 'tcx> {
-    fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
     fn store(
         &self,
         bx: &mut Builder<'_, 'll, 'tcx>,
@@ -188,12 +187,6 @@ fn store_fn_arg(
 }
 
 impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
-    /// Gets the LLVM type for a place of the original Rust type of
-    /// this argument/return, i.e., the result of `type_of::type_of`.
-    fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
-        self.layout.llvm_type(cx)
-    }
-
     /// Stores a direct/indirect value described by this ArgAbi into a
     /// place for the original Rust type of this argument/return.
     /// Can be used for both storing formal arguments into Rust variables
@@ -302,9 +295,6 @@ fn store_arg(
     ) {
         arg_abi.store(self, val, dst)
     }
-    fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
-        arg_abi.memory_ty(self)
-    }
 }
 
 pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -799,7 +799,7 @@ fn drop(&mut self) {
     }
 }
 
-pub(crate) unsafe fn optimize_thin_module(
+pub(crate) fn optimize_thin_module(
     thin_module: ThinModule<LlvmCodegenBackend>,
     cgcx: &CodegenContext<LlvmCodegenBackend>,
 ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -704,7 +704,7 @@ pub(crate) unsafe fn llvm_optimize(
 }
 
 // Unsafe due to LLVM calls.
-pub(crate) unsafe fn optimize(
+pub(crate) fn optimize(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
     dcx: DiagCtxtHandle<'_>,
     module: &mut ModuleCodegen<ModuleLlvm>,
@@ -815,7 +815,7 @@ pub(crate) fn link(
     Ok(modules.remove(0))
 }
 
-pub(crate) unsafe fn codegen(
+pub(crate) fn codegen(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
     dcx: DiagCtxtHandle<'_>,
     module: ModuleCodegen<ModuleLlvm>,

compiler/rustc_codegen_llvm/src/base.rs

@@ -86,17 +86,24 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: Symbol) -> ModuleCodegen<ModuleLlvm
             let mut cx = CodegenCx::new(tcx, cgu, &llvm_module);
             let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx);
             for &(mono_item, data) in &mono_items {
-                mono_item.predefine::<Builder<'_, '_, '_>>(&cx, data.linkage, data.visibility);
+                mono_item.predefine::<Builder<'_, '_, '_>>(
+                    &mut cx,
+                    cgu_name.as_str(),
+                    data.linkage,
+                    data.visibility,
+                );
             }
 
             // ... and now that we have everything pre-defined, fill out those definitions.
             for &(mono_item, item_data) in &mono_items {
-                mono_item.define::<Builder<'_, '_, '_>>(&mut cx, item_data);
+                mono_item.define::<Builder<'_, '_, '_>>(&mut cx, cgu_name.as_str(), item_data);
             }
 
             // If this codegen unit contains the main function, also create the
             // wrapper here
-            if let Some(entry) = maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx) {
+            if let Some(entry) =
+                maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit)
+            {
                 let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty());
                 attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs);
             }
@@ -108,14 +115,11 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: Symbol) -> ModuleCodegen<ModuleLlvm
             }
 
             // Create the llvm.used and llvm.compiler.used variables.
-            if !cx.used_statics.borrow().is_empty() {
-                cx.create_used_variable_impl(c"llvm.used", &*cx.used_statics.borrow());
+            if !cx.used_statics.is_empty() {
+                cx.create_used_variable_impl(c"llvm.used", &cx.used_statics);
             }
-            if !cx.compiler_used_statics.borrow().is_empty() {
-                cx.create_used_variable_impl(
-                    c"llvm.compiler.used",
-                    &*cx.compiler_used_statics.borrow(),
-                );
+            if !cx.compiler_used_statics.is_empty() {
+                cx.create_used_variable_impl(c"llvm.compiler.used", &cx.compiler_used_statics);
             }
 
             // Run replace-all-uses-with for statics that need it. This must

compiler/rustc_codegen_llvm/src/builder.rs

@@ -612,7 +612,7 @@ fn atomic_load(
         &mut self,
         ty: &'ll Type,
         ptr: &'ll Value,
-        order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_middle::ty::AtomicOrdering,
         size: Size,
     ) -> &'ll Value {
         unsafe {
@@ -851,7 +851,7 @@ fn atomic_store(
         &mut self,
         val: &'ll Value,
         ptr: &'ll Value,
-        order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_middle::ty::AtomicOrdering,
         size: Size,
     ) {
         debug!("Store {:?} -> {:?}", val, ptr);
@@ -1307,8 +1307,8 @@ fn atomic_cmpxchg(
         dst: &'ll Value,
         cmp: &'ll Value,
         src: &'ll Value,
-        order: rustc_codegen_ssa::common::AtomicOrdering,
-        failure_order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_middle::ty::AtomicOrdering,
+        failure_order: rustc_middle::ty::AtomicOrdering,
         weak: bool,
     ) -> (&'ll Value, &'ll Value) {
         let weak = if weak { llvm::True } else { llvm::False };
@@ -1334,7 +1334,7 @@ fn atomic_rmw(
         op: rustc_codegen_ssa::common::AtomicRmwBinOp,
         dst: &'ll Value,
         mut src: &'ll Value,
-        order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_middle::ty::AtomicOrdering,
     ) -> &'ll Value {
         // The only RMW operation that LLVM supports on pointers is compare-exchange.
         let requires_cast_to_int = self.val_ty(src) == self.type_ptr()
@@ -1360,7 +1360,7 @@ fn atomic_rmw(
 
     fn atomic_fence(
         &mut self,
-        order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_middle::ty::AtomicOrdering,
         scope: SynchronizationScope,
     ) {
         let single_threaded = match scope {
@@ -1452,9 +1452,15 @@ fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
 impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
     fn get_static(&mut self, def_id: DefId) -> &'ll Value {
         // Forward to the `get_static` method of `CodegenCx`
-        let s = self.cx().get_static(def_id);
-        // Cast to default address space if globals are in a different addrspace
-        self.cx().const_pointercast(s, self.type_ptr())
+        let global = self.cx().get_static(def_id);
+        if self.cx().tcx.is_thread_local_static(def_id) {
+            let pointer = self.call_intrinsic("llvm.threadlocal.address", &[global]);
+            // Cast to default address space if globals are in a different addrspace
+            self.pointercast(pointer, self.type_ptr())
+        } else {
+            // Cast to default address space if globals are in a different addrspace
+            self.cx().const_pointercast(global, self.type_ptr())
+        }
     }
 }
 

compiler/rustc_codegen_llvm/src/consts.rs

@@ -411,7 +411,7 @@ fn get_static_inner(&self, def_id: DefId, llty: &'ll Type) -> &'ll Value {
         g
     }
 
-    fn codegen_static_item(&self, def_id: DefId) {
+    fn codegen_static_item(&mut self, def_id: DefId) {
         unsafe {
             assert!(
                 llvm::LLVMGetInitializer(
@@ -557,6 +557,17 @@ fn codegen_static_item(&self, def_id: DefId) {
             }
         }
     }
+
+    /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr.
+    pub(crate) fn add_used_global(&mut self, global: &'ll Value) {
+        self.used_statics.push(global);
+    }
+
+    /// Add a global value to a list to be stored in the `llvm.compiler.used` variable,
+    /// an array of ptr.
+    pub(crate) fn add_compiler_used_global(&mut self, global: &'ll Value) {
+        self.compiler_used_statics.push(global);
+    }
 }
 
 impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> {
@@ -571,18 +582,7 @@ fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'
         self.const_pointercast(gv, self.type_ptr())
     }
 
-    fn codegen_static(&self, def_id: DefId) {
+    fn codegen_static(&mut self, def_id: DefId) {
         self.codegen_static_item(def_id)
     }
-
-    /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr.
-    fn add_used_global(&self, global: &'ll Value) {
-        self.used_statics.borrow_mut().push(global);
-    }
-
-    /// Add a global value to a list to be stored in the `llvm.compiler.used` variable,
-    /// an array of ptr.
-    fn add_compiler_used_global(&self, global: &'ll Value) {
-        self.compiler_used_statics.borrow_mut().push(global);
-    }
 }

compiler/rustc_codegen_llvm/src/context.rs

@@ -2,7 +2,7 @@
 use std::cell::{Cell, RefCell};
 use std::ffi::{CStr, c_char, c_uint};
 use std::marker::PhantomData;
-use std::ops::Deref;
+use std::ops::{Deref, DerefMut};
 use std::str;
 
 use rustc_abi::{HasDataLayout, Size, TargetDataLayout, VariantIdx};
@@ -77,6 +77,13 @@ fn deref(&self) -> &Self::Target {
     }
 }
 
+impl<'ll, T: Borrow<SCx<'ll>>> DerefMut for GenericCx<'ll, T> {
+    #[inline]
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
 pub(crate) type SimpleCx<'ll> = GenericCx<'ll, SCx<'ll>>;
 
 /// There is one `CodegenCx` per codegen unit. Each one has its own LLVM
@@ -110,11 +117,11 @@ pub(crate) struct FullCx<'ll, 'tcx> {
 
     /// Statics that will be placed in the llvm.used variable
     /// See <https://llvm.org/docs/LangRef.html#the-llvm-used-global-variable> for details
-    pub used_statics: RefCell<Vec<&'ll Value>>,
+    pub used_statics: Vec<&'ll Value>,
 
     /// Statics that will be placed in the llvm.compiler.used variable
     /// See <https://llvm.org/docs/LangRef.html#the-llvm-compiler-used-global-variable> for details
-    pub compiler_used_statics: RefCell<Vec<&'ll Value>>,
+    pub compiler_used_statics: Vec<&'ll Value>,
 
     /// Mapping of non-scalar types to llvm types.
     pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), &'ll Type>>,
@@ -606,8 +613,8 @@ pub(crate) fn new(
                 const_str_cache: Default::default(),
                 const_globals: Default::default(),
                 statics_to_rauw: RefCell::new(Vec::new()),
-                used_statics: RefCell::new(Vec::new()),
-                compiler_used_statics: RefCell::new(Vec::new()),
+                used_statics: Vec::new(),
+                compiler_used_statics: Vec::new(),
                 type_lowering: Default::default(),
                 scalar_lltypes: Default::default(),
                 coverage_cx,
@@ -801,10 +808,6 @@ fn sess(&self) -> &Session {
         self.tcx.sess
     }
 
-    fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx> {
-        self.codegen_unit
-    }
-
     fn set_frame_pointer_type(&self, llfn: &'ll Value) {
         if let Some(attr) = attributes::frame_pointer_type_attr(self) {
             attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]);
@@ -1240,6 +1243,7 @@ macro_rules! mk_struct {
         }
 
         ifn!("llvm.ptrmask", fn(ptr, t_isize) -> ptr);
+        ifn!("llvm.threadlocal.address", fn(ptr) -> ptr);
 
         None
     }

compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs

@@ -2,9 +2,7 @@
 
 use itertools::Itertools;
 use rustc_abi::Align;
-use rustc_codegen_ssa::traits::{
-    BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods,
-};
+use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_index::IndexVec;
 use rustc_middle::ty::TyCtxt;
@@ -27,7 +25,7 @@
 ///
 /// Those sections are then read and understood by LLVM's `llvm-cov` tool,
 /// which is distributed in the `llvm-tools` rustup component.
-pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) {
+pub(crate) fn finalize(cx: &mut CodegenCx<'_, '_>) {
     let tcx = cx.tcx;
 
     // Ensure that LLVM is using a version of the coverage mapping format that
@@ -62,6 +60,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) {
         .sorted_by_cached_key(|&instance| tcx.symbol_name(instance).name)
         .filter_map(|instance| prepare_covfun_record(tcx, instance, true))
         .collect::<Vec<_>>();
+    drop(instances_used);
 
     // In a single designated CGU, also prepare covfun records for functions
     // in this crate that were instrumented for coverage, but are unused.
@@ -206,7 +205,7 @@ fn resolve_all(&self, global_file_table: &GlobalFileTable) -> Option<Vec<u32>> {
 /// Generates the contents of the covmap record for this CGU, which mostly
 /// consists of a header and a list of filenames. The record is then stored
 /// as a global variable in the `__llvm_covmap` section.
-fn generate_covmap_record<'ll>(cx: &CodegenCx<'ll, '_>, version: u32, filenames_buffer: &[u8]) {
+fn generate_covmap_record<'ll>(cx: &mut CodegenCx<'ll, '_>, version: u32, filenames_buffer: &[u8]) {
     // A covmap record consists of four target-endian u32 values, followed by
     // the encoded filenames table. Two of the header fields are unused in
     // modern versions of the LLVM coverage mapping format, and are always 0.

compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs

@@ -8,9 +8,7 @@
 use std::sync::Arc;
 
 use rustc_abi::Align;
-use rustc_codegen_ssa::traits::{
-    BaseTypeCodegenMethods as _, ConstCodegenMethods, StaticCodegenMethods,
-};
+use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods as _, ConstCodegenMethods};
 use rustc_middle::mir::coverage::{
     BasicCoverageBlock, CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping,
     MappingKind, Op,
@@ -181,7 +179,7 @@ fn fill_region_tables<'tcx>(
 /// contains the function's coverage mapping data. The record is then stored
 /// as a global variable in the `__llvm_covfun` section.
 pub(crate) fn generate_covfun_record<'tcx>(
-    cx: &CodegenCx<'_, 'tcx>,
+    cx: &mut CodegenCx<'_, 'tcx>,
     global_file_table: &GlobalFileTable,
     covfun: &CovfunRecord<'tcx>,
 ) {

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -56,7 +56,7 @@ fn try_get_mcdc_condition_bitmap(
 }
 
 impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
-    pub(crate) fn coverageinfo_finalize(&self) {
+    pub(crate) fn coverageinfo_finalize(&mut self) {
         mapgen::finalize(self)
     }
 

compiler/rustc_codegen_llvm/src/lib.rs

@@ -189,13 +189,13 @@ fn run_thin_lto(
     ) -> Result<(Vec<LtoModuleCodegen<Self>>, Vec<WorkProduct>), FatalError> {
         back::lto::run_thin(cgcx, modules, cached_modules)
     }
-    unsafe fn optimize(
+    fn optimize(
         cgcx: &CodegenContext<Self>,
         dcx: DiagCtxtHandle<'_>,
         module: &mut ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<(), FatalError> {
-        unsafe { back::write::optimize(cgcx, dcx, module, config) }
+        back::write::optimize(cgcx, dcx, module, config)
     }
     fn optimize_fat(
         cgcx: &CodegenContext<Self>,
@@ -205,19 +205,19 @@ fn optimize_fat(
         let dcx = dcx.handle();
         back::lto::run_pass_manager(cgcx, dcx, module, false)
     }
-    unsafe fn optimize_thin(
+    fn optimize_thin(
         cgcx: &CodegenContext<Self>,
         thin: ThinModule<Self>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
-        unsafe { back::lto::optimize_thin_module(thin, cgcx) }
+        back::lto::optimize_thin_module(thin, cgcx)
     }
-    unsafe fn codegen(
+    fn codegen(
         cgcx: &CodegenContext<Self>,
         dcx: DiagCtxtHandle<'_>,
         module: ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<CompiledModule, FatalError> {
-        unsafe { back::write::codegen(cgcx, dcx, module, config) }
+        back::write::codegen(cgcx, dcx, module, config)
     }
     fn prepare_thin(
         module: ModuleCodegen<Self::Module>,

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -426,14 +426,14 @@ pub(crate) enum AtomicOrdering {
 }
 
 impl AtomicOrdering {
-    pub(crate) fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self {
-        use rustc_codegen_ssa::common::AtomicOrdering as Common;
+    pub(crate) fn from_generic(ao: rustc_middle::ty::AtomicOrdering) -> Self {
+        use rustc_middle::ty::AtomicOrdering as Common;
         match ao {
             Common::Relaxed => Self::Monotonic,
             Common::Acquire => Self::Acquire,
             Common::Release => Self::Release,
-            Common::AcquireRelease => Self::AcquireRelease,
-            Common::SequentiallyConsistent => Self::SequentiallyConsistent,
+            Common::AcqRel => Self::AcquireRelease,
+            Common::SeqCst => Self::SequentiallyConsistent,
         }
     }
 }

compiler/rustc_codegen_llvm/src/mono_item.rs

@@ -16,7 +16,7 @@
 
 impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
     fn predefine_static(
-        &self,
+        &mut self,
         def_id: DefId,
         linkage: Linkage,
         visibility: Visibility,
@@ -44,7 +44,7 @@ fn predefine_static(
     }
 
     fn predefine_fn(
-        &self,
+        &mut self,
         instance: Instance<'tcx>,
         linkage: Linkage,
         visibility: Visibility,

compiler/rustc_codegen_llvm/src/va_arg.rs

@@ -1,7 +1,10 @@
-use rustc_abi::{Align, Endian, HasDataLayout, Size};
+use rustc_abi::{Align, BackendRepr, Endian, HasDataLayout, Primitive, Size, TyAndLayout};
+use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::IntPredicate;
 use rustc_codegen_ssa::mir::operand::OperandRef;
-use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods};
+use rustc_codegen_ssa::traits::{
+    BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, LayoutTypeCodegenMethods,
+};
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 
@@ -303,6 +306,313 @@ fn emit_s390x_va_arg<'ll, 'tcx>(
     bx.load(val_type, val_addr, layout.align.abi)
 }
 
+fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    list: OperandRef<'tcx, &'ll Value>,
+    target_ty: Ty<'tcx>,
+) -> &'ll Value {
+    let dl = bx.cx.data_layout();
+
+    // Implementation of the systemv x86_64 ABI calling convention for va_args, see
+    // https://gitlab.com/x86-psABIs/x86-64-ABI (section 3.5.7). This implementation is heavily
+    // based on the one in clang.
+
+    // We're able to take some shortcuts because the return type of `va_arg` must implement the
+    // `VaArgSafe` trait. Currently, only pointers, f64, i32, u32, i64 and u64 implement this trait.
+
+    // typedef struct __va_list_tag {
+    //     unsigned int gp_offset;
+    //     unsigned int fp_offset;
+    //     void *overflow_arg_area;
+    //     void *reg_save_area;
+    // } va_list[1];
+    let va_list_addr = list.immediate();
+
+    // Peel off any newtype wrappers.
+    //
+    // The "C" ABI does not unwrap newtypes (see `ReprOptions::inhibit_newtype_abi_optimization`).
+    // Here, we do actually want the unwrapped representation, because that is how LLVM/Clang
+    // pass such types to variadic functions.
+    //
+    // An example of a type that must be unwrapped is `Foo` below. Without the unwrapping, it has
+    // `BackendRepr::Memory`, but we need it to be `BackendRepr::Scalar` to generate correct code.
+    //
+    // ```
+    // #[repr(C)]
+    // struct Empty;
+    //
+    // #[repr(C)]
+    // struct Foo([Empty; 8], i32);
+    // ```
+    let layout = {
+        let mut layout = bx.cx.layout_of(target_ty);
+
+        while let Some((_, inner)) = layout.non_1zst_field(bx.cx) {
+            layout = inner;
+        }
+
+        layout
+    };
+
+    // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
+    // in the registers. If not go to step 7.
+
+    // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
+    // general purpose registers needed to pass type and num_fp to hold
+    // the number of floating point registers needed.
+
+    let mut num_gp_registers = 0;
+    let mut num_fp_registers = 0;
+
+    let mut registers_for_primitive = |p| match p {
+        Primitive::Int(integer, _is_signed) => {
+            num_gp_registers += integer.size().bytes().div_ceil(8) as u32;
+        }
+        Primitive::Float(float) => {
+            num_fp_registers += float.size().bytes().div_ceil(16) as u32;
+        }
+        Primitive::Pointer(_) => {
+            num_gp_registers += 1;
+        }
+    };
+
+    match layout.layout.backend_repr() {
+        BackendRepr::Scalar(scalar) => {
+            registers_for_primitive(scalar.primitive());
+        }
+        BackendRepr::ScalarPair(scalar1, scalar2) => {
+            registers_for_primitive(scalar1.primitive());
+            registers_for_primitive(scalar2.primitive());
+        }
+        BackendRepr::SimdVector { .. } => {
+            // Because no instance of VaArgSafe uses a non-scalar `BackendRepr`.
+            unreachable!(
+                "No x86-64 SysV va_arg implementation for {:?}",
+                layout.layout.backend_repr()
+            )
+        }
+        BackendRepr::Memory { .. } => {
+            let mem_addr = x86_64_sysv64_va_arg_from_memory(bx, va_list_addr, layout);
+            return bx.load(layout.llvm_type(bx), mem_addr, layout.align.abi);
+        }
+    };
+
+    // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
+    // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
+    // l->fp_offset > 176 - num_fp * 16 go to step 7.
+
+    let unsigned_int_offset = 4;
+    let ptr_offset = 8;
+    let gp_offset_ptr = va_list_addr;
+    let fp_offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(unsigned_int_offset));
+
+    let gp_offset_v = bx.load(bx.type_i32(), gp_offset_ptr, Align::from_bytes(8).unwrap());
+    let fp_offset_v = bx.load(bx.type_i32(), fp_offset_ptr, Align::from_bytes(4).unwrap());
+
+    let mut use_regs = bx.const_bool(false);
+
+    if num_gp_registers > 0 {
+        let max_offset_val = 48u32 - num_gp_registers * 8;
+        let fits_in_gp = bx.icmp(IntPredicate::IntULE, gp_offset_v, bx.const_u32(max_offset_val));
+        use_regs = fits_in_gp;
+    }
+
+    if num_fp_registers > 0 {
+        let max_offset_val = 176u32 - num_fp_registers * 16;
+        let fits_in_fp = bx.icmp(IntPredicate::IntULE, fp_offset_v, bx.const_u32(max_offset_val));
+        use_regs = if num_gp_registers > 0 { bx.and(use_regs, fits_in_fp) } else { fits_in_fp };
+    }
+
+    let in_reg = bx.append_sibling_block("va_arg.in_reg");
+    let in_mem = bx.append_sibling_block("va_arg.in_mem");
+    let end = bx.append_sibling_block("va_arg.end");
+
+    bx.cond_br(use_regs, in_reg, in_mem);
+
+    // Emit code to load the value if it was passed in a register.
+    bx.switch_to_block(in_reg);
+
+    // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
+    // an offset of l->gp_offset and/or l->fp_offset. This may require
+    // copying to a temporary location in case the parameter is passed
+    // in different register classes or requires an alignment greater
+    // than 8 for general purpose registers and 16 for XMM registers.
+    //
+    // FIXME(llvm): This really results in shameful code when we end up needing to
+    // collect arguments from different places; often what should result in a
+    // simple assembling of a structure from scattered addresses has many more
+    // loads than necessary. Can we clean this up?
+    let reg_save_area_ptr =
+        bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * unsigned_int_offset + ptr_offset));
+    let reg_save_area_v = bx.load(bx.type_ptr(), reg_save_area_ptr, dl.pointer_align.abi);
+
+    let reg_addr = match layout.layout.backend_repr() {
+        BackendRepr::Scalar(scalar) => match scalar.primitive() {
+            Primitive::Int(_, _) | Primitive::Pointer(_) => {
+                let reg_addr = bx.inbounds_ptradd(reg_save_area_v, gp_offset_v);
+
+                // Copy into a temporary if the type is more aligned than the register save area.
+                let gp_align = Align::from_bytes(8).unwrap();
+                copy_to_temporary_if_more_aligned(bx, reg_addr, layout, gp_align)
+            }
+            Primitive::Float(_) => bx.inbounds_ptradd(reg_save_area_v, fp_offset_v),
+        },
+        BackendRepr::ScalarPair(scalar1, scalar2) => {
+            let ty_lo = bx.cx().scalar_pair_element_backend_type(layout, 0, false);
+            let ty_hi = bx.cx().scalar_pair_element_backend_type(layout, 1, false);
+
+            let align_lo = layout.field(bx.cx, 0).layout.align().abi;
+            let align_hi = layout.field(bx.cx, 1).layout.align().abi;
+
+            match (scalar1.primitive(), scalar2.primitive()) {
+                (Primitive::Float(_), Primitive::Float(_)) => {
+                    // SSE registers are spaced 16 bytes apart in the register save
+                    // area, we need to collect the two eightbytes together.
+                    // The ABI isn't explicit about this, but it seems reasonable
+                    // to assume that the slots are 16-byte aligned, since the stack is
+                    // naturally 16-byte aligned and the prologue is expected to store
+                    // all the SSE registers to the RSA.
+                    let reg_lo_addr = bx.inbounds_ptradd(reg_save_area_v, fp_offset_v);
+                    let reg_hi_addr = bx.inbounds_ptradd(reg_lo_addr, bx.const_i32(16));
+
+                    let align = layout.layout.align().abi;
+                    let tmp = bx.alloca(layout.layout.size(), align);
+
+                    let reg_lo = bx.load(ty_lo, reg_lo_addr, align_lo);
+                    let reg_hi = bx.load(ty_hi, reg_hi_addr, align_hi);
+
+                    let offset = scalar1.size(bx.cx).align_to(align_hi).bytes();
+                    let field0 = tmp;
+                    let field1 = bx.inbounds_ptradd(tmp, bx.const_u32(offset as u32));
+
+                    bx.store(reg_lo, field0, align);
+                    bx.store(reg_hi, field1, align);
+
+                    tmp
+                }
+                (Primitive::Float(_), _) | (_, Primitive::Float(_)) => {
+                    let gp_addr = bx.inbounds_ptradd(reg_save_area_v, gp_offset_v);
+                    let fp_addr = bx.inbounds_ptradd(reg_save_area_v, fp_offset_v);
+
+                    let (reg_lo_addr, reg_hi_addr) = match scalar1.primitive() {
+                        Primitive::Float(_) => (fp_addr, gp_addr),
+                        Primitive::Int(_, _) | Primitive::Pointer(_) => (gp_addr, fp_addr),
+                    };
+
+                    let tmp = bx.alloca(layout.layout.size(), layout.layout.align().abi);
+
+                    let reg_lo = bx.load(ty_lo, reg_lo_addr, align_lo);
+                    let reg_hi = bx.load(ty_hi, reg_hi_addr, align_hi);
+
+                    let offset = scalar1.size(bx.cx).align_to(align_hi).bytes();
+                    let field0 = tmp;
+                    let field1 = bx.inbounds_ptradd(tmp, bx.const_u32(offset as u32));
+
+                    bx.store(reg_lo, field0, align_lo);
+                    bx.store(reg_hi, field1, align_hi);
+
+                    tmp
+                }
+                (_, _) => {
+                    // Two integer/pointer values are just contiguous in memory.
+                    let reg_addr = bx.inbounds_ptradd(reg_save_area_v, gp_offset_v);
+
+                    // Copy into a temporary if the type is more aligned than the register save area.
+                    let gp_align = Align::from_bytes(8).unwrap();
+                    copy_to_temporary_if_more_aligned(bx, reg_addr, layout, gp_align)
+                }
+            }
+        }
+        // The Previous match on `BackendRepr` means control flow already escaped.
+        BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => unreachable!(),
+    };
+
+    // AMD64-ABI 3.5.7p5: Step 5. Set:
+    // l->gp_offset = l->gp_offset + num_gp * 8
+    if num_gp_registers > 0 {
+        let offset = bx.const_u32(num_gp_registers * 8);
+        let sum = bx.add(gp_offset_v, offset);
+        // An alignment of 8 because `__va_list_tag` is 8-aligned and this is its first field.
+        bx.store(sum, gp_offset_ptr, Align::from_bytes(8).unwrap());
+    }
+
+    // l->fp_offset = l->fp_offset + num_fp * 16.
+    if num_fp_registers > 0 {
+        let offset = bx.const_u32(num_fp_registers * 16);
+        let sum = bx.add(fp_offset_v, offset);
+        bx.store(sum, fp_offset_ptr, Align::from_bytes(4).unwrap());
+    }
+
+    bx.br(end);
+
+    bx.switch_to_block(in_mem);
+    let mem_addr = x86_64_sysv64_va_arg_from_memory(bx, va_list_addr, layout);
+    bx.br(end);
+
+    bx.switch_to_block(end);
+
+    let val_type = layout.llvm_type(bx);
+    let val_addr = bx.phi(bx.type_ptr(), &[reg_addr, mem_addr], &[in_reg, in_mem]);
+
+    bx.load(val_type, val_addr, layout.align.abi)
+}
+
+/// Copy into a temporary if the type is more aligned than the register save area.
+fn copy_to_temporary_if_more_aligned<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    reg_addr: &'ll Value,
+    layout: TyAndLayout<'tcx, Ty<'tcx>>,
+    src_align: Align,
+) -> &'ll Value {
+    if layout.layout.align.abi > src_align {
+        let tmp = bx.alloca(layout.layout.size(), layout.layout.align().abi);
+        bx.memcpy(
+            tmp,
+            layout.layout.align.abi,
+            reg_addr,
+            src_align,
+            bx.const_u32(layout.layout.size().bytes() as u32),
+            MemFlags::empty(),
+        );
+        tmp
+    } else {
+        reg_addr
+    }
+}
+
+fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    va_list_addr: &'ll Value,
+    layout: TyAndLayout<'tcx, Ty<'tcx>>,
+) -> &'ll Value {
+    let dl = bx.cx.data_layout();
+
+    let overflow_arg_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.const_usize(8));
+
+    let overflow_arg_area_v = bx.load(bx.type_ptr(), overflow_arg_area_ptr, dl.pointer_align.abi);
+    // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
+    // byte boundary if alignment needed by type exceeds 8 byte boundary.
+    // It isn't stated explicitly in the standard, but in practice we use
+    // alignment greater than 16 where necessary.
+    if layout.layout.align.abi.bytes() > 8 {
+        unreachable!("all instances of VaArgSafe have an alignment <= 8");
+    }
+
+    // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
+    let mem_addr = overflow_arg_area_v;
+
+    // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
+    // l->overflow_arg_area + sizeof(type).
+    // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
+    // an 8 byte boundary.
+    let size_in_bytes = layout.layout.size().bytes();
+    let offset = bx.const_i32(size_in_bytes.next_multiple_of(8) as i32);
+    let overflow_arg_area = bx.inbounds_ptradd(overflow_arg_area_v, offset);
+    bx.store(overflow_arg_area, overflow_arg_area_ptr, dl.pointer_align.abi);
+
+    mem_addr
+}
+
 fn emit_xtensa_va_arg<'ll, 'tcx>(
     bx: &mut Builder<'_, 'll, 'tcx>,
     list: OperandRef<'tcx, &'ll Value>,
@@ -334,8 +644,7 @@ fn emit_xtensa_va_arg<'ll, 'tcx>(
     // (*va).va_ndx
     let va_reg_offset = 4;
     let va_ndx_offset = va_reg_offset + 4;
-    let offset_ptr =
-        bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(va_ndx_offset)]);
+    let offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(va_ndx_offset));
 
     let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align.abi);
     let offset = round_up_to_alignment(bx, offset, layout.align.abi);
@@ -356,11 +665,10 @@ fn emit_xtensa_va_arg<'ll, 'tcx>(
     bx.store(offset_next, offset_ptr, bx.tcx().data_layout.pointer_align.abi);
 
     // (*va).va_reg
-    let regsave_area_ptr =
-        bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(va_reg_offset)]);
+    let regsave_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(va_reg_offset));
     let regsave_area =
         bx.load(bx.type_ptr(), regsave_area_ptr, bx.tcx().data_layout.pointer_align.abi);
-    let regsave_value_ptr = bx.inbounds_gep(bx.type_i8(), regsave_area, &[offset]);
+    let regsave_value_ptr = bx.inbounds_ptradd(regsave_area, offset);
     bx.br(end);
 
     bx.switch_to_block(from_stack);
@@ -381,9 +689,9 @@ fn emit_xtensa_va_arg<'ll, 'tcx>(
     bx.store(offset_next_corrected, offset_ptr, bx.tcx().data_layout.pointer_align.abi);
 
     // let stack_value_ptr = unsafe { (*va).va_stk.byte_add(offset_corrected) };
-    let stack_area_ptr = bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(0)]);
+    let stack_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(0));
     let stack_area = bx.load(bx.type_ptr(), stack_area_ptr, bx.tcx().data_layout.pointer_align.abi);
-    let stack_value_ptr = bx.inbounds_gep(bx.type_i8(), stack_area, &[offset_corrected]);
+    let stack_value_ptr = bx.inbounds_ptradd(stack_area, offset_corrected);
     bx.br(end);
 
     bx.switch_to_block(end);
@@ -449,6 +757,8 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
                 AllowHigherAlign::No,
             )
         }
+        // This includes `target.is_like_darwin`, which on x86_64 targets is like sysv64.
+        "x86_64" => emit_x86_64_sysv64_va_arg(bx, addr, target_ty),
         "xtensa" => emit_xtensa_va_arg(bx, addr, target_ty),
         // For all other architecture/OS combinations fall back to using
         // the LLVM va_arg instruction.

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -6,7 +6,7 @@
 use std::io::{BufWriter, Write};
 use std::ops::{ControlFlow, Deref};
 use std::path::{Path, PathBuf};
-use std::process::{ExitStatus, Output, Stdio};
+use std::process::{Output, Stdio};
 use std::{env, fmt, fs, io, mem, str};
 
 use cc::windows_registry;
@@ -736,13 +736,10 @@ fn link_natively(
 
     // Invoke the system linker
     info!("{cmd:?}");
-    let retry_on_segfault = env::var("RUSTC_RETRY_LINKER_ON_SEGFAULT").is_ok();
     let unknown_arg_regex =
         Regex::new(r"(unknown|unrecognized) (command line )?(option|argument)").unwrap();
     let mut prog;
-    let mut i = 0;
     loop {
-        i += 1;
         prog = sess.time("run_linker", || exec_linker(sess, &cmd, out_filename, flavor, tmpdir));
         let Ok(ref output) = prog else {
             break;
@@ -858,54 +855,7 @@ fn link_natively(
             continue;
         }
 
-        // Here's a terribly awful hack that really shouldn't be present in any
-        // compiler. Here an environment variable is supported to automatically
-        // retry the linker invocation if the linker looks like it segfaulted.
-        //
-        // Gee that seems odd, normally segfaults are things we want to know
-        // about!  Unfortunately though in rust-lang/rust#38878 we're
-        // experiencing the linker segfaulting on Travis quite a bit which is
-        // causing quite a bit of pain to land PRs when they spuriously fail
-        // due to a segfault.
-        //
-        // The issue #38878 has some more debugging information on it as well,
-        // but this unfortunately looks like it's just a race condition in
-        // macOS's linker with some thread pool working in the background. It
-        // seems that no one currently knows a fix for this so in the meantime
-        // we're left with this...
-        if !retry_on_segfault || i > 3 {
-            break;
-        }
-        let msg_segv = "clang: error: unable to execute command: Segmentation fault: 11";
-        let msg_bus = "clang: error: unable to execute command: Bus error: 10";
-        if out.contains(msg_segv) || out.contains(msg_bus) {
-            warn!(
-                ?cmd, %out,
-                "looks like the linker segfaulted when we tried to call it, \
-                 automatically retrying again",
-            );
-            continue;
-        }
-
-        if is_illegal_instruction(&output.status) {
-            warn!(
-                ?cmd, %out, status = %output.status,
-                "looks like the linker hit an illegal instruction when we \
-                 tried to call it, automatically retrying again.",
-            );
-            continue;
-        }
-
-        #[cfg(unix)]
-        fn is_illegal_instruction(status: &ExitStatus) -> bool {
-            use std::os::unix::prelude::*;
-            status.signal() == Some(libc::SIGILL)
-        }
-
-        #[cfg(not(unix))]
-        fn is_illegal_instruction(_status: &ExitStatus) -> bool {
-            false
-        }
+        break;
     }
 
     match prog {

compiler/rustc_codegen_ssa/src/back/lto.rs

@@ -56,12 +56,7 @@ pub fn name(&self) -> &str {
     }
 
     /// Optimize this module within the given codegen context.
-    ///
-    /// This function is unsafe as it'll return a `ModuleCodegen` still
-    /// points to LLVM data structures owned by this `LtoModuleCodegen`.
-    /// It's intended that the module returned is immediately code generated and
-    /// dropped, and then this LTO module is dropped.
-    pub unsafe fn optimize(
+    pub fn optimize(
         self,
         cgcx: &CodegenContext<B>,
     ) -> Result<ModuleCodegen<B::Module>, FatalError> {
@@ -70,7 +65,7 @@ pub unsafe fn optimize(
                 B::optimize_fat(cgcx, &mut module)?;
                 Ok(module)
             }
-            LtoModuleCodegen::Thin(thin) => unsafe { B::optimize_thin(cgcx, thin) },
+            LtoModuleCodegen::Thin(thin) => B::optimize_thin(cgcx, thin),
         }
     }
 
@@ -85,7 +80,7 @@ pub fn cost(&self) -> u64 {
     }
 
     /// Run autodiff on Fat LTO module
-    pub unsafe fn autodiff(
+    pub fn autodiff(
         self,
         cgcx: &CodegenContext<B>,
         diff_fncs: Vec<AutoDiffItem>,

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -383,7 +383,7 @@ pub struct CodegenContext<B: WriteBackendMethods> {
     pub coordinator_send: Sender<Box<dyn Any + Send>>,
     /// `true` if the codegen should be run in parallel.
     ///
-    /// Depends on [`CodegenBackend::supports_parallel()`] and `-Zno_parallel_backend`.
+    /// Depends on [`ExtraBackendMethods::supports_parallel()`] and `-Zno_parallel_backend`.
     pub parallel: bool,
 }
 
@@ -416,8 +416,7 @@ fn generate_lto_work<B: ExtraBackendMethods>(
             B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
         if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
             let config = cgcx.config(ModuleKind::Regular);
-            module =
-                unsafe { module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise()) };
+            module = module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise());
         }
         // We are adding a single work item, so the cost doesn't matter.
         vec![(WorkItem::LTO(module), 0)]
@@ -887,9 +886,7 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>(
     let dcx = cgcx.create_dcx();
     let dcx = dcx.handle();
 
-    unsafe {
-        B::optimize(cgcx, dcx, &mut module, module_config)?;
-    }
+    B::optimize(cgcx, dcx, &mut module, module_config)?;
 
     // After we've done the initial round of optimizations we need to
     // decide whether to synchronously codegen this module or ship it
@@ -1020,7 +1017,7 @@ fn execute_lto_work_item<B: ExtraBackendMethods>(
     module: lto::LtoModuleCodegen<B>,
     module_config: &ModuleConfig,
 ) -> Result<WorkItemResult<B>, FatalError> {
-    let module = unsafe { module.optimize(cgcx)? };
+    let module = module.optimize(cgcx)?;
     finish_intra_module_work(cgcx, module, module_config)
 }
 
@@ -1036,7 +1033,7 @@ fn finish_intra_module_work<B: ExtraBackendMethods>(
         || module.kind == ModuleKind::Metadata
         || module.kind == ModuleKind::Allocator
     {
-        let module = unsafe { B::codegen(cgcx, dcx, module, module_config)? };
+        let module = B::codegen(cgcx, dcx, module, module_config)?;
         Ok(WorkItemResult::Finished(module))
     } else {
         Ok(WorkItemResult::NeedsLink(module))
@@ -1725,9 +1722,8 @@ enum CodegenState {
             let dcx = cgcx.create_dcx();
             let dcx = dcx.handle();
             let module = B::run_link(&cgcx, dcx, needs_link).map_err(|_| ())?;
-            let module = unsafe {
-                B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?
-            };
+            let module =
+                B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?;
             compiled_modules.push(module);
         }
 

compiler/rustc_codegen_ssa/src/base.rs

@@ -492,6 +492,7 @@ pub fn codegen_global_asm<'tcx, Cx>(cx: &mut Cx, item_id: ItemId)
 /// users main function.
 pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     cx: &'a Bx::CodegenCx,
+    cgu: &CodegenUnit<'tcx>,
 ) -> Option<Bx::Function> {
     let (main_def_id, entry_type) = cx.tcx().entry_fn(())?;
     let main_is_local = main_def_id.is_local();
@@ -500,10 +501,10 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     if main_is_local {
         // We want to create the wrapper in the same codegen unit as Rust's main
         // function.
-        if !cx.codegen_unit().contains_item(&MonoItem::Fn(instance)) {
+        if !cgu.contains_item(&MonoItem::Fn(instance)) {
             return None;
         }
-    } else if !cx.codegen_unit().is_primary() {
+    } else if !cgu.is_primary() {
         // We want to create the wrapper only when the codegen unit is the primary one
         return None;
     }

compiler/rustc_codegen_ssa/src/common.rs

@@ -59,15 +59,6 @@ pub enum AtomicRmwBinOp {
     AtomicUMin,
 }
 
-#[derive(Copy, Clone, Debug)]
-pub enum AtomicOrdering {
-    Relaxed,
-    Acquire,
-    Release,
-    AcquireRelease,
-    SequentiallyConsistent,
-}
-
 #[derive(Copy, Clone, Debug)]
 pub enum SynchronizationScope {
     SingleThread,

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -99,6 +99,17 @@ pub fn codegen_intrinsic_call(
 
         let llret_ty = bx.backend_type(bx.layout_of(ret_ty));
 
+        let ret_llval = |bx: &mut Bx, llval| {
+            if result.layout.ty.is_bool() {
+                OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout)
+                    .val
+                    .store(bx, result);
+            } else if !result.layout.ty.is_unit() {
+                bx.store_to_place(llval, result.val);
+            }
+            Ok(())
+        };
+
         let llval = match name {
             sym::abort => {
                 bx.abort();
@@ -334,9 +345,48 @@ pub fn codegen_intrinsic_call(
             // This requires that atomic intrinsics follow a specific naming pattern:
             // "atomic_<operation>[_<ordering>]"
             name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
-                use crate::common::AtomicOrdering::*;
+                use rustc_middle::ty::AtomicOrdering::*;
+
                 use crate::common::{AtomicRmwBinOp, SynchronizationScope};
 
+                let invalid_monomorphization = |ty| {
+                    bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
+                        span,
+                        name,
+                        ty,
+                    });
+                };
+
+                let parse_const_generic_ordering = |ord: ty::Value<'tcx>| {
+                    let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
+                    discr.to_atomic_ordering()
+                };
+
+                // Some intrinsics have the ordering already converted to a const generic parameter, we handle those first.
+                match name {
+                    sym::atomic_load => {
+                        let ty = fn_args.type_at(0);
+                        let ordering = fn_args.const_at(1).to_value();
+                        if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
+                            invalid_monomorphization(ty);
+                            return Ok(());
+                        }
+                        let layout = bx.layout_of(ty);
+                        let source = args[0].immediate();
+                        let llval = bx.atomic_load(
+                            bx.backend_type(layout),
+                            source,
+                            parse_const_generic_ordering(ordering),
+                            layout.size,
+                        );
+
+                        return ret_llval(bx, llval);
+                    }
+
+                    // The rest falls back to below.
+                    _ => {}
+                }
+
                 let Some((instruction, ordering)) = atomic.split_once('_') else {
                     bx.sess().dcx().emit_fatal(errors::MissingMemoryOrdering);
                 };
@@ -345,19 +395,11 @@ pub fn codegen_intrinsic_call(
                     "relaxed" => Relaxed,
                     "acquire" => Acquire,
                     "release" => Release,
-                    "acqrel" => AcquireRelease,
-                    "seqcst" => SequentiallyConsistent,
+                    "acqrel" => AcqRel,
+                    "seqcst" => SeqCst,
                     _ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOrdering),
                 };
 
-                let invalid_monomorphization = |ty| {
-                    bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
-                        span,
-                        name,
-                        ty,
-                    });
-                };
-
                 match instruction {
                     "cxchg" | "cxchgweak" => {
                         let Some((success, failure)) = ordering.split_once('_') else {
@@ -390,24 +432,6 @@ pub fn codegen_intrinsic_call(
                         return Ok(());
                     }
 
-                    "load" => {
-                        let ty = fn_args.type_at(0);
-                        if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
-                            let layout = bx.layout_of(ty);
-                            let size = layout.size;
-                            let source = args[0].immediate();
-                            bx.atomic_load(
-                                bx.backend_type(layout),
-                                source,
-                                parse_ordering(bx, ordering),
-                                size,
-                            )
-                        } else {
-                            invalid_monomorphization(ty);
-                            return Ok(());
-                        }
-                    }
-
                     "store" => {
                         let ty = fn_args.type_at(0);
                         if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
@@ -538,14 +562,7 @@ pub fn codegen_intrinsic_call(
             }
         };
 
-        if result.layout.ty.is_bool() {
-            OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout)
-                .val
-                .store(bx, result);
-        } else if !result.layout.ty.is_unit() {
-            bx.store_to_place(llval, result.val);
-        }
-        Ok(())
+        ret_llval(bx, llval)
     }
 }
 

compiler/rustc_codegen_ssa/src/mono_item.rs

@@ -11,11 +11,13 @@ pub trait MonoItemExt<'a, 'tcx> {
     fn define<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
         cx: &'a mut Bx::CodegenCx,
+        cgu_name: &str,
         item_data: MonoItemData,
     );
     fn predefine<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        cx: &'a Bx::CodegenCx,
+        cx: &'a mut Bx::CodegenCx,
+        cgu_name: &str,
         linkage: Linkage,
         visibility: Visibility,
     );
@@ -26,14 +28,10 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
     fn define<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
         cx: &'a mut Bx::CodegenCx,
+        cgu_name: &str,
         item_data: MonoItemData,
     ) {
-        debug!(
-            "BEGIN IMPLEMENTING '{} ({})' in cgu {}",
-            self,
-            self.to_raw_string(),
-            cx.codegen_unit().name()
-        );
+        debug!("BEGIN IMPLEMENTING '{} ({})' in cgu {}", self, self.to_raw_string(), cgu_name);
 
         match *self {
             MonoItem::Static(def_id) => {
@@ -56,26 +54,17 @@ fn define<Bx: BuilderMethods<'a, 'tcx>>(
             }
         }
 
-        debug!(
-            "END IMPLEMENTING '{} ({})' in cgu {}",
-            self,
-            self.to_raw_string(),
-            cx.codegen_unit().name()
-        );
+        debug!("END IMPLEMENTING '{} ({})' in cgu {}", self, self.to_raw_string(), cgu_name);
     }
 
     fn predefine<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        cx: &'a Bx::CodegenCx,
+        cx: &'a mut Bx::CodegenCx,
+        cgu_name: &str,
         linkage: Linkage,
         visibility: Visibility,
     ) {
-        debug!(
-            "BEGIN PREDEFINING '{} ({})' in cgu {}",
-            self,
-            self.to_raw_string(),
-            cx.codegen_unit().name()
-        );
+        debug!("BEGIN PREDEFINING '{} ({})' in cgu {}", self, self.to_raw_string(), cgu_name);
 
         let symbol_name = self.symbol_name(cx.tcx()).name;
 
@@ -97,12 +86,7 @@ fn predefine<Bx: BuilderMethods<'a, 'tcx>>(
             MonoItem::GlobalAsm(..) => {}
         }
 
-        debug!(
-            "END PREDEFINING '{} ({})' in cgu {}",
-            self,
-            self.to_raw_string(),
-            cx.codegen_unit().name()
-        );
+        debug!("END PREDEFINING '{} ({})' in cgu {}", self, self.to_raw_string(), cgu_name);
     }
 
     fn to_raw_string(&self) -> String {

compiler/rustc_codegen_ssa/src/traits/backend.rs

@@ -21,12 +21,12 @@
 use crate::{CodegenResults, ModuleCodegen, TargetConfig};
 
 pub trait BackendTypes {
-    type Value: CodegenObject;
+    type Value: CodegenObject + PartialEq;
     type Metadata: CodegenObject;
     type Function: CodegenObject;
 
     type BasicBlock: Copy;
-    type Type: CodegenObject;
+    type Type: CodegenObject + PartialEq;
     type Funclet;
 
     // FIXME(eddyb) find a common convention for all of the debuginfo-related
@@ -97,13 +97,6 @@ fn join_codegen(
     fn link(&self, sess: &Session, codegen_results: CodegenResults, outputs: &OutputFilenames) {
         link_binary(sess, &ArArchiveBuilderBuilder, codegen_results, outputs);
     }
-
-    /// Returns `true` if this backend can be safely called from multiple threads.
-    ///
-    /// Defaults to `true`.
-    fn supports_parallel(&self) -> bool {
-        true
-    }
 }
 
 pub trait ExtraBackendMethods:
@@ -144,4 +137,11 @@ fn spawn_named_thread<F, T>(
     {
         std::thread::Builder::new().name(name).spawn(f)
     }
+
+    /// Returns `true` if this backend can be safely called from multiple threads.
+    ///
+    /// Defaults to `true`.
+    fn supports_parallel(&self) -> bool {
+        true
+    }
 }

compiler/rustc_codegen_ssa/src/traits/builder.rs

@@ -4,7 +4,7 @@
 use rustc_abi::{Align, Scalar, Size, WrappingRange};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
-use rustc_middle::ty::{Instance, Ty};
+use rustc_middle::ty::{AtomicOrdering, Instance, Ty};
 use rustc_session::config::OptLevel;
 use rustc_span::Span;
 use rustc_target::callconv::FnAbi;
@@ -19,9 +19,7 @@
 use super::type_::{ArgAbiBuilderMethods, BaseTypeCodegenMethods, LayoutTypeCodegenMethods};
 use super::{CodegenMethods, StaticBuilderMethods};
 use crate::MemFlags;
-use crate::common::{
-    AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind,
-};
+use crate::common::{AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
 use crate::mir::operand::{OperandRef, OperandValue};
 use crate::mir::place::{PlaceRef, PlaceValue};
 
@@ -514,11 +512,11 @@ fn select(
     fn extract_value(&mut self, agg_val: Self::Value, idx: u64) -> Self::Value;
     fn insert_value(&mut self, agg_val: Self::Value, elt: Self::Value, idx: u64) -> Self::Value;
 
-    fn set_personality_fn(&mut self, personality: Self::Value);
+    fn set_personality_fn(&mut self, personality: Self::Function);
 
     // These are used by everyone except msvc
-    fn cleanup_landing_pad(&mut self, pers_fn: Self::Value) -> (Self::Value, Self::Value);
-    fn filter_landing_pad(&mut self, pers_fn: Self::Value) -> (Self::Value, Self::Value);
+    fn cleanup_landing_pad(&mut self, pers_fn: Self::Function) -> (Self::Value, Self::Value);
+    fn filter_landing_pad(&mut self, pers_fn: Self::Function) -> (Self::Value, Self::Value);
     fn resume(&mut self, exn0: Self::Value, exn1: Self::Value);
 
     // These are used only by msvc

compiler/rustc_codegen_ssa/src/traits/declare.rs

@@ -4,14 +4,14 @@
 
 pub trait PreDefineCodegenMethods<'tcx> {
     fn predefine_static(
-        &self,
+        &mut self,
         def_id: DefId,
         linkage: Linkage,
         visibility: Visibility,
         symbol_name: &str,
     );
     fn predefine_fn(
-        &self,
+        &mut self,
         instance: Instance<'tcx>,
         linkage: Linkage,
         visibility: Visibility,

compiler/rustc_codegen_ssa/src/traits/misc.rs

@@ -1,7 +1,6 @@
 use std::cell::RefCell;
 
 use rustc_data_structures::fx::FxHashMap;
-use rustc_middle::mir::mono::CodegenUnit;
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_session::Session;
 
@@ -20,9 +19,8 @@ fn apply_vcall_visibility_metadata(
     }
     fn get_fn(&self, instance: Instance<'tcx>) -> Self::Function;
     fn get_fn_addr(&self, instance: Instance<'tcx>) -> Self::Value;
-    fn eh_personality(&self) -> Self::Value;
+    fn eh_personality(&self) -> Self::Function;
     fn sess(&self) -> &Session;
-    fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx>;
     fn set_frame_pointer_type(&self, llfn: Self::Function);
     fn apply_target_cpu_attr(&self, llfn: Self::Function);
     /// Declares the extern "C" main function for the entry point. Returns None if the symbol

compiler/rustc_codegen_ssa/src/traits/mod.rs

@@ -50,7 +50,7 @@
 };
 pub use self::write::{ModuleBufferMethods, ThinBufferMethods, WriteBackendMethods};
 
-pub trait CodegenObject = Copy + PartialEq + fmt::Debug;
+pub trait CodegenObject = Copy + fmt::Debug;
 
 pub trait CodegenMethods<'tcx> = LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>
     + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>>

compiler/rustc_codegen_ssa/src/traits/statics.rs

@@ -5,19 +5,7 @@
 
 pub trait StaticCodegenMethods: BackendTypes {
     fn static_addr_of(&self, cv: Self::Value, align: Align, kind: Option<&str>) -> Self::Value;
-    fn codegen_static(&self, def_id: DefId);
-
-    /// Mark the given global value as "used", to prevent the compiler and linker from potentially
-    /// removing a static variable that may otherwise appear unused.
-    fn add_used_global(&self, global: Self::Value);
-
-    /// Same as add_used_global(), but only prevent the compiler from potentially removing an
-    /// otherwise unused symbol. The linker is still permitted to drop it.
-    ///
-    /// This corresponds to the documented semantics of the `#[used]` attribute, although
-    /// on some targets (non-ELF), we may use `add_used_global` for `#[used]` statics
-    /// instead.
-    fn add_compiler_used_global(&self, global: Self::Value);
+    fn codegen_static(&mut self, def_id: DefId);
 }
 
 pub trait StaticBuilderMethods: BackendTypes {

compiler/rustc_codegen_ssa/src/traits/type_.rs

@@ -158,7 +158,6 @@ fn store_arg(
         val: Self::Value,
         dst: PlaceRef<'tcx, Self::Value>,
     );
-    fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> Self::Type;
 }
 
 pub trait TypeCodegenMethods<'tcx> = DerivedTypeCodegenMethods<'tcx>

compiler/rustc_codegen_ssa/src/traits/write.rs

@@ -6,7 +6,7 @@
 use crate::back::write::{CodegenContext, FatLtoInput, ModuleConfig};
 use crate::{CompiledModule, ModuleCodegen};
 
-pub trait WriteBackendMethods: 'static + Sized + Clone {
+pub trait WriteBackendMethods: Clone + 'static {
     type Module: Send + Sync;
     type TargetMachine;
     type TargetMachineError;
@@ -37,7 +37,7 @@ fn run_thin_lto(
     ) -> Result<(Vec<LtoModuleCodegen<Self>>, Vec<WorkProduct>), FatalError>;
     fn print_pass_timings(&self);
     fn print_statistics(&self);
-    unsafe fn optimize(
+    fn optimize(
         cgcx: &CodegenContext<Self>,
         dcx: DiagCtxtHandle<'_>,
         module: &mut ModuleCodegen<Self::Module>,
@@ -47,11 +47,11 @@ fn optimize_fat(
         cgcx: &CodegenContext<Self>,
         llmod: &mut ModuleCodegen<Self::Module>,
     ) -> Result<(), FatalError>;
-    unsafe fn optimize_thin(
+    fn optimize_thin(
         cgcx: &CodegenContext<Self>,
         thin: ThinModule<Self>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError>;
-    unsafe fn codegen(
+    fn codegen(
         cgcx: &CodegenContext<Self>,
         dcx: DiagCtxtHandle<'_>,
         module: ModuleCodegen<Self::Module>,

compiler/rustc_expand/Cargo.toml

@@ -23,6 +23,9 @@ rustc_lexer = { path = "../rustc_lexer" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_parse = { path = "../rustc_parse" }
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+rustc_proc_macro = { path = "../rustc_proc_macro" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }

compiler/rustc_expand/src/config.rs

@@ -2,7 +2,6 @@
 
 use std::iter;
 
-use rustc_ast::ptr::P;
 use rustc_ast::token::{Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{
     AttrTokenStream, AttrTokenTree, LazyAttrTokenStream, Spacing, TokenTree,
@@ -433,7 +432,7 @@ pub(crate) fn maybe_emit_expr_attr_err(&self, attr: &Attribute) {
     }
 
     #[instrument(level = "trace", skip(self))]
-    pub fn configure_expr(&self, expr: &mut P<ast::Expr>, method_receiver: bool) {
+    pub fn configure_expr(&self, expr: &mut ast::Expr, method_receiver: bool) {
         if !method_receiver {
             for attr in expr.attrs.iter() {
                 self.maybe_emit_expr_attr_err(attr);

compiler/rustc_expand/src/expand.rs

@@ -3,15 +3,14 @@
 use std::sync::Arc;
 use std::{iter, mem};
 
-use rustc_ast as ast;
 use rustc_ast::mut_visit::*;
 use rustc_ast::ptr::P;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{self, AssocCtxt, Visitor, VisitorResult, try_visit, walk_list};
 use rustc_ast::{
-    AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind, ForeignItemKind,
-    HasAttrs, HasNodeId, Inline, ItemKind, MacStmtStyle, MetaItemInner, MetaItemKind, ModKind,
-    NodeId, PatKind, StmtKind, TyKind, token,
+    self as ast, AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, DUMMY_NODE_ID,
+    ExprKind, ForeignItemKind, HasAttrs, HasNodeId, Inline, ItemKind, MacStmtStyle, MetaItemInner,
+    MetaItemKind, ModKind, NodeId, PatKind, StmtKind, TyKind, token,
 };
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
@@ -131,13 +130,9 @@ fn make_ast<T: InvocationCollectorNode>(self) -> T::OutputTy {
             pub(crate) fn mut_visit_with<F: MutVisitor>(&mut self, vis: &mut F) {
                 match self {
                     AstFragment::OptExpr(opt_expr) => {
-                        visit_clobber(opt_expr, |opt_expr| {
-                            if let Some(expr) = opt_expr {
-                                vis.filter_map_expr(expr)
-                            } else {
-                                None
-                            }
-                        });
+                        if let Some(expr) = opt_expr.take() {
+                            *opt_expr = vis.filter_map_expr(expr)
+                        }
                     }
                     AstFragment::MethodReceiverExpr(expr) => vis.visit_method_receiver_expr(expr),
                     $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)*
@@ -1782,11 +1777,7 @@ fn pre_flat_map_node_collect_attr(cfg: &StripUnconfigured<'_>, attr: &ast::Attri
 /// This struct is a hack to workaround unstable of `stmt_expr_attributes`.
 /// It can be removed once that feature is stabilized.
 struct MethodReceiverTag;
-impl DummyAstNode for MethodReceiverTag {
-    fn dummy() -> MethodReceiverTag {
-        MethodReceiverTag
-    }
-}
+
 impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, MethodReceiverTag> {
     type OutputTy = Self;
     const KIND: AstFragmentKind = AstFragmentKind::MethodReceiverExpr;
@@ -1852,6 +1843,57 @@ fn build_single_delegations<'a, Node: InvocationCollectorNode>(
     })
 }
 
+/// Required for `visit_node` obtained an owned `Node` from `&mut Node`.
+trait DummyAstNode {
+    fn dummy() -> Self;
+}
+
+impl DummyAstNode for ast::Crate {
+    fn dummy() -> Self {
+        ast::Crate {
+            attrs: Default::default(),
+            items: Default::default(),
+            spans: Default::default(),
+            id: DUMMY_NODE_ID,
+            is_placeholder: Default::default(),
+        }
+    }
+}
+
+impl DummyAstNode for P<ast::Ty> {
+    fn dummy() -> Self {
+        P(ast::Ty {
+            id: DUMMY_NODE_ID,
+            kind: TyKind::Dummy,
+            span: Default::default(),
+            tokens: Default::default(),
+        })
+    }
+}
+
+impl DummyAstNode for P<ast::Pat> {
+    fn dummy() -> Self {
+        P(ast::Pat {
+            id: DUMMY_NODE_ID,
+            kind: PatKind::Wild,
+            span: Default::default(),
+            tokens: Default::default(),
+        })
+    }
+}
+
+impl DummyAstNode for P<ast::Expr> {
+    fn dummy() -> Self {
+        ast::Expr::dummy()
+    }
+}
+
+impl DummyAstNode for AstNodeWrapper<P<ast::Expr>, MethodReceiverTag> {
+    fn dummy() -> Self {
+        AstNodeWrapper::new(ast::Expr::dummy(), MethodReceiverTag)
+    }
+}
+
 struct InvocationCollector<'a, 'b> {
     cx: &'a mut ExtCtxt<'b>,
     invocations: Vec<(Invocation, Option<Arc<SyntaxExtension>>)>,
@@ -2155,18 +2197,19 @@ fn visit_node<Node: InvocationCollectorNode<OutputTy = Node> + DummyAstNode>(
                         self.expand_cfg_attr(node, &attr, pos);
                         continue;
                     }
-                    _ => visit_clobber(node, |node| {
-                        self.collect_attr((attr, pos, derives), node.to_annotatable(), Node::KIND)
+                    _ => {
+                        let n = mem::replace(node, Node::dummy());
+                        *node = self
+                            .collect_attr((attr, pos, derives), n.to_annotatable(), Node::KIND)
                             .make_ast::<Node>()
-                    }),
+                    }
                 },
                 None if node.is_mac_call() => {
-                    visit_clobber(node, |node| {
-                        // Do not clobber unless it's actually a macro (uncommon case).
-                        let (mac, attrs, _) = node.take_mac_call();
-                        self.check_attributes(&attrs, &mac);
-                        self.collect_bang(mac, Node::KIND).make_ast::<Node>()
-                    })
+                    let n = mem::replace(node, Node::dummy());
+                    let (mac, attrs, _) = n.take_mac_call();
+                    self.check_attributes(&attrs, &mac);
+
+                    *node = self.collect_bang(mac, Node::KIND).make_ast::<Node>()
                 }
                 None if node.delegation().is_some() => unreachable!(),
                 None => {
@@ -2293,18 +2336,14 @@ fn visit_expr(&mut self, node: &mut P<ast::Expr>) {
     }
 
     fn visit_method_receiver_expr(&mut self, node: &mut P<ast::Expr>) {
-        visit_clobber(node, |node| {
-            let mut wrapper = AstNodeWrapper::new(node, MethodReceiverTag);
-            self.visit_node(&mut wrapper);
-            wrapper.wrapped
-        })
+        self.visit_node(AstNodeWrapper::from_mut(node, MethodReceiverTag))
     }
 
     fn filter_map_expr(&mut self, node: P<ast::Expr>) -> Option<P<ast::Expr>> {
         self.flat_map_node(AstNodeWrapper::new(node, OptExprTag))
     }
 
-    fn visit_block(&mut self, node: &mut P<ast::Block>) {
+    fn visit_block(&mut self, node: &mut ast::Block) {
         let orig_dir_ownership = mem::replace(
             &mut self.cx.current_expansion.dir_ownership,
             DirOwnership::UnownedViaBlock,

compiler/rustc_expand/src/lib.rs

@@ -14,8 +14,6 @@
 #![feature(yeet_expr)]
 // tidy-alphabetical-end
 
-extern crate proc_macro as pm;
-
 mod build;
 mod errors;
 // FIXME(Nilstrieb) Translate macro_rules diagnostics

compiler/rustc_expand/src/proc_macro.rs

@@ -1,4 +1,3 @@
-use rustc_ast as ast;
 use rustc_ast::ptr::P;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_errors::ErrorGuaranteed;
@@ -6,6 +5,7 @@
 use rustc_session::config::ProcMacroExecutionStrategy;
 use rustc_span::Span;
 use rustc_span::profiling::SpannedEventArgRecorder;
+use {rustc_ast as ast, rustc_proc_macro as pm};
 
 use crate::base::{self, *};
 use crate::{errors, proc_macro_server};

compiler/rustc_expand/src/proc_macro_server.rs

@@ -1,10 +1,6 @@
 use std::ops::{Bound, Range};
 
 use ast::token::IdentIsRaw;
-use pm::bridge::{
-    DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, server,
-};
-use pm::{Delimiter, Level};
 use rustc_ast as ast;
 use rustc_ast::token;
 use rustc_ast::tokenstream::{self, DelimSpacing, Spacing, TokenStream};
@@ -15,6 +11,10 @@
 use rustc_parse::lexer::nfc_normalize;
 use rustc_parse::parser::Parser;
 use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
+use rustc_proc_macro::bridge::{
+    DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, server,
+};
+use rustc_proc_macro::{Delimiter, Level};
 use rustc_session::parse::ParseSess;
 use rustc_span::def_id::CrateNum;
 use rustc_span::{BytePos, FileName, Pos, Span, Symbol, sym};
@@ -66,7 +66,7 @@ fn from_internal(kind: token::LitKind) -> Self {
             token::CStr => LitKind::CStr,
             token::CStrRaw(n) => LitKind::CStrRaw(n),
             token::Err(_guar) => {
-                // This is the only place a `pm::bridge::LitKind::ErrWithGuar`
+                // This is the only place a `rustc_proc_macro::bridge::LitKind::ErrWithGuar`
                 // is constructed. Note that an `ErrorGuaranteed` is available,
                 // as required. See the comment in `to_internal`.
                 LitKind::ErrWithGuar
@@ -149,7 +149,7 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
                     }
 
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::from_internal(delim),
+                        delimiter: rustc_proc_macro::Delimiter::from_internal(delim),
                         stream: Some(stream),
                         span: DelimSpan {
                             open: span.open,
@@ -270,7 +270,7 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
                     let stream =
                         TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span);
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::None,
+                        delimiter: rustc_proc_macro::Delimiter::None,
                         stream: Some(stream),
                         span: DelimSpan::from_single(span),
                     }))
@@ -302,7 +302,7 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
                         trees.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span }));
                     }
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::Bracket,
+                        delimiter: rustc_proc_macro::Delimiter::Bracket,
                         stream: Some(stream),
                         span: DelimSpan::from_single(span),
                     }));

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -204,24 +204,25 @@ pub(crate) fn check_intrinsic_type(
 
         // Each atomic op has variants with different suffixes (`_seq_cst`, `_acquire`, etc.). Use
         // string ops to strip the suffixes, because the variants all get the same treatment here.
-        let (n_tps, inputs, output) = match split[1] {
+        let (n_tps, n_cts, inputs, output) = match split[1] {
             "cxchg" | "cxchgweak" => (
                 1,
+                0,
                 vec![Ty::new_mut_ptr(tcx, param(0)), param(0), param(0)],
                 Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
             ),
-            "load" => (1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
-            "store" => (1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit),
+            "load" => (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
+            "store" => (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit),
 
             "xchg" | "xadd" | "xsub" | "and" | "nand" | "or" | "xor" | "max" | "min" | "umax"
-            | "umin" => (1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
-            "fence" | "singlethreadfence" => (0, Vec::new(), tcx.types.unit),
+            | "umin" => (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
+            "fence" | "singlethreadfence" => (0, 0, Vec::new(), tcx.types.unit),
             op => {
                 tcx.dcx().emit_err(UnrecognizedAtomicOperation { span, op });
                 return;
             }
         };
-        (n_tps, 0, 0, inputs, output, hir::Safety::Unsafe)
+        (n_tps, 0, n_cts, inputs, output, hir::Safety::Unsafe)
     } else if intrinsic_name == sym::contract_check_ensures {
         // contract_check_ensures::<Ret, C>(Ret, C) -> Ret
         // where C: for<'a> Fn(&'a Ret) -> bool,

compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs

@@ -81,7 +81,7 @@ pub(crate) struct FnCtxt<'a, 'tcx> {
     /// you get indicates whether any subexpression that was
     /// evaluating up to and including `X` diverged.
     ///
-    /// We currently use this flag only for diagnostic purposes:
+    /// We currently use this flag for the following purposes:
     ///
     /// - To warn about unreachable code: if, after processing a
     ///   sub-expression but before we have applied the effects of the
@@ -94,6 +94,8 @@ pub(crate) struct FnCtxt<'a, 'tcx> {
     ///   warning. This corresponds to something like `{return;
     ///   foo();}` or `{return; 22}`, where we would warn on the
     ///   `foo()` or `22`.
+    /// - To assign the `!` type to block expressions with diverging
+    ///   statements.
     ///
     /// An expression represents dead code if, after checking it,
     /// the diverges flag is set to something other than `Maybe`.

compiler/rustc_lint/messages.ftl

@@ -960,7 +960,8 @@ lint_unused_doc_comment = unused doc comment
     .help = to document an item produced by a macro, the macro must produce the documentation as part of its expansion
 
 lint_unused_extern_crate = unused extern crate
-    .suggestion = remove it
+    .label = unused
+    .suggestion = remove the unused `extern crate`
 
 lint_unused_import_braces = braces around {$node} is unnecessary
 

compiler/rustc_lint/src/early/diagnostics.rs

@@ -187,6 +187,27 @@ pub fn decorate_builtin_lint(
                 lints::ReservedMultihash { suggestion }.decorate_lint(diag);
             }
         }
+        BuiltinLintDiag::HiddenUnicodeCodepoints {
+            label,
+            count,
+            span_label,
+            labels,
+            escape,
+            spans,
+        } => {
+            lints::HiddenUnicodeCodepointsDiag {
+                label: &label,
+                count,
+                span_label,
+                labels: labels.map(|spans| lints::HiddenUnicodeCodepointsDiagLabels { spans }),
+                sub: if escape {
+                    lints::HiddenUnicodeCodepointsDiagSub::Escape { spans }
+                } else {
+                    lints::HiddenUnicodeCodepointsDiagSub::NoEscape { spans }
+                },
+            }
+            .decorate_lint(diag);
+        }
         BuiltinLintDiag::UnusedBuiltinAttribute { attr_name, macro_name, invoc_span } => {
             lints::UnusedBuiltinAttribute { invoc_span, attr_name, macro_name }.decorate_lint(diag);
         }
@@ -292,8 +313,8 @@ pub fn decorate_builtin_lint(
         BuiltinLintDiag::ByteSliceInPackedStructWithDerive { ty } => {
             lints::ByteSliceInPackedStructWithDerive { ty }.decorate_lint(diag);
         }
-        BuiltinLintDiag::UnusedExternCrate { removal_span } => {
-            lints::UnusedExternCrate { removal_span }.decorate_lint(diag);
+        BuiltinLintDiag::UnusedExternCrate { span, removal_span } => {
+            lints::UnusedExternCrate { span, removal_span }.decorate_lint(diag);
         }
         BuiltinLintDiag::ExternCrateNotIdiomatic { vis_span, ident_span } => {
             let suggestion_span = vis_span.between(ident_span);

compiler/rustc_lint/src/hidden_unicode_codepoints.rs

@@ -1,136 +0,0 @@
-use ast::util::unicode::{TEXT_FLOW_CONTROL_CHARS, contains_text_flow_control_chars};
-use rustc_ast as ast;
-use rustc_session::{declare_lint, declare_lint_pass};
-use rustc_span::{BytePos, Span, Symbol};
-
-use crate::lints::{
-    HiddenUnicodeCodepointsDiag, HiddenUnicodeCodepointsDiagLabels, HiddenUnicodeCodepointsDiagSub,
-};
-use crate::{EarlyContext, EarlyLintPass, LintContext};
-
-declare_lint! {
-    #[allow(text_direction_codepoint_in_literal)]
-    /// The `text_direction_codepoint_in_literal` lint detects Unicode codepoints that change the
-    /// visual representation of text on screen in a way that does not correspond to their on
-    /// memory representation.
-    ///
-    /// ### Explanation
-    ///
-    /// The unicode characters `\u{202A}`, `\u{202B}`, `\u{202D}`, `\u{202E}`, `\u{2066}`,
-    /// `\u{2067}`, `\u{2068}`, `\u{202C}` and `\u{2069}` make the flow of text on screen change
-    /// its direction on software that supports these codepoints. This makes the text "abc" display
-    /// as "cba" on screen. By leveraging software that supports these, people can write specially
-    /// crafted literals that make the surrounding code seem like it's performing one action, when
-    /// in reality it is performing another. Because of this, we proactively lint against their
-    /// presence to avoid surprises.
-    ///
-    /// ### Example
-    ///
-    /// ```rust,compile_fail
-    /// #![deny(text_direction_codepoint_in_literal)]
-    /// fn main() {
-    ///     println!("{:?}", '‮');
-    /// }
-    /// ```
-    ///
-    /// {{produces}}
-    ///
-    pub TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
-    Deny,
-    "detect special Unicode codepoints that affect the visual representation of text on screen, \
-     changing the direction in which text flows",
-}
-
-declare_lint_pass!(HiddenUnicodeCodepoints => [TEXT_DIRECTION_CODEPOINT_IN_LITERAL]);
-
-impl HiddenUnicodeCodepoints {
-    fn lint_text_direction_codepoint(
-        &self,
-        cx: &EarlyContext<'_>,
-        text: Symbol,
-        span: Span,
-        padding: u32,
-        point_at_inner_spans: bool,
-        label: &str,
-    ) {
-        // Obtain the `Span`s for each of the forbidden chars.
-        let spans: Vec<_> = text
-            .as_str()
-            .char_indices()
-            .filter_map(|(i, c)| {
-                TEXT_FLOW_CONTROL_CHARS.contains(&c).then(|| {
-                    let lo = span.lo() + BytePos(i as u32 + padding);
-                    (c, span.with_lo(lo).with_hi(lo + BytePos(c.len_utf8() as u32)))
-                })
-            })
-            .collect();
-
-        let count = spans.len();
-        let labels = point_at_inner_spans
-            .then_some(HiddenUnicodeCodepointsDiagLabels { spans: spans.clone() });
-        let sub = if point_at_inner_spans && !spans.is_empty() {
-            HiddenUnicodeCodepointsDiagSub::Escape { spans }
-        } else {
-            HiddenUnicodeCodepointsDiagSub::NoEscape { spans }
-        };
-
-        cx.emit_span_lint(
-            TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
-            span,
-            HiddenUnicodeCodepointsDiag { label, count, span_label: span, labels, sub },
-        );
-    }
-
-    fn check_literal(
-        &mut self,
-        cx: &EarlyContext<'_>,
-        text: Symbol,
-        lit_kind: ast::token::LitKind,
-        span: Span,
-        label: &'static str,
-    ) {
-        if !contains_text_flow_control_chars(text.as_str()) {
-            return;
-        }
-        let (padding, point_at_inner_spans) = match lit_kind {
-            // account for `"` or `'`
-            ast::token::LitKind::Str | ast::token::LitKind::Char => (1, true),
-            // account for `c"`
-            ast::token::LitKind::CStr => (2, true),
-            // account for `r###"`
-            ast::token::LitKind::StrRaw(n) => (n as u32 + 2, true),
-            // account for `cr###"`
-            ast::token::LitKind::CStrRaw(n) => (n as u32 + 3, true),
-            // suppress bad literals.
-            ast::token::LitKind::Err(_) => return,
-            // Be conservative just in case new literals do support these.
-            _ => (0, false),
-        };
-        self.lint_text_direction_codepoint(cx, text, span, padding, point_at_inner_spans, label);
-    }
-}
-
-impl EarlyLintPass for HiddenUnicodeCodepoints {
-    fn check_attribute(&mut self, cx: &EarlyContext<'_>, attr: &ast::Attribute) {
-        if let ast::AttrKind::DocComment(_, comment) = attr.kind {
-            if contains_text_flow_control_chars(comment.as_str()) {
-                self.lint_text_direction_codepoint(cx, comment, attr.span, 0, false, "doc comment");
-            }
-        }
-    }
-
-    #[inline]
-    fn check_expr(&mut self, cx: &EarlyContext<'_>, expr: &ast::Expr) {
-        // byte strings are already handled well enough by `EscapeError::NonAsciiCharInByteString`
-        match &expr.kind {
-            ast::ExprKind::Lit(token_lit) => {
-                self.check_literal(cx, token_lit.symbol, token_lit.kind, expr.span, "literal");
-            }
-            ast::ExprKind::FormatArgs(args) => {
-                let (lit_kind, text) = args.uncooked_fmt_str;
-                self.check_literal(cx, text, lit_kind, args.span, "format string");
-            }
-            _ => {}
-        };
-    }
-}

compiler/rustc_lint/src/lib.rs

@@ -48,7 +48,6 @@
 mod expect;
 mod for_loops_over_fallibles;
 mod foreign_modules;
-pub mod hidden_unicode_codepoints;
 mod if_let_rescope;
 mod impl_trait_overcaptures;
 mod internal;
@@ -92,7 +91,6 @@
 use drop_forget_useless::*;
 use enum_intrinsics_non_enums::EnumIntrinsicsNonEnums;
 use for_loops_over_fallibles::*;
-use hidden_unicode_codepoints::*;
 use if_let_rescope::IfLetRescope;
 use impl_trait_overcaptures::ImplTraitOvercaptures;
 use internal::*;
@@ -177,7 +175,6 @@ fn lint_mod(tcx: TyCtxt<'_>, module_def_id: LocalModDefId) {
             DeprecatedAttr: DeprecatedAttr::default(),
             WhileTrue: WhileTrue,
             NonAsciiIdents: NonAsciiIdents,
-            HiddenUnicodeCodepoints: HiddenUnicodeCodepoints,
             IncompleteInternalFeatures: IncompleteInternalFeatures,
             RedundantSemicolons: RedundantSemicolons,
             UnusedDocComment: UnusedDocComment,

compiler/rustc_lint/src/lints.rs

@@ -3077,7 +3077,9 @@ pub(crate) struct ByteSliceInPackedStructWithDerive {
 #[derive(LintDiagnostic)]
 #[diag(lint_unused_extern_crate)]
 pub(crate) struct UnusedExternCrate {
-    #[suggestion(code = "", applicability = "machine-applicable")]
+    #[label]
+    pub span: Span,
+    #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
     pub removal_span: Span,
 }
 

compiler/rustc_lint_defs/src/builtin.rs

@@ -103,6 +103,7 @@
         TAIL_EXPR_DROP_ORDER,
         TEST_UNSTABLE_LINT,
         TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
+        TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
         TRIVIAL_CASTS,
         TRIVIAL_NUMERIC_CASTS,
         TYVAR_BEHIND_RAW_POINTER,
@@ -3782,7 +3783,6 @@
 }
 
 declare_lint! {
-    #[allow(text_direction_codepoint_in_literal)]
     /// The `text_direction_codepoint_in_comment` lint detects Unicode codepoints in comments that
     /// change the visual representation of text on screen in a way that does not correspond to
     /// their on memory representation.
@@ -3792,7 +3792,7 @@
     /// ```rust,compile_fail
     /// #![deny(text_direction_codepoint_in_comment)]
     /// fn main() {
-    ///     println!("{:?}"); // '‮');
+    #[doc = "    println!(\"{:?}\"); // '\u{202E}');"]
     /// }
     /// ```
     ///
@@ -3807,7 +3807,43 @@
     /// their use.
     pub TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
     Deny,
-    "invisible directionality-changing codepoints in comment"
+    "invisible directionality-changing codepoints in comment",
+    crate_level_only
+}
+
+declare_lint! {
+    /// The `text_direction_codepoint_in_literal` lint detects Unicode codepoints that change the
+    /// visual representation of text on screen in a way that does not correspond to their on
+    /// memory representation.
+    ///
+    /// ### Explanation
+    ///
+    /// The unicode characters `\u{202A}`, `\u{202B}`, `\u{202D}`, `\u{202E}`, `\u{2066}`,
+    /// `\u{2067}`, `\u{2068}`, `\u{202C}` and `\u{2069}` make the flow of text on screen change
+    /// its direction on software that supports these codepoints. This makes the text "abc" display
+    /// as "cba" on screen. By leveraging software that supports these, people can write specially
+    /// crafted literals that make the surrounding code seem like it's performing one action, when
+    /// in reality it is performing another. Because of this, we proactively lint against their
+    /// presence to avoid surprises.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,compile_fail
+    /// #![deny(text_direction_codepoint_in_literal)]
+    /// fn main() {
+    // ` - convince tidy that backticks match
+    #[doc = "    println!(\"{:?}\", '\u{202E}');"]
+    // `
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    pub TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
+    Deny,
+    "detect special Unicode codepoints that affect the visual representation of text on screen, \
+     changing the direction in which text flows",
+    crate_level_only
 }
 
 declare_lint! {

compiler/rustc_lint_defs/src/lib.rs

@@ -698,6 +698,14 @@ pub enum BuiltinLintDiag {
         is_string: bool,
         suggestion: Span,
     },
+    HiddenUnicodeCodepoints {
+        label: String,
+        count: usize,
+        span_label: Span,
+        labels: Option<Vec<(char, Span)>>,
+        escape: bool,
+        spans: Vec<(char, Span)>,
+    },
     TrailingMacro(bool, Ident),
     BreakWithLabelAndLoop(Span),
     UnicodeTextFlow(Span, String),
@@ -736,6 +744,7 @@ pub enum BuiltinLintDiag {
         ty: String,
     },
     UnusedExternCrate {
+        span: Span,
         removal_span: Span,
     },
     ExternCrateNotIdiomatic {

compiler/rustc_metadata/Cargo.toml

@@ -23,6 +23,9 @@ rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+rustc_proc_macro = { path = "../rustc_proc_macro" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }

compiler/rustc_metadata/src/creader.rs

@@ -7,7 +7,6 @@
 use std::time::Duration;
 use std::{cmp, env, iter};
 
-use proc_macro::bridge::client::ProcMacro;
 use rustc_ast::expand::allocator::{AllocatorKind, alloc_error_handler_name, global_fn_name};
 use rustc_ast::{self as ast, *};
 use rustc_data_structures::fx::FxHashSet;
@@ -23,6 +22,7 @@
 use rustc_index::IndexVec;
 use rustc_middle::bug;
 use rustc_middle::ty::{TyCtxt, TyCtxtFeed};
+use rustc_proc_macro::bridge::client::ProcMacro;
 use rustc_session::config::{
     self, CrateType, ExtendedTargetModifierInfo, ExternLocation, OptionsTargetModifiers,
     TargetModifier,

compiler/rustc_metadata/src/lib.rs

@@ -16,8 +16,6 @@
 #![feature(trusted_len)]
 // tidy-alphabetical-end
 
-extern crate proc_macro;
-
 pub use rmeta::provide;
 
 mod dependency_format;

compiler/rustc_metadata/src/locator.rs

@@ -435,7 +435,7 @@ fn find_library_crate(
                         info!("lib candidate: {}", spf.path.display());
 
                         let (rlibs, rmetas, dylibs, interfaces) =
-                            candidates.entry(hash.to_string()).or_default();
+                            candidates.entry(hash).or_default();
                         {
                             // As a perforamnce optimisation we canonicalize the path and skip
                             // ones we've already seeen. This allows us to ignore crates

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -6,7 +6,6 @@
 use std::{io, iter, mem};
 
 pub(super) use cstore_impl::provide;
-use proc_macro::bridge::client::ProcMacro;
 use rustc_ast as ast;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxIndexMap;
@@ -26,6 +25,7 @@
 use rustc_middle::ty::Visibility;
 use rustc_middle::ty::codec::TyDecoder;
 use rustc_middle::{bug, implement_ty_decoder};
+use rustc_proc_macro::bridge::client::ProcMacro;
 use rustc_serialize::opaque::MemDecoder;
 use rustc_serialize::{Decodable, Decoder};
 use rustc_session::Session;

compiler/rustc_middle/src/mir/statement.rs

@@ -88,26 +88,31 @@ pub fn from_ty(ty: Ty<'tcx>) -> PlaceTy<'tcx> {
     ///
     /// Note that the resulting type has not been normalized.
     #[instrument(level = "debug", skip(tcx), ret)]
-    pub fn field_ty(self, tcx: TyCtxt<'tcx>, f: FieldIdx) -> Ty<'tcx> {
-        if let Some(variant_index) = self.variant_index {
-            match *self.ty.kind() {
+    pub fn field_ty(
+        tcx: TyCtxt<'tcx>,
+        self_ty: Ty<'tcx>,
+        variant_idx: Option<VariantIdx>,
+        f: FieldIdx,
+    ) -> Ty<'tcx> {
+        if let Some(variant_index) = variant_idx {
+            match *self_ty.kind() {
                 ty::Adt(adt_def, args) if adt_def.is_enum() => {
                     adt_def.variant(variant_index).fields[f].ty(tcx, args)
                 }
                 ty::Coroutine(def_id, args) => {
                     let mut variants = args.as_coroutine().state_tys(def_id, tcx);
                     let Some(mut variant) = variants.nth(variant_index.into()) else {
-                        bug!("variant {variant_index:?} of coroutine out of range: {self:?}");
+                        bug!("variant {variant_index:?} of coroutine out of range: {self_ty:?}");
                     };
 
-                    variant
-                        .nth(f.index())
-                        .unwrap_or_else(|| bug!("field {f:?} out of range: {self:?}"))
+                    variant.nth(f.index()).unwrap_or_else(|| {
+                        bug!("field {f:?} out of range of variant: {self_ty:?} {variant_idx:?}")
+                    })
                 }
-                _ => bug!("can't downcast non-adt non-coroutine type: {self:?}"),
+                _ => bug!("can't downcast non-adt non-coroutine type: {self_ty:?}"),
             }
         } else {
-            match self.ty.kind() {
+            match self_ty.kind() {
                 ty::Adt(adt_def, args) if !adt_def.is_enum() => {
                     adt_def.non_enum_variant().fields[f].ty(tcx, args)
                 }
@@ -116,26 +121,25 @@ pub fn field_ty(self, tcx: TyCtxt<'tcx>, f: FieldIdx) -> Ty<'tcx> {
                     .upvar_tys()
                     .get(f.index())
                     .copied()
-                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self:?}")),
+                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")),
                 ty::CoroutineClosure(_, args) => args
                     .as_coroutine_closure()
                     .upvar_tys()
                     .get(f.index())
                     .copied()
-                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self:?}")),
+                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")),
                 // Only prefix fields (upvars and current state) are
                 // accessible without a variant index.
-                ty::Coroutine(_, args) => args
-                    .as_coroutine()
-                    .prefix_tys()
-                    .get(f.index())
-                    .copied()
-                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self:?}")),
+                ty::Coroutine(_, args) => {
+                    args.as_coroutine().prefix_tys().get(f.index()).copied().unwrap_or_else(|| {
+                        bug!("field {f:?} out of range of prefixes for {self_ty}")
+                    })
+                }
                 ty::Tuple(tys) => tys
                     .get(f.index())
                     .copied()
-                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self:?}")),
-                _ => bug!("can't project out of {self:?}"),
+                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")),
+                _ => bug!("can't project out of {self_ty:?}"),
             }
         }
     }
@@ -148,11 +152,11 @@ pub fn multi_projection_ty(
         elems.iter().fold(self, |place_ty, &elem| place_ty.projection_ty(tcx, elem))
     }
 
-    /// Convenience wrapper around `projection_ty_core` for
-    /// `PlaceElem`, where we can just use the `Ty` that is already
-    /// stored inline on field projection elems.
+    /// Convenience wrapper around `projection_ty_core` for `PlaceElem`,
+    /// where we can just use the `Ty` that is already stored inline on
+    /// field projection elems.
     pub fn projection_ty(self, tcx: TyCtxt<'tcx>, elem: PlaceElem<'tcx>) -> PlaceTy<'tcx> {
-        self.projection_ty_core(tcx, &elem, |_, _, ty| ty, |_, ty| ty)
+        self.projection_ty_core(tcx, &elem, |ty| ty, |_, _, _, ty| ty, |ty| ty)
     }
 
     /// `place_ty.projection_ty_core(tcx, elem, |...| { ... })`
@@ -164,8 +168,9 @@ pub fn projection_ty_core<V, T>(
         self,
         tcx: TyCtxt<'tcx>,
         elem: &ProjectionElem<V, T>,
-        mut handle_field: impl FnMut(&Self, FieldIdx, T) -> Ty<'tcx>,
-        mut handle_opaque_cast_and_subtype: impl FnMut(&Self, T) -> Ty<'tcx>,
+        mut structurally_normalize: impl FnMut(Ty<'tcx>) -> Ty<'tcx>,
+        mut handle_field: impl FnMut(Ty<'tcx>, Option<VariantIdx>, FieldIdx, T) -> Ty<'tcx>,
+        mut handle_opaque_cast_and_subtype: impl FnMut(T) -> Ty<'tcx>,
     ) -> PlaceTy<'tcx>
     where
         V: ::std::fmt::Debug,
@@ -176,16 +181,16 @@ pub fn projection_ty_core<V, T>(
         }
         let answer = match *elem {
             ProjectionElem::Deref => {
-                let ty = self.ty.builtin_deref(true).unwrap_or_else(|| {
+                let ty = structurally_normalize(self.ty).builtin_deref(true).unwrap_or_else(|| {
                     bug!("deref projection of non-dereferenceable ty {:?}", self)
                 });
                 PlaceTy::from_ty(ty)
             }
             ProjectionElem::Index(_) | ProjectionElem::ConstantIndex { .. } => {
-                PlaceTy::from_ty(self.ty.builtin_index().unwrap())
+                PlaceTy::from_ty(structurally_normalize(self.ty).builtin_index().unwrap())
             }
             ProjectionElem::Subslice { from, to, from_end } => {
-                PlaceTy::from_ty(match self.ty.kind() {
+                PlaceTy::from_ty(match structurally_normalize(self.ty).kind() {
                     ty::Slice(..) => self.ty,
                     ty::Array(inner, _) if !from_end => Ty::new_array(tcx, *inner, to - from),
                     ty::Array(inner, size) if from_end => {
@@ -201,17 +206,18 @@ pub fn projection_ty_core<V, T>(
             ProjectionElem::Downcast(_name, index) => {
                 PlaceTy { ty: self.ty, variant_index: Some(index) }
             }
-            ProjectionElem::Field(f, fty) => PlaceTy::from_ty(handle_field(&self, f, fty)),
-            ProjectionElem::OpaqueCast(ty) => {
-                PlaceTy::from_ty(handle_opaque_cast_and_subtype(&self, ty))
-            }
-            ProjectionElem::Subtype(ty) => {
-                PlaceTy::from_ty(handle_opaque_cast_and_subtype(&self, ty))
-            }
+            ProjectionElem::Field(f, fty) => PlaceTy::from_ty(handle_field(
+                structurally_normalize(self.ty),
+                self.variant_index,
+                f,
+                fty,
+            )),
+            ProjectionElem::OpaqueCast(ty) => PlaceTy::from_ty(handle_opaque_cast_and_subtype(ty)),
+            ProjectionElem::Subtype(ty) => PlaceTy::from_ty(handle_opaque_cast_and_subtype(ty)),
 
             // FIXME(unsafe_binders): Rename `handle_opaque_cast_and_subtype` to be more general.
             ProjectionElem::UnwrapUnsafeBinder(ty) => {
-                PlaceTy::from_ty(handle_opaque_cast_and_subtype(&self, ty))
+                PlaceTy::from_ty(handle_opaque_cast_and_subtype(ty))
             }
         };
         debug!("projection_ty self: {:?} elem: {:?} yields: {:?}", self, elem, answer);

compiler/rustc_middle/src/query/mod.rs

@@ -1137,7 +1137,7 @@
     /// their respective impl (i.e., part of the derive macro)
     query live_symbols_and_ignored_derived_traits(_: ()) -> &'tcx (
         LocalDefIdSet,
-        LocalDefIdMap<Vec<(DefId, DefId)>>
+        LocalDefIdMap<FxIndexSet<(DefId, DefId)>>
     ) {
         arena_cache
         desc { "finding live symbols in crate" }

compiler/rustc_middle/src/ty/consts/int.rs

@@ -26,6 +26,19 @@ pub fn new(int: ScalarInt, signed: bool, is_ptr_sized_integral: bool) -> Self {
     }
 }
 
+/// An enum to represent the compiler-side view of `intrinsics::AtomicOrdering`.
+/// This lives here because there's a method in this file that needs it and it is entirely unclear
+/// where else to put this...
+#[derive(Debug, Copy, Clone)]
+pub enum AtomicOrdering {
+    // These values must match `intrinsics::AtomicOrdering`!
+    Relaxed = 0,
+    Release = 1,
+    Acquire = 2,
+    AcqRel = 3,
+    SeqCst = 4,
+}
+
 impl std::fmt::Debug for ConstInt {
     fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let Self { int, signed, is_ptr_sized_integral } = *self;
@@ -318,6 +331,25 @@ pub fn to_target_usize(&self, tcx: TyCtxt<'_>) -> u64 {
         self.to_uint(tcx.data_layout.pointer_size).try_into().unwrap()
     }
 
+    #[inline]
+    pub fn to_atomic_ordering(self) -> AtomicOrdering {
+        use AtomicOrdering::*;
+        let val = self.to_u32();
+        if val == Relaxed as u32 {
+            Relaxed
+        } else if val == Release as u32 {
+            Release
+        } else if val == Acquire as u32 {
+            Acquire
+        } else if val == AcqRel as u32 {
+            AcqRel
+        } else if val == SeqCst as u32 {
+            SeqCst
+        } else {
+            panic!("not a valid atomic ordering")
+        }
+    }
+
     /// Converts the `ScalarInt` to `bool`.
     /// Panics if the `size` of the `ScalarInt` is not equal to 1 byte.
     /// Errors if it is not a valid `bool`.
@@ -488,7 +520,7 @@ fn from(int: ScalarInt) -> Self {
 impl From<std::cmp::Ordering> for ScalarInt {
     #[inline]
     fn from(c: std::cmp::Ordering) -> Self {
-        // Here we rely on `Ordering` having the same values in host and target!
+        // Here we rely on `cmp::Ordering` having the same values in host and target!
         ScalarInt::from(c as i8)
     }
 }

compiler/rustc_middle/src/ty/mod.rs

@@ -74,8 +74,8 @@
     place_to_string_for_capture,
 };
 pub use self::consts::{
-    AnonConstKind, Const, ConstInt, ConstKind, Expr, ExprKind, ScalarInt, UnevaluatedConst,
-    ValTree, ValTreeKind, Value,
+    AnonConstKind, AtomicOrdering, Const, ConstInt, ConstKind, Expr, ExprKind, ScalarInt,
+    UnevaluatedConst, ValTree, ValTreeKind, Value,
 };
 pub use self::context::{
     CtxtInterners, CurrentGcx, DeducedParamAttrs, Feed, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt,

compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs

@@ -323,9 +323,9 @@ fn parse_place(&self, expr_id: ExprId) -> PResult<Place<'tcx>> {
     fn parse_place_inner(&self, expr_id: ExprId) -> PResult<(Place<'tcx>, PlaceTy<'tcx>)> {
         let (parent, proj) = parse_by_kind!(self, expr_id, expr, "place",
             @call(mir_field, args) => {
-                let (parent, ty) = self.parse_place_inner(args[0])?;
+                let (parent, place_ty) = self.parse_place_inner(args[0])?;
                 let field = FieldIdx::from_u32(self.parse_integer_literal(args[1])? as u32);
-                let field_ty = ty.field_ty(self.tcx, field);
+                let field_ty = PlaceTy::field_ty(self.tcx, place_ty.ty, place_ty.variant_index, field);
                 let proj = PlaceElem::Field(field, field_ty);
                 let place = parent.project_deeper(&[proj], self.tcx);
                 return Ok((place, PlaceTy::from_ty(field_ty)));

compiler/rustc_mir_transform/src/match_branches.rs

@@ -284,12 +284,14 @@ fn can_cast(
     let v = match src_layout.ty.kind() {
         ty::Uint(_) => from_scalar.to_uint(src_layout.size),
         ty::Int(_) => from_scalar.to_int(src_layout.size) as u128,
-        _ => unreachable!("invalid int"),
+        // We can also transform the values of other integer representations (such as char),
+        // although this may not be practical in real-world scenarios.
+        _ => return false,
     };
     let size = match *cast_ty.kind() {
         ty::Int(t) => Integer::from_int_ty(&tcx, t).size(),
         ty::Uint(t) => Integer::from_uint_ty(&tcx, t).size(),
-        _ => unreachable!("invalid int"),
+        _ => return false,
     };
     let v = size.truncate(v);
     let cast_scalar = ScalarInt::try_from_uint(v, size).unwrap();

compiler/rustc_next_trait_solver/src/canonicalizer.rs

@@ -94,8 +94,8 @@ pub fn canonicalize_response<T: TypeFoldable<I>>(
         } else {
             value
         };
-        assert!(!value.has_infer(), "unexpected infer in {value:?}");
-        assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
+        debug_assert!(!value.has_infer(), "unexpected infer in {value:?}");
+        debug_assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
         let (max_universe, variables) = canonicalizer.finalize();
         Canonical { max_universe, variables, value }
     }
@@ -173,8 +173,8 @@ pub fn canonicalize_input<P: TypeFoldable<I>>(
 
         let value = QueryInput { goal, predefined_opaques_in_body };
 
-        assert!(!value.has_infer(), "unexpected infer in {value:?}");
-        assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
+        debug_assert!(!value.has_infer(), "unexpected infer in {value:?}");
+        debug_assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
         let (max_universe, variables) = rest_canonicalizer.finalize();
         Canonical { max_universe, variables, value }
     }
@@ -337,7 +337,7 @@ fn finalize(self) -> (ty::UniverseIndex, I::CanonicalVarKinds) {
                     first_region = false;
                     curr_compressed_uv = curr_compressed_uv.next_universe();
                 }
-                assert!(var.is_existential());
+                debug_assert!(var.is_existential());
                 *var = var.with_updated_universe(curr_compressed_uv);
             }
         }
@@ -350,7 +350,7 @@ fn cached_fold_ty(&mut self, t: I::Ty) -> I::Ty {
         let kind = match t.kind() {
             ty::Infer(i) => match i {
                 ty::TyVar(vid) => {
-                    assert_eq!(
+                    debug_assert_eq!(
                         self.delegate.opportunistic_resolve_ty_var(vid),
                         t,
                         "ty vid should have been resolved fully before canonicalization"
@@ -363,7 +363,7 @@ fn cached_fold_ty(&mut self, t: I::Ty) -> I::Ty {
                     ))
                 }
                 ty::IntVar(vid) => {
-                    assert_eq!(
+                    debug_assert_eq!(
                         self.delegate.opportunistic_resolve_int_var(vid),
                         t,
                         "ty vid should have been resolved fully before canonicalization"
@@ -371,7 +371,7 @@ fn cached_fold_ty(&mut self, t: I::Ty) -> I::Ty {
                     CanonicalVarKind::Ty(CanonicalTyVarKind::Int)
                 }
                 ty::FloatVar(vid) => {
-                    assert_eq!(
+                    debug_assert_eq!(
                         self.delegate.opportunistic_resolve_float_var(vid),
                         t,
                         "ty vid should have been resolved fully before canonicalization"
@@ -496,7 +496,7 @@ fn fold_region(&mut self, r: I::Region) -> I::Region {
             },
 
             ty::ReVar(vid) => {
-                assert_eq!(
+                debug_assert_eq!(
                     self.delegate.opportunistic_resolve_lt_var(vid),
                     r,
                     "region vid should have been resolved fully before canonicalization"
@@ -522,7 +522,8 @@ fn fold_ty(&mut self, t: I::Ty) -> I::Ty {
             ty
         } else {
             let res = self.cached_fold_ty(t);
-            assert!(self.cache.insert((self.binder_index, t), res).is_none());
+            let old = self.cache.insert((self.binder_index, t), res);
+            assert_eq!(old, None);
             res
         }
     }
@@ -531,7 +532,7 @@ fn fold_const(&mut self, c: I::Const) -> I::Const {
         let kind = match c.kind() {
             ty::ConstKind::Infer(i) => match i {
                 ty::InferConst::Var(vid) => {
-                    assert_eq!(
+                    debug_assert_eq!(
                         self.delegate.opportunistic_resolve_ct_var(vid),
                         c,
                         "const vid should have been resolved fully before canonicalization"

compiler/rustc_parse/src/lexer/mod.rs

@@ -4,7 +4,7 @@
 use rustc_ast::ast::{self, AttrStyle};
 use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Token, TokenKind};
 use rustc_ast::tokenstream::TokenStream;
-use rustc_ast::util::unicode::contains_text_flow_control_chars;
+use rustc_ast::util::unicode::{TEXT_FLOW_CONTROL_CHARS, contains_text_flow_control_chars};
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, Diag, DiagCtxtHandle, StashKey};
 use rustc_lexer::{
@@ -14,7 +14,7 @@
 use rustc_session::lint::BuiltinLintDiag;
 use rustc_session::lint::builtin::{
     RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX, RUST_2024_GUARDED_STRING_INCOMPATIBLE_SYNTAX,
-    TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
+    TEXT_DIRECTION_CODEPOINT_IN_COMMENT, TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
 };
 use rustc_session::parse::ParseSess;
 use rustc_span::{BytePos, Pos, Span, Symbol, sym};
@@ -174,6 +174,7 @@ fn next_token_from_cursor(&mut self) -> (Token, bool) {
                     // Opening delimiter of the length 3 is not included into the symbol.
                     let content_start = start + BytePos(3);
                     let content = self.str_from(content_start);
+                    self.lint_doc_comment_unicode_text_flow(start, content);
                     self.cook_doc_comment(content_start, content, CommentKind::Line, doc_style)
                 }
                 rustc_lexer::TokenKind::BlockComment { doc_style, terminated } => {
@@ -193,6 +194,7 @@ fn next_token_from_cursor(&mut self) -> (Token, bool) {
                     let content_start = start + BytePos(3);
                     let content_end = self.pos - BytePos(if terminated { 2 } else { 0 });
                     let content = self.str_from_to(content_start, content_end);
+                    self.lint_doc_comment_unicode_text_flow(start, content);
                     self.cook_doc_comment(content_start, content, CommentKind::Block, doc_style)
                 }
                 rustc_lexer::TokenKind::Frontmatter { has_invalid_preceding_whitespace, invalid_infostring } => {
@@ -287,6 +289,7 @@ fn next_token_from_cursor(&mut self) -> (Token, bool) {
                     } else {
                         None
                     };
+                    self.lint_literal_unicode_text_flow(symbol, kind, self.mk_sp(start, self.pos), "literal");
                     token::Literal(token::Lit { kind, symbol, suffix })
                 }
                 rustc_lexer::TokenKind::Lifetime { starts_with_number } => {
@@ -481,6 +484,88 @@ fn lint_unicode_text_flow(&self, start: BytePos) {
         }
     }
 
+    fn lint_doc_comment_unicode_text_flow(&mut self, start: BytePos, content: &str) {
+        if contains_text_flow_control_chars(content) {
+            self.report_text_direction_codepoint(
+                content,
+                self.mk_sp(start, self.pos),
+                0,
+                false,
+                "doc comment",
+            );
+        }
+    }
+
+    fn lint_literal_unicode_text_flow(
+        &mut self,
+        text: Symbol,
+        lit_kind: token::LitKind,
+        span: Span,
+        label: &'static str,
+    ) {
+        if !contains_text_flow_control_chars(text.as_str()) {
+            return;
+        }
+        let (padding, point_at_inner_spans) = match lit_kind {
+            // account for `"` or `'`
+            token::LitKind::Str | token::LitKind::Char => (1, true),
+            // account for `c"`
+            token::LitKind::CStr => (2, true),
+            // account for `r###"`
+            token::LitKind::StrRaw(n) => (n as u32 + 2, true),
+            // account for `cr###"`
+            token::LitKind::CStrRaw(n) => (n as u32 + 3, true),
+            // suppress bad literals.
+            token::LitKind::Err(_) => return,
+            // Be conservative just in case new literals do support these.
+            _ => (0, false),
+        };
+        self.report_text_direction_codepoint(
+            text.as_str(),
+            span,
+            padding,
+            point_at_inner_spans,
+            label,
+        );
+    }
+
+    fn report_text_direction_codepoint(
+        &self,
+        text: &str,
+        span: Span,
+        padding: u32,
+        point_at_inner_spans: bool,
+        label: &str,
+    ) {
+        // Obtain the `Span`s for each of the forbidden chars.
+        let spans: Vec<_> = text
+            .char_indices()
+            .filter_map(|(i, c)| {
+                TEXT_FLOW_CONTROL_CHARS.contains(&c).then(|| {
+                    let lo = span.lo() + BytePos(i as u32 + padding);
+                    (c, span.with_lo(lo).with_hi(lo + BytePos(c.len_utf8() as u32)))
+                })
+            })
+            .collect();
+
+        let count = spans.len();
+        let labels = point_at_inner_spans.then_some(spans.clone());
+
+        self.psess.buffer_lint(
+            TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
+            span,
+            ast::CRATE_NODE_ID,
+            BuiltinLintDiag::HiddenUnicodeCodepoints {
+                label: label.to_string(),
+                count,
+                span_label: span,
+                labels,
+                escape: point_at_inner_spans && !spans.is_empty(),
+                spans,
+            },
+        );
+    }
+
     fn validate_frontmatter(
         &self,
         start: BytePos,

compiler/rustc_parse/src/parser/expr.rs

@@ -3311,26 +3311,44 @@ pub(super) fn parse_arm(&mut self) -> PResult<'a, Arm> {
                             let sm = this.psess.source_map();
                             if let Ok(expr_lines) = sm.span_to_lines(expr_span)
                                 && let Ok(arm_start_lines) = sm.span_to_lines(arm_start_span)
-                                && arm_start_lines.lines[0].end_col == expr_lines.lines[0].end_col
                                 && expr_lines.lines.len() == 2
                             {
-                                // We check whether there's any trailing code in the parse span,
-                                // if there isn't, we very likely have the following:
-                                //
-                                // X |     &Y => "y"
-                                //   |        --    - missing comma
-                                //   |        |
-                                //   |        arrow_span
-                                // X |     &X => "x"
-                                //   |      - ^^ self.token.span
-                                //   |      |
-                                //   |      parsed until here as `"y" & X`
-                                err.span_suggestion_short(
-                                    arm_start_span.shrink_to_hi(),
-                                    "missing a comma here to end this `match` arm",
-                                    ",",
-                                    Applicability::MachineApplicable,
-                                );
+                                if arm_start_lines.lines[0].end_col == expr_lines.lines[0].end_col {
+                                    // We check whether there's any trailing code in the parse span,
+                                    // if there isn't, we very likely have the following:
+                                    //
+                                    // X |     &Y => "y"
+                                    //   |        --    - missing comma
+                                    //   |        |
+                                    //   |        arrow_span
+                                    // X |     &X => "x"
+                                    //   |      - ^^ self.token.span
+                                    //   |      |
+                                    //   |      parsed until here as `"y" & X`
+                                    err.span_suggestion_short(
+                                        arm_start_span.shrink_to_hi(),
+                                        "missing a comma here to end this `match` arm",
+                                        ",",
+                                        Applicability::MachineApplicable,
+                                    );
+                                } else if arm_start_lines.lines[0].end_col + rustc_span::CharPos(1)
+                                    == expr_lines.lines[0].end_col
+                                {
+                                    // similar to the above, but we may typo a `.` or `/` at the end of the line
+                                    let comma_span = arm_start_span
+                                        .shrink_to_hi()
+                                        .with_hi(arm_start_span.hi() + rustc_span::BytePos(1));
+                                    if let Ok(res) = sm.span_to_snippet(comma_span)
+                                        && (res == "." || res == "/")
+                                    {
+                                        err.span_suggestion_short(
+                                            comma_span,
+                                            "you might have meant to write a `,` to end this `match` arm",
+                                            ",",
+                                            Applicability::MachineApplicable,
+                                        );
+                                    }
+                                }
                             }
                         } else {
                             err.span_label(

compiler/rustc_passes/src/dead.rs

@@ -8,12 +8,13 @@
 use hir::ItemKind;
 use hir::def_id::{LocalDefIdMap, LocalDefIdSet};
 use rustc_abi::FieldIdx;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::MultiSpan;
 use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId};
 use rustc_hir::intravisit::{self, Visitor};
-use rustc_hir::{self as hir, Node, PatKind, TyKind};
+use rustc_hir::{self as hir, Node, PatKind, QPath, TyKind};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::middle::privacy::Level;
 use rustc_middle::query::Providers;
@@ -44,15 +45,20 @@ fn should_explore(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     )
 }
 
-fn ty_ref_to_pub_struct(tcx: TyCtxt<'_>, ty: &hir::Ty<'_>) -> bool {
-    if let TyKind::Path(hir::QPath::Resolved(_, path)) = ty.kind
-        && let Res::Def(def_kind, def_id) = path.res
-        && def_id.is_local()
-        && matches!(def_kind, DefKind::Struct | DefKind::Enum | DefKind::Union)
-    {
-        tcx.visibility(def_id).is_public()
-    } else {
-        true
+/// Returns the local def id of the ADT if the given ty refers to a local one.
+fn local_adt_def_of_ty<'tcx>(ty: &hir::Ty<'tcx>) -> Option<LocalDefId> {
+    match ty.kind {
+        TyKind::Path(QPath::Resolved(_, path)) => {
+            if let Res::Def(def_kind, def_id) = path.res
+                && let Some(local_def_id) = def_id.as_local()
+                && matches!(def_kind, DefKind::Struct | DefKind::Enum | DefKind::Union)
+            {
+                Some(local_def_id)
+            } else {
+                None
+            }
+        }
+        _ => None,
     }
 }
 
@@ -78,7 +84,7 @@ struct MarkSymbolVisitor<'tcx> {
     // maps from ADTs to ignored derived traits (e.g. Debug and Clone)
     // and the span of their respective impl (i.e., part of the derive
     // macro)
-    ignored_derived_traits: LocalDefIdMap<Vec<(DefId, DefId)>>,
+    ignored_derived_traits: LocalDefIdMap<FxIndexSet<(DefId, DefId)>>,
 }
 
 impl<'tcx> MarkSymbolVisitor<'tcx> {
@@ -360,7 +366,7 @@ fn should_ignore_item(&mut self, def_id: DefId) -> bool {
                 && let Some(fn_sig) =
                     self.tcx.hir_fn_sig_by_hir_id(self.tcx.local_def_id_to_hir_id(local_def_id))
                 && matches!(fn_sig.decl.implicit_self, hir::ImplicitSelfKind::None)
-                && let TyKind::Path(hir::QPath::Resolved(_, path)) =
+                && let TyKind::Path(QPath::Resolved(_, path)) =
                     self.tcx.hir_expect_item(local_impl_of).expect_impl().self_ty.kind
                 && let Res::Def(def_kind, did) = path.res
             {
@@ -388,7 +394,7 @@ fn should_ignore_item(&mut self, def_id: DefId) -> bool {
                     self.ignored_derived_traits
                         .entry(adt_def_id)
                         .or_default()
-                        .push((trait_of, impl_of));
+                        .insert((trait_of, impl_of));
                 }
                 return true;
             }
@@ -420,51 +426,22 @@ fn visit_node(&mut self, node: Node<'tcx>) {
                     intravisit::walk_item(self, item)
                 }
                 hir::ItemKind::ForeignMod { .. } => {}
-                hir::ItemKind::Trait(..) => {
-                    for &impl_def_id in self.tcx.local_trait_impls(item.owner_id.def_id) {
-                        if let ItemKind::Impl(impl_ref) = self.tcx.hir_expect_item(impl_def_id).kind
-                        {
-                            // skip items
-                            // mark dependent traits live
-                            intravisit::walk_generics(self, impl_ref.generics);
-                            // mark dependent parameters live
-                            intravisit::walk_path(self, impl_ref.of_trait.unwrap().path);
+                hir::ItemKind::Trait(.., trait_item_refs) => {
+                    // mark assoc ty live if the trait is live
+                    for trait_item in trait_item_refs {
+                        if matches!(trait_item.kind, hir::AssocItemKind::Type) {
+                            self.check_def_id(trait_item.id.owner_id.to_def_id());
                         }
                     }
-
                     intravisit::walk_item(self, item)
                 }
                 _ => intravisit::walk_item(self, item),
             },
             Node::TraitItem(trait_item) => {
-                // mark corresponding ImplTerm live
+                // mark the trait live
                 let trait_item_id = trait_item.owner_id.to_def_id();
                 if let Some(trait_id) = self.tcx.trait_of_item(trait_item_id) {
-                    // mark the trait live
                     self.check_def_id(trait_id);
-
-                    for impl_id in self.tcx.all_impls(trait_id) {
-                        if let Some(local_impl_id) = impl_id.as_local()
-                            && let ItemKind::Impl(impl_ref) =
-                                self.tcx.hir_expect_item(local_impl_id).kind
-                        {
-                            if !matches!(trait_item.kind, hir::TraitItemKind::Type(..))
-                                && !ty_ref_to_pub_struct(self.tcx, impl_ref.self_ty)
-                            {
-                                // skip methods of private ty,
-                                // they would be solved in `solve_rest_impl_items`
-                                continue;
-                            }
-
-                            // mark self_ty live
-                            intravisit::walk_unambig_ty(self, impl_ref.self_ty);
-                            if let Some(&impl_item_id) =
-                                self.tcx.impl_item_implementor_ids(impl_id).get(&trait_item_id)
-                            {
-                                self.check_def_id(impl_item_id);
-                            }
-                        }
-                    }
                 }
                 intravisit::walk_trait_item(self, trait_item);
             }
@@ -508,48 +485,58 @@ fn mark_as_used_if_union(&mut self, adt: ty::AdtDef<'tcx>, fields: &[hir::ExprFi
         }
     }
 
-    fn solve_rest_impl_items(&mut self, mut unsolved_impl_items: Vec<(hir::ItemId, LocalDefId)>) {
-        let mut ready;
-        (ready, unsolved_impl_items) =
-            unsolved_impl_items.into_iter().partition(|&(impl_id, impl_item_id)| {
-                self.impl_item_with_used_self(impl_id, impl_item_id)
-            });
-
-        while !ready.is_empty() {
-            self.worklist =
-                ready.into_iter().map(|(_, id)| (id, ComesFromAllowExpect::No)).collect();
-            self.mark_live_symbols();
-
-            (ready, unsolved_impl_items) =
-                unsolved_impl_items.into_iter().partition(|&(impl_id, impl_item_id)| {
-                    self.impl_item_with_used_self(impl_id, impl_item_id)
-                });
+    /// Returns whether `local_def_id` is potentially alive or not.
+    /// `local_def_id` points to an impl or an impl item,
+    /// both impl and impl item that may be passed to this function are of a trait,
+    /// and added into the unsolved_items during `create_and_seed_worklist`
+    fn check_impl_or_impl_item_live(
+        &mut self,
+        impl_id: hir::ItemId,
+        local_def_id: LocalDefId,
+    ) -> bool {
+        if self.should_ignore_item(local_def_id.to_def_id()) {
+            return false;
         }
-    }
 
-    fn impl_item_with_used_self(&mut self, impl_id: hir::ItemId, impl_item_id: LocalDefId) -> bool {
-        if let TyKind::Path(hir::QPath::Resolved(_, path)) =
-            self.tcx.hir_item(impl_id).expect_impl().self_ty.kind
-            && let Res::Def(def_kind, def_id) = path.res
-            && let Some(local_def_id) = def_id.as_local()
-            && matches!(def_kind, DefKind::Struct | DefKind::Enum | DefKind::Union)
-        {
-            if self.tcx.visibility(impl_item_id).is_public() {
-                // for the public method, we don't know the trait item is used or not,
-                // so we mark the method live if the self is used
-                return self.live_symbols.contains(&local_def_id);
-            }
+        let trait_def_id = match self.tcx.def_kind(local_def_id) {
+            // assoc impl items of traits are live if the corresponding trait items are live
+            DefKind::AssocFn => self.tcx.associated_item(local_def_id).trait_item_def_id,
+            // impl items are live if the corresponding traits are live
+            DefKind::Impl { of_trait: true } => self
+                .tcx
+                .impl_trait_ref(impl_id.owner_id.def_id)
+                .and_then(|trait_ref| Some(trait_ref.skip_binder().def_id)),
+            _ => None,
+        };
 
-            if let Some(trait_item_id) = self.tcx.associated_item(impl_item_id).trait_item_def_id
-                && let Some(local_id) = trait_item_id.as_local()
+        if let Some(trait_def_id) = trait_def_id {
+            if let Some(trait_def_id) = trait_def_id.as_local()
+                && !self.live_symbols.contains(&trait_def_id)
             {
-                // for the private method, we can know the trait item is used or not,
-                // so we mark the method live if the self is used and the trait item is used
-                return self.live_symbols.contains(&local_id)
-                    && self.live_symbols.contains(&local_def_id);
+                return false;
+            }
+
+            // FIXME: legacy logic to check whether the function may construct `Self`,
+            // this can be removed after supporting marking ADTs appearing in patterns
+            // as live, then we can check private impls of public traits directly
+            if let Some(fn_sig) =
+                self.tcx.hir_fn_sig_by_hir_id(self.tcx.local_def_id_to_hir_id(local_def_id))
+                && matches!(fn_sig.decl.implicit_self, hir::ImplicitSelfKind::None)
+                && self.tcx.visibility(trait_def_id).is_public()
+            {
+                return true;
             }
         }
-        false
+
+        // The impl or impl item is used if the corresponding trait or trait item is used and the ty is used.
+        if let Some(local_def_id) =
+            local_adt_def_of_ty(self.tcx.hir_item(impl_id).expect_impl().self_ty)
+            && !self.live_symbols.contains(&local_def_id)
+        {
+            return false;
+        }
+
+        true
     }
 }
 
@@ -584,7 +571,7 @@ fn visit_variant_data(&mut self, def: &'tcx hir::VariantData<'tcx>) {
 
     fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) {
         match expr.kind {
-            hir::ExprKind::Path(ref qpath @ hir::QPath::TypeRelative(..)) => {
+            hir::ExprKind::Path(ref qpath @ QPath::TypeRelative(..)) => {
                 let res = self.typeck_results().qpath_res(qpath, expr.hir_id);
                 self.handle_res(res);
             }
@@ -738,7 +725,7 @@ fn check_item<'tcx>(
     tcx: TyCtxt<'tcx>,
     worklist: &mut Vec<(LocalDefId, ComesFromAllowExpect)>,
     struct_constructors: &mut LocalDefIdMap<LocalDefId>,
-    unsolved_impl_items: &mut Vec<(hir::ItemId, LocalDefId)>,
+    unsolved_items: &mut Vec<(hir::ItemId, LocalDefId)>,
     id: hir::ItemId,
 ) {
     let allow_dead_code = has_allow_dead_code_or_lang_attr(tcx, id.owner_id.def_id);
@@ -764,41 +751,33 @@ fn check_item<'tcx>(
             }
         }
         DefKind::Impl { of_trait } => {
-            // get DefIds from another query
-            let local_def_ids = tcx
-                .associated_item_def_ids(id.owner_id)
-                .iter()
-                .filter_map(|def_id| def_id.as_local());
+            if let Some(comes_from_allow) =
+                has_allow_dead_code_or_lang_attr(tcx, id.owner_id.def_id)
+            {
+                worklist.push((id.owner_id.def_id, comes_from_allow));
+            } else if of_trait {
+                unsolved_items.push((id, id.owner_id.def_id));
+            }
 
-            let ty_is_pub = ty_ref_to_pub_struct(tcx, tcx.hir_item(id).expect_impl().self_ty);
+            for def_id in tcx.associated_item_def_ids(id.owner_id) {
+                let local_def_id = def_id.expect_local();
 
-            // And we access the Map here to get HirId from LocalDefId
-            for local_def_id in local_def_ids {
-                // check the function may construct Self
-                let mut may_construct_self = false;
-                if let Some(fn_sig) =
-                    tcx.hir_fn_sig_by_hir_id(tcx.local_def_id_to_hir_id(local_def_id))
-                {
-                    may_construct_self =
-                        matches!(fn_sig.decl.implicit_self, hir::ImplicitSelfKind::None);
-                }
-
-                // for trait impl blocks,
-                // mark the method live if the self_ty is public,
-                // or the method is public and may construct self
-                if of_trait
-                    && (!matches!(tcx.def_kind(local_def_id), DefKind::AssocFn)
-                        || tcx.visibility(local_def_id).is_public()
-                            && (ty_is_pub || may_construct_self))
-                {
-                    worklist.push((local_def_id, ComesFromAllowExpect::No));
-                } else if let Some(comes_from_allow) =
-                    has_allow_dead_code_or_lang_attr(tcx, local_def_id)
+                if let Some(comes_from_allow) = has_allow_dead_code_or_lang_attr(tcx, local_def_id)
                 {
                     worklist.push((local_def_id, comes_from_allow));
                 } else if of_trait {
-                    // private method || public method not constructs self
-                    unsolved_impl_items.push((id, local_def_id));
+                    // FIXME: This condition can be removed
+                    // if we support dead check for assoc consts and tys.
+                    if !matches!(tcx.def_kind(local_def_id), DefKind::AssocFn) {
+                        worklist.push((local_def_id, ComesFromAllowExpect::No));
+                    } else {
+                        // We only care about associated items of traits,
+                        // because they cannot be visited directly,
+                        // so we later mark them as live if their corresponding traits
+                        // or trait items and self types are both live,
+                        // but inherent associated items can be visited and marked directly.
+                        unsolved_items.push((id, local_def_id));
+                    }
                 }
             }
         }
@@ -892,8 +871,8 @@ fn create_and_seed_worklist(
 fn live_symbols_and_ignored_derived_traits(
     tcx: TyCtxt<'_>,
     (): (),
-) -> (LocalDefIdSet, LocalDefIdMap<Vec<(DefId, DefId)>>) {
-    let (worklist, struct_constructors, unsolved_impl_items) = create_and_seed_worklist(tcx);
+) -> (LocalDefIdSet, LocalDefIdMap<FxIndexSet<(DefId, DefId)>>) {
+    let (worklist, struct_constructors, mut unsolved_items) = create_and_seed_worklist(tcx);
     let mut symbol_visitor = MarkSymbolVisitor {
         worklist,
         tcx,
@@ -907,7 +886,22 @@ fn live_symbols_and_ignored_derived_traits(
         ignored_derived_traits: Default::default(),
     };
     symbol_visitor.mark_live_symbols();
-    symbol_visitor.solve_rest_impl_items(unsolved_impl_items);
+    let mut items_to_check;
+    (items_to_check, unsolved_items) =
+        unsolved_items.into_iter().partition(|&(impl_id, local_def_id)| {
+            symbol_visitor.check_impl_or_impl_item_live(impl_id, local_def_id)
+        });
+
+    while !items_to_check.is_empty() {
+        symbol_visitor.worklist =
+            items_to_check.into_iter().map(|(_, id)| (id, ComesFromAllowExpect::No)).collect();
+        symbol_visitor.mark_live_symbols();
+
+        (items_to_check, unsolved_items) =
+            unsolved_items.into_iter().partition(|&(impl_id, local_def_id)| {
+                symbol_visitor.check_impl_or_impl_item_live(impl_id, local_def_id)
+            });
+    }
 
     (symbol_visitor.live_symbols, symbol_visitor.ignored_derived_traits)
 }
@@ -921,7 +915,7 @@ struct DeadItem {
 struct DeadVisitor<'tcx> {
     tcx: TyCtxt<'tcx>,
     live_symbols: &'tcx LocalDefIdSet,
-    ignored_derived_traits: &'tcx LocalDefIdMap<Vec<(DefId, DefId)>>,
+    ignored_derived_traits: &'tcx LocalDefIdMap<FxIndexSet<(DefId, DefId)>>,
 }
 
 enum ShouldWarnAboutField {
@@ -1188,19 +1182,15 @@ fn check_mod_deathness(tcx: TyCtxt<'_>, module: LocalModDefId) {
         let def_kind = tcx.def_kind(item.owner_id);
 
         let mut dead_codes = Vec::new();
-        // if we have diagnosed the trait, do not diagnose unused methods
-        if matches!(def_kind, DefKind::Impl { .. })
+        // Only diagnose unused assoc items in inherient impl and used trait,
+        // for unused assoc items in impls of trait,
+        // we have diagnosed them in the trait if they are unused,
+        // for unused assoc items in unused trait,
+        // we have diagnosed the unused trait.
+        if matches!(def_kind, DefKind::Impl { of_trait: false })
             || (def_kind == DefKind::Trait && live_symbols.contains(&item.owner_id.def_id))
         {
             for &def_id in tcx.associated_item_def_ids(item.owner_id.def_id) {
-                // We have diagnosed unused methods in traits
-                if matches!(def_kind, DefKind::Impl { of_trait: true })
-                    && tcx.def_kind(def_id) == DefKind::AssocFn
-                    || def_kind == DefKind::Trait && tcx.def_kind(def_id) != DefKind::AssocFn
-                {
-                    continue;
-                }
-
                 if let Some(local_def_id) = def_id.as_local()
                     && !visitor.is_live_code(local_def_id)
                 {

compiler/rustc_proc_macro/Cargo.toml

@@ -0,0 +1,21 @@
+# We need to use a separate crate including library/proc_macro as opposed to a
+# direct path dependency on library/proc_macro because doing the latter will
+# cause two copies of libproc_macro.rlib to end up in the sysroot, breaking
+# proc-macro crates. In addition it confuses the workspace_members function of
+# bootstrap.
+
+[package]
+name = "rustc_proc_macro"
+version = "0.0.0"
+edition = "2024"
+
+[lib]
+path = "../../library/proc_macro/src/lib.rs"
+test = false
+doctest = false
+
+[dependencies]
+rustc-literal-escaper = "0.0.2"
+
+[features]
+rustc-dep-of-std = []

compiler/rustc_resolve/src/check_unused.rs

@@ -154,6 +154,7 @@ fn report_unused_extern_crate_items(
                         extern_crate.id,
                         span,
                         BuiltinLintDiag::UnusedExternCrate {
+                            span: extern_crate.span,
                             removal_span: extern_crate.span_with_attributes,
                         },
                     );

compiler/rustc_resolve/src/late.rs

@@ -934,8 +934,7 @@ fn visit_ty(&mut self, ty: &'ast Ty) {
                 )
             }
             TyKind::UnsafeBinder(unsafe_binder) => {
-                // FIXME(unsafe_binder): Better span
-                let span = ty.span;
+                let span = ty.span.shrink_to_lo().to(unsafe_binder.inner_ty.span.shrink_to_lo());
                 self.with_generic_param_rib(
                     &unsafe_binder.generic_params,
                     RibKind::Normal,

compiler/rustc_span/src/symbol.rs

@@ -515,6 +515,7 @@
         async_iterator_poll_next,
         async_trait_bounds,
         atomic,
+        atomic_load,
         atomic_mod,
         atomics,
         att_syntax,

compiler/rustc_target/src/spec/base/cygwin.rs

@@ -1,7 +1,8 @@
 use std::borrow::Cow;
 
 use crate::spec::{
-    BinaryFormat, Cc, DebuginfoKind, LinkerFlavor, Lld, SplitDebuginfo, TargetOptions, cvs,
+    BinaryFormat, Cc, DebuginfoKind, LinkerFlavor, Lld, SplitDebuginfo, TargetOptions, TlsModel,
+    cvs,
 };
 
 pub(crate) fn opts() -> TargetOptions {
@@ -44,6 +45,8 @@ pub(crate) fn opts() -> TargetOptions {
         eh_frame_header: false,
         debuginfo_kind: DebuginfoKind::Dwarf,
         supported_split_debuginfo: Cow::Borrowed(&[SplitDebuginfo::Off]),
+        tls_model: TlsModel::Emulated,
+        has_thread_local: true,
         ..Default::default()
     }
 }

library/alloc/src/collections/btree/map.rs

@@ -1151,7 +1151,7 @@ pub fn retain<F>(&mut self, mut f: F)
         K: Ord,
         F: FnMut(&K, &mut V) -> bool,
     {
-        self.extract_if(|k, v| !f(k, v)).for_each(drop);
+        self.extract_if(.., |k, v| !f(k, v)).for_each(drop);
     }
 
     /// Moves all elements from `other` into `self`, leaving `other` empty.
@@ -1397,7 +1397,7 @@ pub fn split_off<Q: ?Sized + Ord>(&mut self, key: &Q) -> Self
         }
     }
 
-    /// Creates an iterator that visits all elements (key-value pairs) in
+    /// Creates an iterator that visits elements (key-value pairs) in the specified range in
     /// ascending key order and uses a closure to determine if an element
     /// should be removed.
     ///
@@ -1423,33 +1423,42 @@ pub fn split_off<Q: ?Sized + Ord>(&mut self, key: &Q) -> Self
     /// use std::collections::BTreeMap;
     ///
     /// let mut map: BTreeMap<i32, i32> = (0..8).map(|x| (x, x)).collect();
-    /// let evens: BTreeMap<_, _> = map.extract_if(|k, _v| k % 2 == 0).collect();
+    /// let evens: BTreeMap<_, _> = map.extract_if(.., |k, _v| k % 2 == 0).collect();
     /// let odds = map;
     /// assert_eq!(evens.keys().copied().collect::<Vec<_>>(), [0, 2, 4, 6]);
     /// assert_eq!(odds.keys().copied().collect::<Vec<_>>(), [1, 3, 5, 7]);
+    ///
+    /// let mut map: BTreeMap<i32, i32> = (0..8).map(|x| (x, x)).collect();
+    /// let low: BTreeMap<_, _> = map.extract_if(0..4, |_k, _v| true).collect();
+    /// let high = map;
+    /// assert_eq!(low.keys().copied().collect::<Vec<_>>(), [0, 1, 2, 3]);
+    /// assert_eq!(high.keys().copied().collect::<Vec<_>>(), [4, 5, 6, 7]);
     /// ```
     #[unstable(feature = "btree_extract_if", issue = "70530")]
-    pub fn extract_if<F>(&mut self, pred: F) -> ExtractIf<'_, K, V, F, A>
+    pub fn extract_if<F, R>(&mut self, range: R, pred: F) -> ExtractIf<'_, K, V, R, F, A>
     where
         K: Ord,
+        R: RangeBounds<K>,
         F: FnMut(&K, &mut V) -> bool,
     {
-        let (inner, alloc) = self.extract_if_inner();
+        let (inner, alloc) = self.extract_if_inner(range);
         ExtractIf { pred, inner, alloc }
     }
 
-    pub(super) fn extract_if_inner(&mut self) -> (ExtractIfInner<'_, K, V>, A)
+    pub(super) fn extract_if_inner<R>(&mut self, range: R) -> (ExtractIfInner<'_, K, V, R>, A)
     where
         K: Ord,
+        R: RangeBounds<K>,
     {
         if let Some(root) = self.root.as_mut() {
             let (root, dormant_root) = DormantMutRef::new(root);
-            let front = root.borrow_mut().first_leaf_edge();
+            let first = root.borrow_mut().lower_bound(SearchBound::from_range(range.start_bound()));
             (
                 ExtractIfInner {
                     length: &mut self.length,
                     dormant_root: Some(dormant_root),
-                    cur_leaf_edge: Some(front),
+                    cur_leaf_edge: Some(first),
+                    range,
                 },
                 (*self.alloc).clone(),
             )
@@ -1459,6 +1468,7 @@ pub(super) fn extract_if_inner(&mut self) -> (ExtractIfInner<'_, K, V>, A)
                     length: &mut self.length,
                     dormant_root: None,
                     cur_leaf_edge: None,
+                    range,
                 },
                 (*self.alloc).clone(),
             )
@@ -1917,18 +1927,19 @@ pub struct ExtractIf<
     'a,
     K,
     V,
+    R,
     F,
     #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global,
 > {
     pred: F,
-    inner: ExtractIfInner<'a, K, V>,
+    inner: ExtractIfInner<'a, K, V, R>,
     /// The BTreeMap will outlive this IntoIter so we don't care about drop order for `alloc`.
     alloc: A,
 }
 
 /// Most of the implementation of ExtractIf are generic over the type
 /// of the predicate, thus also serving for BTreeSet::ExtractIf.
-pub(super) struct ExtractIfInner<'a, K, V> {
+pub(super) struct ExtractIfInner<'a, K, V, R> {
     /// Reference to the length field in the borrowed map, updated live.
     length: &'a mut usize,
     /// Buried reference to the root field in the borrowed map.
@@ -1938,10 +1949,13 @@ pub(super) struct ExtractIfInner<'a, K, V> {
     /// Empty if the map has no root, if iteration went beyond the last leaf edge,
     /// or if a panic occurred in the predicate.
     cur_leaf_edge: Option<Handle<NodeRef<marker::Mut<'a>, K, V, marker::Leaf>, marker::Edge>>,
+    /// Range over which iteration was requested.  We don't need the left side, but we
+    /// can't extract the right side without requiring K: Clone.
+    range: R,
 }
 
 #[unstable(feature = "btree_extract_if", issue = "70530")]
-impl<K, V, F, A> fmt::Debug for ExtractIf<'_, K, V, F, A>
+impl<K, V, R, F, A> fmt::Debug for ExtractIf<'_, K, V, R, F, A>
 where
     K: fmt::Debug,
     V: fmt::Debug,
@@ -1953,8 +1967,10 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 }
 
 #[unstable(feature = "btree_extract_if", issue = "70530")]
-impl<K, V, F, A: Allocator + Clone> Iterator for ExtractIf<'_, K, V, F, A>
+impl<K, V, R, F, A: Allocator + Clone> Iterator for ExtractIf<'_, K, V, R, F, A>
 where
+    K: PartialOrd,
+    R: RangeBounds<K>,
     F: FnMut(&K, &mut V) -> bool,
 {
     type Item = (K, V);
@@ -1968,7 +1984,7 @@ fn size_hint(&self) -> (usize, Option<usize>) {
     }
 }
 
-impl<'a, K, V> ExtractIfInner<'a, K, V> {
+impl<'a, K, V, R> ExtractIfInner<'a, K, V, R> {
     /// Allow Debug implementations to predict the next element.
     pub(super) fn peek(&self) -> Option<(&K, &V)> {
         let edge = self.cur_leaf_edge.as_ref()?;
@@ -1978,10 +1994,22 @@ pub(super) fn peek(&self) -> Option<(&K, &V)> {
     /// Implementation of a typical `ExtractIf::next` method, given the predicate.
     pub(super) fn next<F, A: Allocator + Clone>(&mut self, pred: &mut F, alloc: A) -> Option<(K, V)>
     where
+        K: PartialOrd,
+        R: RangeBounds<K>,
         F: FnMut(&K, &mut V) -> bool,
     {
         while let Ok(mut kv) = self.cur_leaf_edge.take()?.next_kv() {
             let (k, v) = kv.kv_mut();
+
+            // On creation, we navigated directly to the left bound, so we need only check the
+            // right bound here to decide whether to stop.
+            match self.range.end_bound() {
+                Bound::Included(ref end) if (*k).le(end) => (),
+                Bound::Excluded(ref end) if (*k).lt(end) => (),
+                Bound::Unbounded => (),
+                _ => return None,
+            }
+
             if pred(k, v) {
                 *self.length -= 1;
                 let (kv, pos) = kv.remove_kv_tracking(
@@ -2013,7 +2041,13 @@ pub(super) fn size_hint(&self) -> (usize, Option<usize>) {
 }
 
 #[unstable(feature = "btree_extract_if", issue = "70530")]
-impl<K, V, F> FusedIterator for ExtractIf<'_, K, V, F> where F: FnMut(&K, &mut V) -> bool {}
+impl<K, V, R, F> FusedIterator for ExtractIf<'_, K, V, R, F>
+where
+    K: PartialOrd,
+    R: RangeBounds<K>,
+    F: FnMut(&K, &mut V) -> bool,
+{
+}
 
 #[stable(feature = "btree_range", since = "1.17.0")]
 impl<'a, K, V> Iterator for Range<'a, K, V> {

library/alloc/src/collections/btree/map/tests.rs

@@ -944,7 +944,7 @@ mod test_extract_if {
     #[test]
     fn empty() {
         let mut map: BTreeMap<i32, i32> = BTreeMap::new();
-        map.extract_if(|_, _| unreachable!("there's nothing to decide on")).for_each(drop);
+        map.extract_if(.., |_, _| unreachable!("there's nothing to decide on")).for_each(drop);
         assert_eq!(map.height(), None);
         map.check();
     }
@@ -954,7 +954,7 @@ fn empty() {
     fn consumed_keeping_all() {
         let pairs = (0..3).map(|i| (i, i));
         let mut map = BTreeMap::from_iter(pairs);
-        assert!(map.extract_if(|_, _| false).eq(iter::empty()));
+        assert!(map.extract_if(.., |_, _| false).eq(iter::empty()));
         map.check();
     }
 
@@ -963,18 +963,42 @@ fn consumed_keeping_all() {
     fn consumed_removing_all() {
         let pairs = (0..3).map(|i| (i, i));
         let mut map = BTreeMap::from_iter(pairs.clone());
-        assert!(map.extract_if(|_, _| true).eq(pairs));
+        assert!(map.extract_if(.., |_, _| true).eq(pairs));
         assert!(map.is_empty());
         map.check();
     }
 
+    #[test]
+    fn consumed_removing_some() {
+        let pairs = (0..3).map(|i| (i, i));
+        let map = BTreeMap::from_iter(pairs);
+        for x in 0..3 {
+            for y in 0..3 {
+                let mut map = map.clone();
+                assert!(map.extract_if(x..y, |_, _| true).eq((x..y).map(|i| (i, i))));
+                for i in 0..3 {
+                    assert_ne!(map.contains_key(&i), (x..y).contains(&i));
+                }
+            }
+        }
+        for x in 0..3 {
+            for y in 0..2 {
+                let mut map = map.clone();
+                assert!(map.extract_if(x..=y, |_, _| true).eq((x..=y).map(|i| (i, i))));
+                for i in 0..3 {
+                    assert_ne!(map.contains_key(&i), (x..=y).contains(&i));
+                }
+            }
+        }
+    }
+
     // Explicitly consumes the iterator and modifies values through it.
     #[test]
     fn mutating_and_keeping() {
         let pairs = (0..3).map(|i| (i, i));
         let mut map = BTreeMap::from_iter(pairs);
         assert!(
-            map.extract_if(|_, v| {
+            map.extract_if(.., |_, v| {
                 *v += 6;
                 false
             })
@@ -991,7 +1015,7 @@ fn mutating_and_removing() {
         let pairs = (0..3).map(|i| (i, i));
         let mut map = BTreeMap::from_iter(pairs);
         assert!(
-            map.extract_if(|_, v| {
+            map.extract_if(.., |_, v| {
                 *v += 6;
                 true
             })
@@ -1005,7 +1029,7 @@ fn mutating_and_removing() {
     fn underfull_keeping_all() {
         let pairs = (0..3).map(|i| (i, i));
         let mut map = BTreeMap::from_iter(pairs);
-        map.extract_if(|_, _| false).for_each(drop);
+        map.extract_if(.., |_, _| false).for_each(drop);
         assert!(map.keys().copied().eq(0..3));
         map.check();
     }
@@ -1015,7 +1039,7 @@ fn underfull_removing_one() {
         let pairs = (0..3).map(|i| (i, i));
         for doomed in 0..3 {
             let mut map = BTreeMap::from_iter(pairs.clone());
-            map.extract_if(|i, _| *i == doomed).for_each(drop);
+            map.extract_if(.., |i, _| *i == doomed).for_each(drop);
             assert_eq!(map.len(), 2);
             map.check();
         }
@@ -1026,7 +1050,7 @@ fn underfull_keeping_one() {
         let pairs = (0..3).map(|i| (i, i));
         for sacred in 0..3 {
             let mut map = BTreeMap::from_iter(pairs.clone());
-            map.extract_if(|i, _| *i != sacred).for_each(drop);
+            map.extract_if(.., |i, _| *i != sacred).for_each(drop);
             assert!(map.keys().copied().eq(sacred..=sacred));
             map.check();
         }
@@ -1036,7 +1060,7 @@ fn underfull_keeping_one() {
     fn underfull_removing_all() {
         let pairs = (0..3).map(|i| (i, i));
         let mut map = BTreeMap::from_iter(pairs);
-        map.extract_if(|_, _| true).for_each(drop);
+        map.extract_if(.., |_, _| true).for_each(drop);
         assert!(map.is_empty());
         map.check();
     }
@@ -1045,7 +1069,7 @@ fn underfull_removing_all() {
     fn height_0_keeping_all() {
         let pairs = (0..node::CAPACITY).map(|i| (i, i));
         let mut map = BTreeMap::from_iter(pairs);
-        map.extract_if(|_, _| false).for_each(drop);
+        map.extract_if(.., |_, _| false).for_each(drop);
         assert!(map.keys().copied().eq(0..node::CAPACITY));
         map.check();
     }
@@ -1055,7 +1079,7 @@ fn height_0_removing_one() {
         let pairs = (0..node::CAPACITY).map(|i| (i, i));
         for doomed in 0..node::CAPACITY {
             let mut map = BTreeMap::from_iter(pairs.clone());
-            map.extract_if(|i, _| *i == doomed).for_each(drop);
+            map.extract_if(.., |i, _| *i == doomed).for_each(drop);
             assert_eq!(map.len(), node::CAPACITY - 1);
             map.check();
         }
@@ -1066,7 +1090,7 @@ fn height_0_keeping_one() {
         let pairs = (0..node::CAPACITY).map(|i| (i, i));
         for sacred in 0..node::CAPACITY {
             let mut map = BTreeMap::from_iter(pairs.clone());
-            map.extract_if(|i, _| *i != sacred).for_each(drop);
+            map.extract_if(.., |i, _| *i != sacred).for_each(drop);
             assert!(map.keys().copied().eq(sacred..=sacred));
             map.check();
         }
@@ -1076,7 +1100,7 @@ fn height_0_keeping_one() {
     fn height_0_removing_all() {
         let pairs = (0..node::CAPACITY).map(|i| (i, i));
         let mut map = BTreeMap::from_iter(pairs);
-        map.extract_if(|_, _| true).for_each(drop);
+        map.extract_if(.., |_, _| true).for_each(drop);
         assert!(map.is_empty());
         map.check();
     }
@@ -1084,7 +1108,7 @@ fn height_0_removing_all() {
     #[test]
     fn height_0_keeping_half() {
         let mut map = BTreeMap::from_iter((0..16).map(|i| (i, i)));
-        assert_eq!(map.extract_if(|i, _| *i % 2 == 0).count(), 8);
+        assert_eq!(map.extract_if(.., |i, _| *i % 2 == 0).count(), 8);
         assert_eq!(map.len(), 8);
         map.check();
     }
@@ -1093,7 +1117,7 @@ fn height_0_keeping_half() {
     fn height_1_removing_all() {
         let pairs = (0..MIN_INSERTS_HEIGHT_1).map(|i| (i, i));
         let mut map = BTreeMap::from_iter(pairs);
-        map.extract_if(|_, _| true).for_each(drop);
+        map.extract_if(.., |_, _| true).for_each(drop);
         assert!(map.is_empty());
         map.check();
     }
@@ -1103,7 +1127,7 @@ fn height_1_removing_one() {
         let pairs = (0..MIN_INSERTS_HEIGHT_1).map(|i| (i, i));
         for doomed in 0..MIN_INSERTS_HEIGHT_1 {
             let mut map = BTreeMap::from_iter(pairs.clone());
-            map.extract_if(|i, _| *i == doomed).for_each(drop);
+            map.extract_if(.., |i, _| *i == doomed).for_each(drop);
             assert_eq!(map.len(), MIN_INSERTS_HEIGHT_1 - 1);
             map.check();
         }
@@ -1114,7 +1138,7 @@ fn height_1_keeping_one() {
         let pairs = (0..MIN_INSERTS_HEIGHT_1).map(|i| (i, i));
         for sacred in 0..MIN_INSERTS_HEIGHT_1 {
             let mut map = BTreeMap::from_iter(pairs.clone());
-            map.extract_if(|i, _| *i != sacred).for_each(drop);
+            map.extract_if(.., |i, _| *i != sacred).for_each(drop);
             assert!(map.keys().copied().eq(sacred..=sacred));
             map.check();
         }
@@ -1125,7 +1149,7 @@ fn height_2_removing_one() {
         let pairs = (0..MIN_INSERTS_HEIGHT_2).map(|i| (i, i));
         for doomed in (0..MIN_INSERTS_HEIGHT_2).step_by(12) {
             let mut map = BTreeMap::from_iter(pairs.clone());
-            map.extract_if(|i, _| *i == doomed).for_each(drop);
+            map.extract_if(.., |i, _| *i == doomed).for_each(drop);
             assert_eq!(map.len(), MIN_INSERTS_HEIGHT_2 - 1);
             map.check();
         }
@@ -1136,7 +1160,7 @@ fn height_2_keeping_one() {
         let pairs = (0..MIN_INSERTS_HEIGHT_2).map(|i| (i, i));
         for sacred in (0..MIN_INSERTS_HEIGHT_2).step_by(12) {
             let mut map = BTreeMap::from_iter(pairs.clone());
-            map.extract_if(|i, _| *i != sacred).for_each(drop);
+            map.extract_if(.., |i, _| *i != sacred).for_each(drop);
             assert!(map.keys().copied().eq(sacred..=sacred));
             map.check();
         }
@@ -1146,7 +1170,7 @@ fn height_2_keeping_one() {
     fn height_2_removing_all() {
         let pairs = (0..MIN_INSERTS_HEIGHT_2).map(|i| (i, i));
         let mut map = BTreeMap::from_iter(pairs);
-        map.extract_if(|_, _| true).for_each(drop);
+        map.extract_if(.., |_, _| true).for_each(drop);
         assert!(map.is_empty());
         map.check();
     }
@@ -1162,7 +1186,7 @@ fn drop_panic_leak() {
         map.insert(b.spawn(Panic::InDrop), ());
         map.insert(c.spawn(Panic::Never), ());
 
-        catch_unwind(move || map.extract_if(|dummy, _| dummy.query(true)).for_each(drop))
+        catch_unwind(move || map.extract_if(.., |dummy, _| dummy.query(true)).for_each(drop))
             .unwrap_err();
 
         assert_eq!(a.queried(), 1);
@@ -1185,7 +1209,7 @@ fn pred_panic_leak() {
         map.insert(c.spawn(Panic::InQuery), ());
 
         catch_unwind(AssertUnwindSafe(|| {
-            map.extract_if(|dummy, _| dummy.query(true)).for_each(drop)
+            map.extract_if(.., |dummy, _| dummy.query(true)).for_each(drop)
         }))
         .unwrap_err();
 
@@ -1214,7 +1238,7 @@ fn pred_panic_reuse() {
         map.insert(c.spawn(Panic::InQuery), ());
 
         {
-            let mut it = map.extract_if(|dummy, _| dummy.query(true));
+            let mut it = map.extract_if(.., |dummy, _| dummy.query(true));
             catch_unwind(AssertUnwindSafe(|| while it.next().is_some() {})).unwrap_err();
             // Iterator behavior after a panic is explicitly unspecified,
             // so this is just the current implementation:
@@ -1658,7 +1682,7 @@ fn into_values<T: Sync + Ord>(v: BTreeMap<T, T>) -> impl Sync {
     }
 
     fn extract_if<T: Sync + Ord>(v: &mut BTreeMap<T, T>) -> impl Sync + '_ {
-        v.extract_if(|_, _| false)
+        v.extract_if(.., |_, _| false)
     }
 
     fn iter<T: Sync>(v: &BTreeMap<T, T>) -> impl Sync + '_ {
@@ -1727,7 +1751,7 @@ fn into_values<T: Send + Ord>(v: BTreeMap<T, T>) -> impl Send {
     }
 
     fn extract_if<T: Send + Ord>(v: &mut BTreeMap<T, T>) -> impl Send + '_ {
-        v.extract_if(|_, _| false)
+        v.extract_if(.., |_, _| false)
     }
 
     fn iter<T: Send + Sync>(v: &BTreeMap<T, T>) -> impl Send + '_ {

library/alloc/src/collections/btree/set.rs

@@ -1109,7 +1109,7 @@ pub fn retain<F>(&mut self, mut f: F)
         T: Ord,
         F: FnMut(&T) -> bool,
     {
-        self.extract_if(|v| !f(v)).for_each(drop);
+        self.extract_if(.., |v| !f(v)).for_each(drop);
     }
 
     /// Moves all elements from `other` into `self`, leaving `other` empty.
@@ -1187,7 +1187,7 @@ pub fn split_off<Q: ?Sized + Ord>(&mut self, value: &Q) -> Self
         BTreeSet { map: self.map.split_off(value) }
     }
 
-    /// Creates an iterator that visits all elements in ascending order and
+    /// Creates an iterator that visits elements in the specified range in ascending order and
     /// uses a closure to determine if an element should be removed.
     ///
     /// If the closure returns `true`, the element is removed from the set and
@@ -1208,18 +1208,25 @@ pub fn split_off<Q: ?Sized + Ord>(&mut self, value: &Q) -> Self
     /// use std::collections::BTreeSet;
     ///
     /// let mut set: BTreeSet<i32> = (0..8).collect();
-    /// let evens: BTreeSet<_> = set.extract_if(|v| v % 2 == 0).collect();
+    /// let evens: BTreeSet<_> = set.extract_if(.., |v| v % 2 == 0).collect();
     /// let odds = set;
     /// assert_eq!(evens.into_iter().collect::<Vec<_>>(), vec![0, 2, 4, 6]);
     /// assert_eq!(odds.into_iter().collect::<Vec<_>>(), vec![1, 3, 5, 7]);
+    ///
+    /// let mut map: BTreeSet<i32> = (0..8).collect();
+    /// let low: BTreeSet<_> = map.extract_if(0..4, |_v| true).collect();
+    /// let high = map;
+    /// assert_eq!(low.into_iter().collect::<Vec<_>>(), [0, 1, 2, 3]);
+    /// assert_eq!(high.into_iter().collect::<Vec<_>>(), [4, 5, 6, 7]);
     /// ```
     #[unstable(feature = "btree_extract_if", issue = "70530")]
-    pub fn extract_if<'a, F>(&'a mut self, pred: F) -> ExtractIf<'a, T, F, A>
+    pub fn extract_if<'a, F, R>(&'a mut self, range: R, pred: F) -> ExtractIf<'a, T, R, F, A>
     where
         T: Ord,
+        R: RangeBounds<T>,
         F: 'a + FnMut(&T) -> bool,
     {
-        let (inner, alloc) = self.map.extract_if_inner();
+        let (inner, alloc) = self.map.extract_if_inner(range);
         ExtractIf { pred, inner, alloc }
     }
 
@@ -1554,17 +1561,18 @@ fn into_iter(self) -> Iter<'a, T> {
 pub struct ExtractIf<
     'a,
     T,
+    R,
     F,
     #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + Clone = Global,
 > {
     pred: F,
-    inner: super::map::ExtractIfInner<'a, T, SetValZST>,
+    inner: super::map::ExtractIfInner<'a, T, SetValZST, R>,
     /// The BTreeMap will outlive this IntoIter so we don't care about drop order for `alloc`.
     alloc: A,
 }
 
 #[unstable(feature = "btree_extract_if", issue = "70530")]
-impl<T, F, A> fmt::Debug for ExtractIf<'_, T, F, A>
+impl<T, R, F, A> fmt::Debug for ExtractIf<'_, T, R, F, A>
 where
     T: fmt::Debug,
     A: Allocator + Clone,
@@ -1577,8 +1585,10 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 }
 
 #[unstable(feature = "btree_extract_if", issue = "70530")]
-impl<'a, T, F, A: Allocator + Clone> Iterator for ExtractIf<'_, T, F, A>
+impl<'a, T, R, F, A: Allocator + Clone> Iterator for ExtractIf<'_, T, R, F, A>
 where
+    T: PartialOrd,
+    R: RangeBounds<T>,
     F: 'a + FnMut(&T) -> bool,
 {
     type Item = T;
@@ -1595,7 +1605,13 @@ fn size_hint(&self) -> (usize, Option<usize>) {
 }
 
 #[unstable(feature = "btree_extract_if", issue = "70530")]
-impl<T, F, A: Allocator + Clone> FusedIterator for ExtractIf<'_, T, F, A> where F: FnMut(&T) -> bool {}
+impl<T, R, F, A: Allocator + Clone> FusedIterator for ExtractIf<'_, T, R, F, A>
+where
+    T: PartialOrd,
+    R: RangeBounds<T>,
+    F: FnMut(&T) -> bool,
+{
+}
 
 #[stable(feature = "rust1", since = "1.0.0")]
 impl<T: Ord, A: Allocator + Clone> Extend<T> for BTreeSet<T, A> {

library/alloc/src/collections/btree/set/tests.rs

@@ -368,8 +368,8 @@ fn test_extract_if() {
     let mut x = BTreeSet::from([1]);
     let mut y = BTreeSet::from([1]);
 
-    x.extract_if(|_| true).for_each(drop);
-    y.extract_if(|_| false).for_each(drop);
+    x.extract_if(.., |_| true).for_each(drop);
+    y.extract_if(.., |_| false).for_each(drop);
     assert_eq!(x.len(), 0);
     assert_eq!(y.len(), 1);
 }
@@ -385,7 +385,7 @@ fn test_extract_if_drop_panic_leak() {
     set.insert(b.spawn(Panic::InDrop));
     set.insert(c.spawn(Panic::Never));
 
-    catch_unwind(move || set.extract_if(|dummy| dummy.query(true)).for_each(drop)).ok();
+    catch_unwind(move || set.extract_if(.., |dummy| dummy.query(true)).for_each(drop)).ok();
 
     assert_eq!(a.queried(), 1);
     assert_eq!(b.queried(), 1);
@@ -406,7 +406,7 @@ fn test_extract_if_pred_panic_leak() {
     set.insert(b.spawn(Panic::InQuery));
     set.insert(c.spawn(Panic::InQuery));
 
-    catch_unwind(AssertUnwindSafe(|| set.extract_if(|dummy| dummy.query(true)).for_each(drop)))
+    catch_unwind(AssertUnwindSafe(|| set.extract_if(.., |dummy| dummy.query(true)).for_each(drop)))
         .ok();
 
     assert_eq!(a.queried(), 1);
@@ -605,7 +605,7 @@ fn range<T: Sync + Ord>(v: &BTreeSet<T>) -> impl Sync + '_ {
     }
 
     fn extract_if<T: Sync + Ord>(v: &mut BTreeSet<T>) -> impl Sync + '_ {
-        v.extract_if(|_| false)
+        v.extract_if(.., |_| false)
     }
 
     fn difference<T: Sync + Ord>(v: &BTreeSet<T>) -> impl Sync + '_ {
@@ -644,7 +644,7 @@ fn range<T: Send + Sync + Ord>(v: &BTreeSet<T>) -> impl Send + '_ {
     }
 
     fn extract_if<T: Send + Ord>(v: &mut BTreeSet<T>) -> impl Send + '_ {
-        v.extract_if(|_| false)
+        v.extract_if(.., |_| false)
     }
 
     fn difference<T: Send + Sync + Ord>(v: &BTreeSet<T>) -> impl Send + '_ {

library/alloctests/benches/btree/map.rs

@@ -386,7 +386,7 @@ pub fn clone_slim_100_and_clear(b: &mut Bencher) {
 #[bench]
 pub fn clone_slim_100_and_drain_all(b: &mut Bencher) {
     let src = slim_map(100);
-    b.iter(|| src.clone().extract_if(|_, _| true).count())
+    b.iter(|| src.clone().extract_if(.., |_, _| true).count())
 }
 
 #[bench]
@@ -394,7 +394,7 @@ pub fn clone_slim_100_and_drain_half(b: &mut Bencher) {
     let src = slim_map(100);
     b.iter(|| {
         let mut map = src.clone();
-        assert_eq!(map.extract_if(|i, _| i % 2 == 0).count(), 100 / 2);
+        assert_eq!(map.extract_if(.., |i, _| i % 2 == 0).count(), 100 / 2);
         assert_eq!(map.len(), 100 / 2);
     })
 }
@@ -457,7 +457,7 @@ pub fn clone_slim_10k_and_clear(b: &mut Bencher) {
 #[bench]
 pub fn clone_slim_10k_and_drain_all(b: &mut Bencher) {
     let src = slim_map(10_000);
-    b.iter(|| src.clone().extract_if(|_, _| true).count())
+    b.iter(|| src.clone().extract_if(.., |_, _| true).count())
 }
 
 #[bench]
@@ -465,7 +465,7 @@ pub fn clone_slim_10k_and_drain_half(b: &mut Bencher) {
     let src = slim_map(10_000);
     b.iter(|| {
         let mut map = src.clone();
-        assert_eq!(map.extract_if(|i, _| i % 2 == 0).count(), 10_000 / 2);
+        assert_eq!(map.extract_if(.., |i, _| i % 2 == 0).count(), 10_000 / 2);
         assert_eq!(map.len(), 10_000 / 2);
     })
 }
@@ -528,7 +528,7 @@ pub fn clone_fat_val_100_and_clear(b: &mut Bencher) {
 #[bench]
 pub fn clone_fat_val_100_and_drain_all(b: &mut Bencher) {
     let src = fat_val_map(100);
-    b.iter(|| src.clone().extract_if(|_, _| true).count())
+    b.iter(|| src.clone().extract_if(.., |_, _| true).count())
 }
 
 #[bench]
@@ -536,7 +536,7 @@ pub fn clone_fat_val_100_and_drain_half(b: &mut Bencher) {
     let src = fat_val_map(100);
     b.iter(|| {
         let mut map = src.clone();
-        assert_eq!(map.extract_if(|i, _| i % 2 == 0).count(), 100 / 2);
+        assert_eq!(map.extract_if(.., |i, _| i % 2 == 0).count(), 100 / 2);
         assert_eq!(map.len(), 100 / 2);
     })
 }

library/alloctests/benches/btree/set.rs

@@ -69,7 +69,7 @@ pub fn clone_100_and_clear(b: &mut Bencher) {
 #[bench]
 pub fn clone_100_and_drain_all(b: &mut Bencher) {
     let src = slim_set(100);
-    b.iter(|| src.clone().extract_if(|_| true).count())
+    b.iter(|| src.clone().extract_if(.., |_| true).count())
 }
 
 #[bench]
@@ -77,7 +77,7 @@ pub fn clone_100_and_drain_half(b: &mut Bencher) {
     let src = slim_set(100);
     b.iter(|| {
         let mut set = src.clone();
-        assert_eq!(set.extract_if(|i| i % 2 == 0).count(), 100 / 2);
+        assert_eq!(set.extract_if(.., |i| i % 2 == 0).count(), 100 / 2);
         assert_eq!(set.len(), 100 / 2);
     })
 }
@@ -140,7 +140,7 @@ pub fn clone_10k_and_clear(b: &mut Bencher) {
 #[bench]
 pub fn clone_10k_and_drain_all(b: &mut Bencher) {
     let src = slim_set(10_000);
-    b.iter(|| src.clone().extract_if(|_| true).count())
+    b.iter(|| src.clone().extract_if(.., |_| true).count())
 }
 
 #[bench]
@@ -148,7 +148,7 @@ pub fn clone_10k_and_drain_half(b: &mut Bencher) {
     let src = slim_set(10_000);
     b.iter(|| {
         let mut set = src.clone();
-        assert_eq!(set.extract_if(|i| i % 2 == 0).count(), 10_000 / 2);
+        assert_eq!(set.extract_if(.., |i| i % 2 == 0).count(), 10_000 / 2);
         assert_eq!(set.len(), 10_000 / 2);
     })
 }

library/alloctests/tests/autotraits.rs

@@ -1,3 +1,5 @@
+use std::ops::Range;
+
 fn require_sync<T: Sync>(_: T) {}
 fn require_send_sync<T: Send + Sync>(_: T) {}
 
@@ -55,7 +57,13 @@ fn test_btree_map() {
 
     require_send_sync(async {
         let _v = None::<
-            alloc::collections::btree_map::ExtractIf<'_, &u32, &u32, fn(&&u32, &mut &u32) -> bool>,
+            alloc::collections::btree_map::ExtractIf<
+                '_,
+                &u32,
+                &u32,
+                Range<u32>,
+                fn(&&u32, &mut &u32) -> bool,
+            >,
         >;
         async {}.await;
     });
@@ -144,7 +152,9 @@ fn test_btree_set() {
     });
 
     require_send_sync(async {
-        let _v = None::<alloc::collections::btree_set::ExtractIf<'_, &u32, fn(&&u32) -> bool>>;
+        let _v = None::<
+            alloc::collections::btree_set::ExtractIf<'_, &u32, Range<u32>, fn(&&u32) -> bool>,
+        >;
         async {}.await;
     });
 

library/core/src/intrinsics/mod.rs

@@ -30,7 +30,7 @@
 //!
 //! The atomic intrinsics provide common atomic operations on machine
 //! words, with multiple possible memory orderings. See the
-//! [atomic types][crate::sync::atomic] docs for details.
+//! [atomic types][atomic] docs for details.
 //!
 //! # Unwinding
 //!
@@ -50,7 +50,7 @@
 )]
 #![allow(missing_docs)]
 
-use crate::marker::{DiscriminantKind, Tuple};
+use crate::marker::{ConstParamTy, DiscriminantKind, Tuple};
 use crate::ptr;
 
 pub mod fallback;
@@ -62,6 +62,20 @@
 #[cfg(all(target_has_atomic = "8", target_has_atomic = "32", target_has_atomic = "ptr"))]
 use crate::sync::atomic::{self, AtomicBool, AtomicI32, AtomicIsize, AtomicU32, Ordering};
 
+/// A type for atomic ordering parameters for intrinsics. This is a separate type from
+/// `atomic::Ordering` so that we can make it `ConstParamTy` and fix the values used here without a
+/// risk of leaking that to stable code.
+#[derive(Debug, ConstParamTy, PartialEq, Eq)]
+pub enum AtomicOrdering {
+    // These values must match the compiler's `AtomicOrdering` defined in
+    // `rustc_middle/src/ty/consts/int.rs`!
+    Relaxed = 0,
+    Release = 1,
+    Acquire = 2,
+    AcqRel = 3,
+    SeqCst = 4,
+}
+
 // N.B., these intrinsics take raw pointers because they mutate aliased
 // memory, which is not valid for either `&` or `&mut`.
 
@@ -391,6 +405,15 @@ pub unsafe fn atomic_cxchgweak_release_acquire<T: Copy>(
 #[rustc_nounwind]
 pub unsafe fn atomic_cxchgweak_seqcst_seqcst<T: Copy>(dst: *mut T, old: T, src: T) -> (T, bool);
 
+/// Loads the current value of the pointer.
+/// `T` must be an integer or pointer type.
+///
+/// The stabilized version of this intrinsic is available on the
+/// [`atomic`] types via the `load` method. For example, [`AtomicBool::load`].
+#[rustc_intrinsic]
+#[rustc_nounwind]
+#[cfg(not(bootstrap))]
+pub unsafe fn atomic_load<T: Copy, const ORD: AtomicOrdering>(src: *const T) -> T;
 /// Loads the current value of the pointer.
 /// `T` must be an integer or pointer type.
 ///
@@ -399,6 +422,7 @@ pub unsafe fn atomic_cxchgweak_release_acquire<T: Copy>(
 /// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicBool::load`].
 #[rustc_intrinsic]
 #[rustc_nounwind]
+#[cfg(bootstrap)]
 pub unsafe fn atomic_load_seqcst<T: Copy>(src: *const T) -> T;
 /// Loads the current value of the pointer.
 /// `T` must be an integer or pointer type.
@@ -408,6 +432,7 @@ pub unsafe fn atomic_cxchgweak_release_acquire<T: Copy>(
 /// [`Ordering::Acquire`] as the `order`. For example, [`AtomicBool::load`].
 #[rustc_intrinsic]
 #[rustc_nounwind]
+#[cfg(bootstrap)]
 pub unsafe fn atomic_load_acquire<T: Copy>(src: *const T) -> T;
 /// Loads the current value of the pointer.
 /// `T` must be an integer or pointer type.
@@ -417,6 +442,7 @@ pub unsafe fn atomic_cxchgweak_release_acquire<T: Copy>(
 /// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicBool::load`].
 #[rustc_intrinsic]
 #[rustc_nounwind]
+#[cfg(bootstrap)]
 pub unsafe fn atomic_load_relaxed<T: Copy>(src: *const T) -> T;
 
 /// Stores the value at the specified memory location.

library/core/src/num/f32.rs

@@ -999,6 +999,7 @@ pub const fn midpoint(self, other: f32) -> f32 {
                 target_arch = "x86_64",
                 target_arch = "aarch64",
                 all(any(target_arch = "riscv32", target_arch = "riscv64"), target_feature = "d"),
+                all(target_arch = "loongarch64", target_feature = "d"),
                 all(target_arch = "arm", target_feature = "vfp2"),
                 target_arch = "wasm32",
                 target_arch = "wasm64",

library/core/src/num/int_macros.rs

@@ -4,7 +4,7 @@ macro_rules! int_impl {
         ActualT = $ActualT:ident,
         UnsignedT = $UnsignedT:ty,
 
-        // There are all for use *only* in doc comments.
+        // These are all for use *only* in doc comments.
         // As such, they're all passed as literals -- passing them as a string
         // literal is fine if they need to be multiple code tokens.
         // In non-comments, use the associated constants rather than these.
@@ -1018,6 +1018,110 @@ pub const fn strict_div_euclid(self, rhs: Self) -> Self {
             if b { overflow_panic::div() } else { a }
         }
 
+        /// Checked integer division without remainder. Computes `self / rhs`,
+        /// returning `None` if `rhs == 0`, the division results in overflow,
+        /// or `self % rhs != 0`.
+        ///
+        /// # Examples
+        ///
+        /// Basic usage:
+        ///
+        /// ```
+        /// #![feature(exact_div)]
+        #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN + 1).checked_exact_div(-1), Some(", stringify!($Max), "));")]
+        #[doc = concat!("assert_eq!((-5", stringify!($SelfT), ").checked_exact_div(2), None);")]
+        #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.checked_exact_div(-1), None);")]
+        #[doc = concat!("assert_eq!((1", stringify!($SelfT), ").checked_exact_div(0), None);")]
+        /// ```
+        #[unstable(
+            feature = "exact_div",
+            issue = "139911",
+        )]
+        #[must_use = "this returns the result of the operation, \
+                      without modifying the original"]
+        #[inline]
+        pub const fn checked_exact_div(self, rhs: Self) -> Option<Self> {
+            if intrinsics::unlikely(rhs == 0 || ((self == Self::MIN) && (rhs == -1))) {
+                None
+            } else {
+                // SAFETY: division by zero and overflow are checked above
+                unsafe {
+                    if intrinsics::unlikely(intrinsics::unchecked_rem(self, rhs) != 0) {
+                        None
+                    } else {
+                        Some(intrinsics::exact_div(self, rhs))
+                    }
+                }
+            }
+        }
+
+        /// Checked integer division without remainder. Computes `self / rhs`.
+        ///
+        /// # Panics
+        ///
+        /// This function will panic  if `rhs == 0`, the division results in overflow,
+        /// or `self % rhs != 0`.
+        ///
+        /// # Examples
+        ///
+        /// Basic usage:
+        ///
+        /// ```
+        /// #![feature(exact_div)]
+        #[doc = concat!("assert_eq!(64", stringify!($SelfT), ".exact_div(2), 32);")]
+        #[doc = concat!("assert_eq!(64", stringify!($SelfT), ".exact_div(32), 2);")]
+        #[doc = concat!("assert_eq!((", stringify!($SelfT), "::MIN + 1).exact_div(-1), ", stringify!($Max), ");")]
+        /// ```
+        ///
+        /// ```should_panic
+        /// #![feature(exact_div)]
+        #[doc = concat!("let _ = 65", stringify!($SelfT), ".exact_div(2);")]
+        /// ```
+        /// ```should_panic
+        /// #![feature(exact_div)]
+        #[doc = concat!("let _ = ", stringify!($SelfT), "::MIN.exact_div(-1);")]
+        /// ```
+        #[unstable(
+            feature = "exact_div",
+            issue = "139911",
+        )]
+        #[must_use = "this returns the result of the operation, \
+                      without modifying the original"]
+        #[inline]
+        pub const fn exact_div(self, rhs: Self) -> Self {
+            match self.checked_exact_div(rhs) {
+                Some(x) => x,
+                None => panic!("Failed to divide without remainder"),
+            }
+        }
+
+        /// Unchecked integer division without remainder. Computes `self / rhs`.
+        ///
+        /// # Safety
+        ///
+        /// This results in undefined behavior when `rhs == 0`, `self % rhs != 0`, or
+        #[doc = concat!("`self == ", stringify!($SelfT), "::MIN && rhs == -1`,")]
+        /// i.e. when [`checked_exact_div`](Self::checked_exact_div) would return `None`.
+        #[unstable(
+            feature = "exact_div",
+            issue = "139911",
+        )]
+        #[must_use = "this returns the result of the operation, \
+                      without modifying the original"]
+        #[inline]
+        pub const unsafe fn unchecked_exact_div(self, rhs: Self) -> Self {
+            assert_unsafe_precondition!(
+                check_language_ub,
+                concat!(stringify!($SelfT), "::unchecked_exact_div cannot overflow, divide by zero, or leave a remainder"),
+                (
+                    lhs: $SelfT = self,
+                    rhs: $SelfT = rhs,
+                ) => rhs > 0 && lhs % rhs == 0 && (lhs != <$SelfT>::MIN || rhs != -1),
+            );
+            // SAFETY: Same precondition
+            unsafe { intrinsics::exact_div(self, rhs) }
+        }
+
         /// Checked integer remainder. Computes `self % rhs`, returning `None` if
         /// `rhs == 0` or the division results in overflow.
         ///

library/core/src/num/uint_macros.rs

@@ -1110,6 +1110,108 @@ pub const fn strict_div_euclid(self, rhs: Self) -> Self {
             self / rhs
         }
 
+        /// Checked integer division without remainder. Computes `self / rhs`.
+        ///
+        /// # Panics
+        ///
+        /// This function will panic  if `rhs == 0` or `self % rhs != 0`.
+        ///
+        /// # Examples
+        ///
+        /// Basic usage:
+        ///
+        /// ```
+        /// #![feature(exact_div)]
+        #[doc = concat!("assert_eq!(64", stringify!($SelfT), ".exact_div(2), 32);")]
+        #[doc = concat!("assert_eq!(64", stringify!($SelfT), ".exact_div(32), 2);")]
+        /// ```
+        ///
+        /// ```should_panic
+        /// #![feature(exact_div)]
+        #[doc = concat!("let _ = 65", stringify!($SelfT), ".exact_div(2);")]
+        /// ```
+        #[unstable(
+            feature = "exact_div",
+            issue = "139911",
+        )]
+        #[must_use = "this returns the result of the operation, \
+                      without modifying the original"]
+        #[inline]
+        pub const fn checked_exact_div(self, rhs: Self) -> Option<Self> {
+            if intrinsics::unlikely(rhs == 0) {
+                None
+            } else {
+                // SAFETY: division by zero is checked above
+                unsafe {
+                    if intrinsics::unlikely(intrinsics::unchecked_rem(self, rhs) != 0) {
+                        None
+                    } else {
+                        Some(intrinsics::exact_div(self, rhs))
+                    }
+                }
+            }
+        }
+
+        /// Checked integer division without remainder. Computes `self / rhs`.
+        ///
+        /// # Panics
+        ///
+        /// This function will panic  if `rhs == 0` or `self % rhs != 0`.
+        ///
+        /// # Examples
+        ///
+        /// Basic usage:
+        ///
+        /// ```
+        /// #![feature(exact_div)]
+        #[doc = concat!("assert_eq!(64", stringify!($SelfT), ".exact_div(2), 32);")]
+        #[doc = concat!("assert_eq!(64", stringify!($SelfT), ".exact_div(32), 2);")]
+        /// ```
+        ///
+        /// ```should_panic
+        /// #![feature(exact_div)]
+        #[doc = concat!("let _ = 65", stringify!($SelfT), ".exact_div(2);")]
+        /// ```
+        #[unstable(
+            feature = "exact_div",
+            issue = "139911",
+        )]
+        #[must_use = "this returns the result of the operation, \
+                      without modifying the original"]
+        #[inline]
+        pub const fn exact_div(self, rhs: Self) -> Self {
+            match self.checked_exact_div(rhs) {
+                Some(x) => x,
+                None => panic!("Failed to divide without remainder"),
+            }
+        }
+
+        /// Unchecked integer division without remainder. Computes `self / rhs`.
+        ///
+        /// # Safety
+        ///
+        /// This results in undefined behavior when `rhs == 0` or `self % rhs != 0`,
+        /// i.e. when [`checked_exact_div`](Self::checked_exact_div) would return `None`.
+        #[unstable(
+            feature = "exact_div",
+            issue = "139911",
+        )]
+        #[must_use = "this returns the result of the operation, \
+                      without modifying the original"]
+        #[inline]
+        pub const unsafe fn unchecked_exact_div(self, rhs: Self) -> Self {
+            assert_unsafe_precondition!(
+                check_language_ub,
+                concat!(stringify!($SelfT), "::unchecked_exact_div divide by zero or leave a remainder"),
+                (
+                    lhs: $SelfT = self,
+                    rhs: $SelfT = rhs,
+                ) => rhs > 0 && lhs % rhs == 0,
+            );
+            // SAFETY: Same precondition
+            unsafe { intrinsics::exact_div(self, rhs) }
+        }
+
         /// Checked integer remainder. Computes `self % rhs`, returning `None`
         /// if `rhs == 0`.
         ///

library/core/src/ptr/const_ptr.rs

@@ -397,35 +397,7 @@ pub const fn to_raw_parts(self) -> (*const (), <T as super::Pointee>::Metadata)
         if self.is_null() { None } else { Some(unsafe { &*(self as *const MaybeUninit<T>) }) }
     }
 
-    /// Adds a signed offset to a pointer.
-    ///
-    /// `count` is in units of T; e.g., a `count` of 3 represents a pointer
-    /// offset of `3 * size_of::<T>()` bytes.
-    ///
-    /// # Safety
-    ///
-    /// If any of the following conditions are violated, the result is Undefined Behavior:
-    ///
-    /// * The offset in bytes, `count * size_of::<T>()`, computed on mathematical integers (without
-    ///   "wrapping around"), must fit in an `isize`.
-    ///
-    /// * If the computed offset is non-zero, then `self` must be [derived from][crate::ptr#provenance] a pointer to some
-    ///   [allocated object], and the entire memory range between `self` and the result must be in
-    ///   bounds of that allocated object. In particular, this range must not "wrap around" the edge
-    ///   of the address space. Note that "range" here refers to a half-open range as usual in Rust,
-    ///   i.e., `self..result` for non-negative offsets and `result..self` for negative offsets.
-    ///
-    /// Allocated objects can never be larger than `isize::MAX` bytes, so if the computed offset
-    /// stays in bounds of the allocated object, it is guaranteed to satisfy the first requirement.
-    /// This implies, for instance, that `vec.as_ptr().add(vec.len())` (for `vec: Vec<T>`) is always
-    /// safe.
-    ///
-    /// Consider using [`wrapping_offset`] instead if these constraints are
-    /// difficult to satisfy. The only advantage of this method is that it
-    /// enables more aggressive compiler optimizations.
-    ///
-    /// [`wrapping_offset`]: #method.wrapping_offset
-    /// [allocated object]: crate::ptr#allocated-object
+    #[doc = include_str!("./docs/offset.md")]
     ///
     /// # Examples
     ///
@@ -905,38 +877,7 @@ pub const fn guaranteed_ne(self, other: *const T) -> Option<bool>
         }
     }
 
-    /// Adds an unsigned offset to a pointer.
-    ///
-    /// This can only move the pointer forward (or not move it). If you need to move forward or
-    /// backward depending on the value, then you might want [`offset`](#method.offset) instead
-    /// which takes a signed offset.
-    ///
-    /// `count` is in units of T; e.g., a `count` of 3 represents a pointer
-    /// offset of `3 * size_of::<T>()` bytes.
-    ///
-    /// # Safety
-    ///
-    /// If any of the following conditions are violated, the result is Undefined Behavior:
-    ///
-    /// * The offset in bytes, `count * size_of::<T>()`, computed on mathematical integers (without
-    ///   "wrapping around"), must fit in an `isize`.
-    ///
-    /// * If the computed offset is non-zero, then `self` must be [derived from][crate::ptr#provenance] a pointer to some
-    ///   [allocated object], and the entire memory range between `self` and the result must be in
-    ///   bounds of that allocated object. In particular, this range must not "wrap around" the edge
-    ///   of the address space.
-    ///
-    /// Allocated objects can never be larger than `isize::MAX` bytes, so if the computed offset
-    /// stays in bounds of the allocated object, it is guaranteed to satisfy the first requirement.
-    /// This implies, for instance, that `vec.as_ptr().add(vec.len())` (for `vec: Vec<T>`) is always
-    /// safe.
-    ///
-    /// Consider using [`wrapping_add`] instead if these constraints are
-    /// difficult to satisfy. The only advantage of this method is that it
-    /// enables more aggressive compiler optimizations.
-    ///
-    /// [`wrapping_add`]: #method.wrapping_add
-    /// [allocated object]: crate::ptr#allocated-object
+    #[doc = include_str!("./docs/add.md")]
     ///
     /// # Examples
     ///

library/core/src/ptr/docs/add.md

@@ -0,0 +1,32 @@
+Adds an unsigned offset to a pointer.
+
+This can only move the pointer forward (or not move it). If you need to move forward or
+backward depending on the value, then you might want [`offset`](#method.offset) instead
+which takes a signed offset.
+
+`count` is in units of T; e.g., a `count` of 3 represents a pointer
+offset of `3 * size_of::<T>()` bytes.
+
+# Safety
+
+If any of the following conditions are violated, the result is Undefined Behavior:
+
+* The offset in bytes, `count * size_of::<T>()`, computed on mathematical integers (without
+"wrapping around"), must fit in an `isize`.
+
+* If the computed offset is non-zero, then `self` must be [derived from][crate::ptr#provenance] a pointer to some
+[allocated object], and the entire memory range between `self` and the result must be in
+bounds of that allocated object. In particular, this range must not "wrap around" the edge
+of the address space.
+
+Allocated objects can never be larger than `isize::MAX` bytes, so if the computed offset
+stays in bounds of the allocated object, it is guaranteed to satisfy the first requirement.
+This implies, for instance, that `vec.as_ptr().add(vec.len())` (for `vec: Vec<T>`) is always
+safe.
+
+Consider using [`wrapping_add`] instead if these constraints are
+difficult to satisfy. The only advantage of this method is that it
+enables more aggressive compiler optimizations.
+
+[`wrapping_add`]: #method.wrapping_add
+[allocated object]: crate::ptr#allocated-object

library/core/src/ptr/docs/offset.md

@@ -0,0 +1,29 @@
+Adds a signed offset to a pointer.
+
+`count` is in units of T; e.g., a `count` of 3 represents a pointer
+offset of `3 * size_of::<T>()` bytes.
+
+# Safety
+
+If any of the following conditions are violated, the result is Undefined Behavior:
+
+* The offset in bytes, `count * size_of::<T>()`, computed on mathematical integers (without
+"wrapping around"), must fit in an `isize`.
+
+* If the computed offset is non-zero, then `self` must be [derived from][crate::ptr#provenance] a pointer to some
+[allocated object], and the entire memory range between `self` and the result must be in
+bounds of that allocated object. In particular, this range must not "wrap around" the edge
+of the address space. Note that "range" here refers to a half-open range as usual in Rust,
+i.e., `self..result` for non-negative offsets and `result..self` for negative offsets.
+
+Allocated objects can never be larger than `isize::MAX` bytes, so if the computed offset
+stays in bounds of the allocated object, it is guaranteed to satisfy the first requirement.
+This implies, for instance, that `vec.as_ptr().add(vec.len())` (for `vec: Vec<T>`) is always
+safe.
+
+Consider using [`wrapping_offset`] instead if these constraints are
+difficult to satisfy. The only advantage of this method is that it
+enables more aggressive compiler optimizations.
+
+[`wrapping_offset`]: #method.wrapping_offset
+[allocated object]: crate::ptr#allocated-object

library/core/src/ptr/mut_ptr.rs

@@ -394,34 +394,7 @@ pub const fn to_raw_parts(self) -> (*mut (), <T as super::Pointee>::Metadata) {
         if self.is_null() { None } else { Some(unsafe { &*(self as *const MaybeUninit<T>) }) }
     }
 
-    /// Adds a signed offset to a pointer.
-    ///
-    /// `count` is in units of T; e.g., a `count` of 3 represents a pointer
-    /// offset of `3 * size_of::<T>()` bytes.
-    ///
-    /// # Safety
-    ///
-    /// If any of the following conditions are violated, the result is Undefined Behavior:
-    ///
-    /// * The offset in bytes, `count * size_of::<T>()`, computed on mathematical integers (without
-    ///   "wrapping around"), must fit in an `isize`.
-    ///
-    /// * If the computed offset is non-zero, then `self` must be [derived from][crate::ptr#provenance] a pointer to some
-    ///   [allocated object], and the entire memory range between `self` and the result must be in
-    ///   bounds of that allocated object. In particular, this range must not "wrap around" the edge
-    ///   of the address space.
-    ///
-    /// Allocated objects can never be larger than `isize::MAX` bytes, so if the computed offset
-    /// stays in bounds of the allocated object, it is guaranteed to satisfy the first requirement.
-    /// This implies, for instance, that `vec.as_ptr().add(vec.len())` (for `vec: Vec<T>`) is always
-    /// safe.
-    ///
-    /// Consider using [`wrapping_offset`] instead if these constraints are
-    /// difficult to satisfy. The only advantage of this method is that it
-    /// enables more aggressive compiler optimizations.
-    ///
-    /// [`wrapping_offset`]: #method.wrapping_offset
-    /// [allocated object]: crate::ptr#allocated-object
+    #[doc = include_str!("./docs/offset.md")]
     ///
     /// # Examples
     ///
@@ -996,44 +969,13 @@ pub const fn guaranteed_ne(self, other: *mut T) -> Option<bool>
         unsafe { (self as *const T).byte_offset_from_unsigned(origin) }
     }
 
-    /// Adds an unsigned offset to a pointer.
-    ///
-    /// This can only move the pointer forward (or not move it). If you need to move forward or
-    /// backward depending on the value, then you might want [`offset`](#method.offset) instead
-    /// which takes a signed offset.
-    ///
-    /// `count` is in units of T; e.g., a `count` of 3 represents a pointer
-    /// offset of `3 * size_of::<T>()` bytes.
-    ///
-    /// # Safety
-    ///
-    /// If any of the following conditions are violated, the result is Undefined Behavior:
-    ///
-    /// * The offset in bytes, `count * size_of::<T>()`, computed on mathematical integers (without
-    ///   "wrapping around"), must fit in an `isize`.
-    ///
-    /// * If the computed offset is non-zero, then `self` must be [derived from][crate::ptr#provenance] a pointer to some
-    ///   [allocated object], and the entire memory range between `self` and the result must be in
-    ///   bounds of that allocated object. In particular, this range must not "wrap around" the edge
-    ///   of the address space.
-    ///
-    /// Allocated objects can never be larger than `isize::MAX` bytes, so if the computed offset
-    /// stays in bounds of the allocated object, it is guaranteed to satisfy the first requirement.
-    /// This implies, for instance, that `vec.as_ptr().add(vec.len())` (for `vec: Vec<T>`) is always
-    /// safe.
-    ///
-    /// Consider using [`wrapping_add`] instead if these constraints are
-    /// difficult to satisfy. The only advantage of this method is that it
-    /// enables more aggressive compiler optimizations.
-    ///
-    /// [`wrapping_add`]: #method.wrapping_add
-    /// [allocated object]: crate::ptr#allocated-object
+    #[doc = include_str!("./docs/add.md")]
     ///
     /// # Examples
     ///
     /// ```
-    /// let s: &str = "123";
-    /// let ptr: *const u8 = s.as_ptr();
+    /// let mut s: String = "123".to_string();
+    /// let ptr: *mut u8 = s.as_mut_ptr();
     ///
     /// unsafe {
     ///     assert_eq!('2', *ptr.add(1) as char);

library/core/src/sync/atomic.rs

@@ -3822,6 +3822,7 @@ unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
 
 #[inline]
 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
+#[cfg(bootstrap)]
 unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_load`.
     unsafe {
@@ -3835,6 +3836,23 @@ unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
     }
 }
 
+#[inline]
+#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
+#[cfg(not(bootstrap))]
+unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
+    use intrinsics::AtomicOrdering;
+    // SAFETY: the caller must uphold the safety contract for `atomic_load`.
+    unsafe {
+        match order {
+            Relaxed => intrinsics::atomic_load::<T, { AtomicOrdering::Relaxed }>(dst),
+            Acquire => intrinsics::atomic_load::<T, { AtomicOrdering::Acquire }>(dst),
+            SeqCst => intrinsics::atomic_load::<T, { AtomicOrdering::SeqCst }>(dst),
+            Release => panic!("there is no such thing as a release load"),
+            AcqRel => panic!("there is no such thing as an acquire-release load"),
+        }
+    }
+}
+
 #[inline]
 #[cfg(target_has_atomic)]
 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces

library/coretests/tests/floats/f128.rs

@@ -1,12 +1,9 @@
 // FIXME(f16_f128): only tested on platforms that have symbols and aren't buggy
 #![cfg(target_has_reliable_f128)]
 
+use core::ops::{Add, Div, Mul, Sub};
 use std::f128::consts;
 use std::num::FpCategory as Fp;
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
-use std::ops::Rem;
-use std::ops::{Add, Div, Mul, Sub};
 
 // Note these tolerances make sense around zero, but not for more extreme exponents.
 
@@ -39,68 +36,46 @@
 /// Second pattern over the mantissa
 const NAN_MASK2: u128 = 0x00005555555555555555555555555555;
 
-/// Compare by representation
-#[allow(unused_macros)]
-macro_rules! assert_f128_biteq {
-    ($a:expr, $b:expr) => {
-        let (l, r): (&f128, &f128) = (&$a, &$b);
-        let lb = l.to_bits();
-        let rb = r.to_bits();
-        assert_eq!(lb, rb, "float {l:?} is not bitequal to {r:?}.\na: {lb:#034x}\nb: {rb:#034x}");
-    };
-}
-
 #[test]
 fn test_num_f128() {
     // FIXME(f16_f128): replace with a `test_num` call once the required `fmodl`/`fmodf128`
     // function is available on all platforms.
     let ten = 10f128;
     let two = 2f128;
-    assert_eq!(ten.add(two), ten + two);
-    assert_eq!(ten.sub(two), ten - two);
-    assert_eq!(ten.mul(two), ten * two);
-    assert_eq!(ten.div(two), ten / two);
+    assert_biteq!(ten.add(two), ten + two);
+    assert_biteq!(ten.sub(two), ten - two);
+    assert_biteq!(ten.mul(two), ten * two);
+    assert_biteq!(ten.div(two), ten / two);
+    #[cfg(any(miri, target_has_reliable_f128_math))]
+    assert_biteq!(core::ops::Rem::rem(ten, two), ten % two);
 }
 
 // FIXME(f16_f128,miri): many of these have to be disabled since miri does not yet support
 // the intrinsics.
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
-fn test_num_f128_rem() {
-    let ten = 10f128;
-    let two = 2f128;
-    assert_eq!(ten.rem(two), ten % two);
-}
-
-#[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_min_nan() {
-    assert_eq!(f128::NAN.min(2.0), 2.0);
-    assert_eq!(2.0f128.min(f128::NAN), 2.0);
+    assert_biteq!(f128::NAN.min(2.0), 2.0);
+    assert_biteq!(2.0f128.min(f128::NAN), 2.0);
 }
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_max_nan() {
-    assert_eq!(f128::NAN.max(2.0), 2.0);
-    assert_eq!(2.0f128.max(f128::NAN), 2.0);
+    assert_biteq!(f128::NAN.max(2.0), 2.0);
+    assert_biteq!(2.0f128.max(f128::NAN), 2.0);
 }
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_minimum() {
     assert!(f128::NAN.minimum(2.0).is_nan());
     assert!(2.0f128.minimum(f128::NAN).is_nan());
 }
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_maximum() {
     assert!(f128::NAN.maximum(2.0).is_nan());
     assert!(2.0f128.maximum(f128::NAN).is_nan());
@@ -147,7 +122,7 @@ fn test_neg_infinity() {
 #[test]
 fn test_zero() {
     let zero: f128 = 0.0f128;
-    assert_eq!(0.0, zero);
+    assert_biteq!(0.0, zero);
     assert!(!zero.is_infinite());
     assert!(zero.is_finite());
     assert!(zero.is_sign_positive());
@@ -161,6 +136,7 @@ fn test_zero() {
 fn test_neg_zero() {
     let neg_zero: f128 = -0.0;
     assert_eq!(0.0, neg_zero);
+    assert_biteq!(-0.0, neg_zero);
     assert!(!neg_zero.is_infinite());
     assert!(neg_zero.is_finite());
     assert!(!neg_zero.is_sign_positive());
@@ -173,7 +149,7 @@ fn test_neg_zero() {
 #[test]
 fn test_one() {
     let one: f128 = 1.0f128;
-    assert_eq!(1.0, one);
+    assert_biteq!(1.0, one);
     assert!(!one.is_infinite());
     assert!(one.is_finite());
     assert!(one.is_sign_positive());
@@ -257,114 +233,107 @@ fn test_classify() {
 }
 
 #[test]
-#[cfg(not(miri))]
 #[cfg(target_has_reliable_f128_math)]
 fn test_floor() {
-    assert_approx_eq!(1.0f128.floor(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.3f128.floor(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.5f128.floor(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.7f128.floor(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(0.0f128.floor(), 0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-0.0f128).floor(), -0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.0f128).floor(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.3f128).floor(), -2.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.5f128).floor(), -2.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.7f128).floor(), -2.0f128, TOL_PRECISE);
+    assert_biteq!(1.0f128.floor(), 1.0f128);
+    assert_biteq!(1.3f128.floor(), 1.0f128);
+    assert_biteq!(1.5f128.floor(), 1.0f128);
+    assert_biteq!(1.7f128.floor(), 1.0f128);
+    assert_biteq!(0.0f128.floor(), 0.0f128);
+    assert_biteq!((-0.0f128).floor(), -0.0f128);
+    assert_biteq!((-1.0f128).floor(), -1.0f128);
+    assert_biteq!((-1.3f128).floor(), -2.0f128);
+    assert_biteq!((-1.5f128).floor(), -2.0f128);
+    assert_biteq!((-1.7f128).floor(), -2.0f128);
 }
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_ceil() {
-    assert_approx_eq!(1.0f128.ceil(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.3f128.ceil(), 2.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.5f128.ceil(), 2.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.7f128.ceil(), 2.0f128, TOL_PRECISE);
-    assert_approx_eq!(0.0f128.ceil(), 0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-0.0f128).ceil(), -0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.0f128).ceil(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.3f128).ceil(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.5f128).ceil(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.7f128).ceil(), -1.0f128, TOL_PRECISE);
+    assert_biteq!(1.0f128.ceil(), 1.0f128);
+    assert_biteq!(1.3f128.ceil(), 2.0f128);
+    assert_biteq!(1.5f128.ceil(), 2.0f128);
+    assert_biteq!(1.7f128.ceil(), 2.0f128);
+    assert_biteq!(0.0f128.ceil(), 0.0f128);
+    assert_biteq!((-0.0f128).ceil(), -0.0f128);
+    assert_biteq!((-1.0f128).ceil(), -1.0f128);
+    assert_biteq!((-1.3f128).ceil(), -1.0f128);
+    assert_biteq!((-1.5f128).ceil(), -1.0f128);
+    assert_biteq!((-1.7f128).ceil(), -1.0f128);
 }
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_round() {
-    assert_approx_eq!(2.5f128.round(), 3.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.0f128.round(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.3f128.round(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.5f128.round(), 2.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.7f128.round(), 2.0f128, TOL_PRECISE);
-    assert_approx_eq!(0.0f128.round(), 0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-0.0f128).round(), -0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.0f128).round(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.3f128).round(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.5f128).round(), -2.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.7f128).round(), -2.0f128, TOL_PRECISE);
+    assert_biteq!(2.5f128.round(), 3.0f128);
+    assert_biteq!(1.0f128.round(), 1.0f128);
+    assert_biteq!(1.3f128.round(), 1.0f128);
+    assert_biteq!(1.5f128.round(), 2.0f128);
+    assert_biteq!(1.7f128.round(), 2.0f128);
+    assert_biteq!(0.0f128.round(), 0.0f128);
+    assert_biteq!((-0.0f128).round(), -0.0f128);
+    assert_biteq!((-1.0f128).round(), -1.0f128);
+    assert_biteq!((-1.3f128).round(), -1.0f128);
+    assert_biteq!((-1.5f128).round(), -2.0f128);
+    assert_biteq!((-1.7f128).round(), -2.0f128);
 }
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_round_ties_even() {
-    assert_approx_eq!(2.5f128.round_ties_even(), 2.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.0f128.round_ties_even(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.3f128.round_ties_even(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.5f128.round_ties_even(), 2.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.7f128.round_ties_even(), 2.0f128, TOL_PRECISE);
-    assert_approx_eq!(0.0f128.round_ties_even(), 0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-0.0f128).round_ties_even(), -0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.0f128).round_ties_even(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.3f128).round_ties_even(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.5f128).round_ties_even(), -2.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.7f128).round_ties_even(), -2.0f128, TOL_PRECISE);
+    assert_biteq!(2.5f128.round_ties_even(), 2.0f128);
+    assert_biteq!(1.0f128.round_ties_even(), 1.0f128);
+    assert_biteq!(1.3f128.round_ties_even(), 1.0f128);
+    assert_biteq!(1.5f128.round_ties_even(), 2.0f128);
+    assert_biteq!(1.7f128.round_ties_even(), 2.0f128);
+    assert_biteq!(0.0f128.round_ties_even(), 0.0f128);
+    assert_biteq!((-0.0f128).round_ties_even(), -0.0f128);
+    assert_biteq!((-1.0f128).round_ties_even(), -1.0f128);
+    assert_biteq!((-1.3f128).round_ties_even(), -1.0f128);
+    assert_biteq!((-1.5f128).round_ties_even(), -2.0f128);
+    assert_biteq!((-1.7f128).round_ties_even(), -2.0f128);
 }
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_trunc() {
-    assert_approx_eq!(1.0f128.trunc(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.3f128.trunc(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.5f128.trunc(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.7f128.trunc(), 1.0f128, TOL_PRECISE);
-    assert_approx_eq!(0.0f128.trunc(), 0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-0.0f128).trunc(), -0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.0f128).trunc(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.3f128).trunc(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.5f128).trunc(), -1.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.7f128).trunc(), -1.0f128, TOL_PRECISE);
+    assert_biteq!(1.0f128.trunc(), 1.0f128);
+    assert_biteq!(1.3f128.trunc(), 1.0f128);
+    assert_biteq!(1.5f128.trunc(), 1.0f128);
+    assert_biteq!(1.7f128.trunc(), 1.0f128);
+    assert_biteq!(0.0f128.trunc(), 0.0f128);
+    assert_biteq!((-0.0f128).trunc(), -0.0f128);
+    assert_biteq!((-1.0f128).trunc(), -1.0f128);
+    assert_biteq!((-1.3f128).trunc(), -1.0f128);
+    assert_biteq!((-1.5f128).trunc(), -1.0f128);
+    assert_biteq!((-1.7f128).trunc(), -1.0f128);
 }
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_fract() {
-    assert_approx_eq!(1.0f128.fract(), 0.0f128, TOL_PRECISE);
-    assert_approx_eq!(1.3f128.fract(), 0.3f128, TOL_PRECISE);
-    assert_approx_eq!(1.5f128.fract(), 0.5f128, TOL_PRECISE);
-    assert_approx_eq!(1.7f128.fract(), 0.7f128, TOL_PRECISE);
-    assert_approx_eq!(0.0f128.fract(), 0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-0.0f128).fract(), -0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.0f128).fract(), -0.0f128, TOL_PRECISE);
-    assert_approx_eq!((-1.3f128).fract(), -0.3f128, TOL_PRECISE);
-    assert_approx_eq!((-1.5f128).fract(), -0.5f128, TOL_PRECISE);
-    assert_approx_eq!((-1.7f128).fract(), -0.7f128, TOL_PRECISE);
+    assert_biteq!(1.0f128.fract(), 0.0f128);
+    assert_biteq!(1.3f128.fract(), 0.300000000000000000000000000000000039f128);
+    assert_biteq!(1.5f128.fract(), 0.5f128);
+    assert_biteq!(1.7f128.fract(), 0.7f128);
+    assert_biteq!(0.0f128.fract(), 0.0f128);
+    assert_biteq!((-0.0f128).fract(), 0.0f128);
+    assert_biteq!((-1.0f128).fract(), 0.0f128);
+    assert_biteq!((-1.3f128).fract(), -0.300000000000000000000000000000000039f128);
+    assert_biteq!((-1.5f128).fract(), -0.5f128);
+    assert_biteq!((-1.7f128).fract(), -0.699999999999999999999999999999999961f128);
 }
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_abs() {
-    assert_eq!(f128::INFINITY.abs(), f128::INFINITY);
-    assert_eq!(1f128.abs(), 1f128);
-    assert_eq!(0f128.abs(), 0f128);
-    assert_eq!((-0f128).abs(), 0f128);
-    assert_eq!((-1f128).abs(), 1f128);
-    assert_eq!(f128::NEG_INFINITY.abs(), f128::INFINITY);
-    assert_eq!((1f128 / f128::NEG_INFINITY).abs(), 0f128);
+    assert_biteq!(f128::INFINITY.abs(), f128::INFINITY);
+    assert_biteq!(1f128.abs(), 1f128);
+    assert_biteq!(0f128.abs(), 0f128);
+    assert_biteq!((-0f128).abs(), 0f128);
+    assert_biteq!((-1f128).abs(), 1f128);
+    assert_biteq!(f128::NEG_INFINITY.abs(), f128::INFINITY);
+    assert_biteq!((1f128 / f128::NEG_INFINITY).abs(), 0f128);
     assert!(f128::NAN.abs().is_nan());
 }
 
@@ -401,27 +370,27 @@ fn test_next_up() {
     let max_down = f128::from_bits(MAX_DOWN_BITS);
     let largest_subnormal = f128::from_bits(LARGEST_SUBNORMAL_BITS);
     let smallest_normal = f128::from_bits(SMALLEST_NORMAL_BITS);
-    assert_f128_biteq!(f128::NEG_INFINITY.next_up(), f128::MIN);
-    assert_f128_biteq!(f128::MIN.next_up(), -max_down);
-    assert_f128_biteq!((-1.0 - f128::EPSILON).next_up(), -1.0);
-    assert_f128_biteq!((-smallest_normal).next_up(), -largest_subnormal);
-    assert_f128_biteq!((-tiny_up).next_up(), -tiny);
-    assert_f128_biteq!((-tiny).next_up(), -0.0f128);
-    assert_f128_biteq!((-0.0f128).next_up(), tiny);
-    assert_f128_biteq!(0.0f128.next_up(), tiny);
-    assert_f128_biteq!(tiny.next_up(), tiny_up);
-    assert_f128_biteq!(largest_subnormal.next_up(), smallest_normal);
-    assert_f128_biteq!(1.0f128.next_up(), 1.0 + f128::EPSILON);
-    assert_f128_biteq!(f128::MAX.next_up(), f128::INFINITY);
-    assert_f128_biteq!(f128::INFINITY.next_up(), f128::INFINITY);
+    assert_biteq!(f128::NEG_INFINITY.next_up(), f128::MIN);
+    assert_biteq!(f128::MIN.next_up(), -max_down);
+    assert_biteq!((-1.0 - f128::EPSILON).next_up(), -1.0f128);
+    assert_biteq!((-smallest_normal).next_up(), -largest_subnormal);
+    assert_biteq!((-tiny_up).next_up(), -tiny);
+    assert_biteq!((-tiny).next_up(), -0.0f128);
+    assert_biteq!((-0.0f128).next_up(), tiny);
+    assert_biteq!(0.0f128.next_up(), tiny);
+    assert_biteq!(tiny.next_up(), tiny_up);
+    assert_biteq!(largest_subnormal.next_up(), smallest_normal);
+    assert_biteq!(1.0f128.next_up(), 1.0 + f128::EPSILON);
+    assert_biteq!(f128::MAX.next_up(), f128::INFINITY);
+    assert_biteq!(f128::INFINITY.next_up(), f128::INFINITY);
 
     // Check that NaNs roundtrip.
     let nan0 = f128::NAN;
     let nan1 = f128::from_bits(f128::NAN.to_bits() ^ 0x002a_aaaa);
     let nan2 = f128::from_bits(f128::NAN.to_bits() ^ 0x0055_5555);
-    assert_f128_biteq!(nan0.next_up(), nan0);
-    assert_f128_biteq!(nan1.next_up(), nan1);
-    assert_f128_biteq!(nan2.next_up(), nan2);
+    assert_biteq!(nan0.next_up(), nan0);
+    assert_biteq!(nan1.next_up(), nan1);
+    assert_biteq!(nan2.next_up(), nan2);
 }
 
 #[test]
@@ -431,28 +400,28 @@ fn test_next_down() {
     let max_down = f128::from_bits(MAX_DOWN_BITS);
     let largest_subnormal = f128::from_bits(LARGEST_SUBNORMAL_BITS);
     let smallest_normal = f128::from_bits(SMALLEST_NORMAL_BITS);
-    assert_f128_biteq!(f128::NEG_INFINITY.next_down(), f128::NEG_INFINITY);
-    assert_f128_biteq!(f128::MIN.next_down(), f128::NEG_INFINITY);
-    assert_f128_biteq!((-max_down).next_down(), f128::MIN);
-    assert_f128_biteq!((-1.0f128).next_down(), -1.0 - f128::EPSILON);
-    assert_f128_biteq!((-largest_subnormal).next_down(), -smallest_normal);
-    assert_f128_biteq!((-tiny).next_down(), -tiny_up);
-    assert_f128_biteq!((-0.0f128).next_down(), -tiny);
-    assert_f128_biteq!((0.0f128).next_down(), -tiny);
-    assert_f128_biteq!(tiny.next_down(), 0.0f128);
-    assert_f128_biteq!(tiny_up.next_down(), tiny);
-    assert_f128_biteq!(smallest_normal.next_down(), largest_subnormal);
-    assert_f128_biteq!((1.0 + f128::EPSILON).next_down(), 1.0f128);
-    assert_f128_biteq!(f128::MAX.next_down(), max_down);
-    assert_f128_biteq!(f128::INFINITY.next_down(), f128::MAX);
+    assert_biteq!(f128::NEG_INFINITY.next_down(), f128::NEG_INFINITY);
+    assert_biteq!(f128::MIN.next_down(), f128::NEG_INFINITY);
+    assert_biteq!((-max_down).next_down(), f128::MIN);
+    assert_biteq!((-1.0f128).next_down(), -1.0 - f128::EPSILON);
+    assert_biteq!((-largest_subnormal).next_down(), -smallest_normal);
+    assert_biteq!((-tiny).next_down(), -tiny_up);
+    assert_biteq!((-0.0f128).next_down(), -tiny);
+    assert_biteq!((0.0f128).next_down(), -tiny);
+    assert_biteq!(tiny.next_down(), 0.0f128);
+    assert_biteq!(tiny_up.next_down(), tiny);
+    assert_biteq!(smallest_normal.next_down(), largest_subnormal);
+    assert_biteq!((1.0 + f128::EPSILON).next_down(), 1.0f128);
+    assert_biteq!(f128::MAX.next_down(), max_down);
+    assert_biteq!(f128::INFINITY.next_down(), f128::MAX);
 
     // Check that NaNs roundtrip.
     let nan0 = f128::NAN;
     let nan1 = f128::from_bits(f128::NAN.to_bits() ^ 0x002a_aaaa);
     let nan2 = f128::from_bits(f128::NAN.to_bits() ^ 0x0055_5555);
-    assert_f128_biteq!(nan0.next_down(), nan0);
-    assert_f128_biteq!(nan1.next_down(), nan1);
-    assert_f128_biteq!(nan2.next_down(), nan2);
+    assert_biteq!(nan0.next_down(), nan0);
+    assert_biteq!(nan1.next_down(), nan1);
+    assert_biteq!(nan2.next_down(), nan2);
 }
 
 #[test]
@@ -462,36 +431,35 @@ fn test_mul_add() {
     let nan: f128 = f128::NAN;
     let inf: f128 = f128::INFINITY;
     let neg_inf: f128 = f128::NEG_INFINITY;
-    assert_approx_eq!(12.3f128.mul_add(4.5, 6.7), 62.05, TOL_PRECISE);
-    assert_approx_eq!((-12.3f128).mul_add(-4.5, -6.7), 48.65, TOL_PRECISE);
-    assert_approx_eq!(0.0f128.mul_add(8.9, 1.2), 1.2, TOL_PRECISE);
-    assert_approx_eq!(3.4f128.mul_add(-0.0, 5.6), 5.6, TOL_PRECISE);
+    assert_biteq!(12.3f128.mul_add(4.5, 6.7), 62.0500000000000000000000000000000037);
+    assert_biteq!((-12.3f128).mul_add(-4.5, -6.7), 48.6500000000000000000000000000000049);
+    assert_biteq!(0.0f128.mul_add(8.9, 1.2), 1.2);
+    assert_biteq!(3.4f128.mul_add(-0.0, 5.6), 5.6);
     assert!(nan.mul_add(7.8, 9.0).is_nan());
-    assert_eq!(inf.mul_add(7.8, 9.0), inf);
-    assert_eq!(neg_inf.mul_add(7.8, 9.0), neg_inf);
-    assert_eq!(8.9f128.mul_add(inf, 3.2), inf);
-    assert_eq!((-3.2f128).mul_add(2.4, neg_inf), neg_inf);
+    assert_biteq!(inf.mul_add(7.8, 9.0), inf);
+    assert_biteq!(neg_inf.mul_add(7.8, 9.0), neg_inf);
+    assert_biteq!(8.9f128.mul_add(inf, 3.2), inf);
+    assert_biteq!((-3.2f128).mul_add(2.4, neg_inf), neg_inf);
 }
 
 #[test]
-#[cfg(not(miri))]
-#[cfg(target_has_reliable_f128_math)]
+#[cfg(any(miri, target_has_reliable_f128_math))]
 fn test_recip() {
     let nan: f128 = f128::NAN;
     let inf: f128 = f128::INFINITY;
     let neg_inf: f128 = f128::NEG_INFINITY;
-    assert_eq!(1.0f128.recip(), 1.0);
-    assert_eq!(2.0f128.recip(), 0.5);
-    assert_eq!((-0.4f128).recip(), -2.5);
-    assert_eq!(0.0f128.recip(), inf);
+    assert_biteq!(1.0f128.recip(), 1.0);
+    assert_biteq!(2.0f128.recip(), 0.5);
+    assert_biteq!((-0.4f128).recip(), -2.5);
+    assert_biteq!(0.0f128.recip(), inf);
     assert_approx_eq!(
         f128::MAX.recip(),
         8.40525785778023376565669454330438228902076605e-4933,
         1e-4900
     );
     assert!(nan.recip().is_nan());
-    assert_eq!(inf.recip(), 0.0);
-    assert_eq!(neg_inf.recip(), 0.0);
+    assert_biteq!(inf.recip(), 0.0);
+    assert_biteq!(neg_inf.recip(), -0.0);
 }
 
 #[test]
@@ -501,13 +469,13 @@ fn test_powi() {
     let nan: f128 = f128::NAN;
     let inf: f128 = f128::INFINITY;
     let neg_inf: f128 = f128::NEG_INFINITY;
-    assert_eq!(1.0f128.powi(1), 1.0);
+    assert_biteq!(1.0f128.powi(1), 1.0);
     assert_approx_eq!((-3.1f128).powi(2), 9.6100000000000005506706202140776519387, TOL);
     assert_approx_eq!(5.9f128.powi(-2), 0.028727377190462507313100483690639638451, TOL);
-    assert_eq!(8.3f128.powi(0), 1.0);
+    assert_biteq!(8.3f128.powi(0), 1.0);
     assert!(nan.powi(2).is_nan());
-    assert_eq!(inf.powi(3), inf);
-    assert_eq!(neg_inf.powi(2), inf);
+    assert_biteq!(inf.powi(3), inf);
+    assert_biteq!(neg_inf.powi(2), inf);
 }
 
 #[test]
@@ -517,10 +485,10 @@ fn test_sqrt_domain() {
     assert!(f128::NAN.sqrt().is_nan());
     assert!(f128::NEG_INFINITY.sqrt().is_nan());
     assert!((-1.0f128).sqrt().is_nan());
-    assert_eq!((-0.0f128).sqrt(), -0.0);
-    assert_eq!(0.0f128.sqrt(), 0.0);
-    assert_eq!(1.0f128.sqrt(), 1.0);
-    assert_eq!(f128::INFINITY.sqrt(), f128::INFINITY);
+    assert_biteq!((-0.0f128).sqrt(), -0.0);
+    assert_biteq!(0.0f128.sqrt(), 0.0);
+    assert_biteq!(1.0f128.sqrt(), 1.0);
+    assert_biteq!(f128::INFINITY.sqrt(), f128::INFINITY);
 }
 
 #[test]
@@ -529,13 +497,13 @@ fn test_to_degrees() {
     let nan: f128 = f128::NAN;
     let inf: f128 = f128::INFINITY;
     let neg_inf: f128 = f128::NEG_INFINITY;
-    assert_eq!(0.0f128.to_degrees(), 0.0);
+    assert_biteq!(0.0f128.to_degrees(), 0.0);
     assert_approx_eq!((-5.8f128).to_degrees(), -332.31552117587745090765431723855668471, TOL);
     assert_approx_eq!(pi.to_degrees(), 180.0, TOL);
     assert!(nan.to_degrees().is_nan());
-    assert_eq!(inf.to_degrees(), inf);
-    assert_eq!(neg_inf.to_degrees(), neg_inf);
-    assert_eq!(1_f128.to_degrees(), 57.2957795130823208767981548141051703);
+    assert_biteq!(inf.to_degrees(), inf);
+    assert_biteq!(neg_inf.to_degrees(), neg_inf);
+    assert_biteq!(1_f128.to_degrees(), 57.2957795130823208767981548141051703);
 }
 
 #[test]
@@ -544,15 +512,15 @@ fn test_to_radians() {
     let nan: f128 = f128::NAN;
     let inf: f128 = f128::INFINITY;
     let neg_inf: f128 = f128::NEG_INFINITY;
-    assert_eq!(0.0f128.to_radians(), 0.0);
+    assert_biteq!(0.0f128.to_radians(), 0.0);
     assert_approx_eq!(154.6f128.to_radians(), 2.6982790235832334267135442069489767804, TOL);
     assert_approx_eq!((-332.31f128).to_radians(), -5.7999036373023566567593094812182763013, TOL);
     // check approx rather than exact because round trip for pi doesn't fall on an exactly
     // representable value (unlike `f32` and `f64`).
     assert_approx_eq!(180.0f128.to_radians(), pi, TOL_PRECISE);
     assert!(nan.to_radians().is_nan());
-    assert_eq!(inf.to_radians(), inf);
-    assert_eq!(neg_inf.to_radians(), neg_inf);
+    assert_biteq!(inf.to_radians(), inf);
+    assert_biteq!(neg_inf.to_radians(), neg_inf);
 }
 
 #[test]
@@ -561,10 +529,10 @@ fn test_float_bits_conv() {
     assert_eq!((12.5f128).to_bits(), 0x40029000000000000000000000000000);
     assert_eq!((1337f128).to_bits(), 0x40094e40000000000000000000000000);
     assert_eq!((-14.25f128).to_bits(), 0xc002c800000000000000000000000000);
-    assert_approx_eq!(f128::from_bits(0x3fff0000000000000000000000000000), 1.0, TOL_PRECISE);
-    assert_approx_eq!(f128::from_bits(0x40029000000000000000000000000000), 12.5, TOL_PRECISE);
-    assert_approx_eq!(f128::from_bits(0x40094e40000000000000000000000000), 1337.0, TOL_PRECISE);
-    assert_approx_eq!(f128::from_bits(0xc002c800000000000000000000000000), -14.25, TOL_PRECISE);
+    assert_biteq!(f128::from_bits(0x3fff0000000000000000000000000000), 1.0);
+    assert_biteq!(f128::from_bits(0x40029000000000000000000000000000), 12.5);
+    assert_biteq!(f128::from_bits(0x40094e40000000000000000000000000), 1337.0);
+    assert_biteq!(f128::from_bits(0xc002c800000000000000000000000000), -14.25);
 
     // Check that NaNs roundtrip their bits regardless of signaling-ness
     // 0xA is 0b1010; 0x5 is 0b0101 -- so these two together clobbers all the mantissa bits
@@ -760,26 +728,26 @@ fn test_algebraic() {
 
 #[test]
 fn test_from() {
-    assert_eq!(f128::from(false), 0.0);
-    assert_eq!(f128::from(true), 1.0);
-    assert_eq!(f128::from(u8::MIN), 0.0);
-    assert_eq!(f128::from(42_u8), 42.0);
-    assert_eq!(f128::from(u8::MAX), 255.0);
-    assert_eq!(f128::from(i8::MIN), -128.0);
-    assert_eq!(f128::from(42_i8), 42.0);
-    assert_eq!(f128::from(i8::MAX), 127.0);
-    assert_eq!(f128::from(u16::MIN), 0.0);
-    assert_eq!(f128::from(42_u16), 42.0);
-    assert_eq!(f128::from(u16::MAX), 65535.0);
-    assert_eq!(f128::from(i16::MIN), -32768.0);
-    assert_eq!(f128::from(42_i16), 42.0);
-    assert_eq!(f128::from(i16::MAX), 32767.0);
-    assert_eq!(f128::from(u32::MIN), 0.0);
-    assert_eq!(f128::from(42_u32), 42.0);
-    assert_eq!(f128::from(u32::MAX), 4294967295.0);
-    assert_eq!(f128::from(i32::MIN), -2147483648.0);
-    assert_eq!(f128::from(42_i32), 42.0);
-    assert_eq!(f128::from(i32::MAX), 2147483647.0);
+    assert_biteq!(f128::from(false), 0.0);
+    assert_biteq!(f128::from(true), 1.0);
+    assert_biteq!(f128::from(u8::MIN), 0.0);
+    assert_biteq!(f128::from(42_u8), 42.0);
+    assert_biteq!(f128::from(u8::MAX), 255.0);
+    assert_biteq!(f128::from(i8::MIN), -128.0);
+    assert_biteq!(f128::from(42_i8), 42.0);
+    assert_biteq!(f128::from(i8::MAX), 127.0);
+    assert_biteq!(f128::from(u16::MIN), 0.0);
+    assert_biteq!(f128::from(42_u16), 42.0);
+    assert_biteq!(f128::from(u16::MAX), 65535.0);
+    assert_biteq!(f128::from(i16::MIN), -32768.0);
+    assert_biteq!(f128::from(42_i16), 42.0);
+    assert_biteq!(f128::from(i16::MAX), 32767.0);
+    assert_biteq!(f128::from(u32::MIN), 0.0);
+    assert_biteq!(f128::from(42_u32), 42.0);
+    assert_biteq!(f128::from(u32::MAX), 4294967295.0);
+    assert_biteq!(f128::from(i32::MIN), -2147483648.0);
+    assert_biteq!(f128::from(42_i32), 42.0);
+    assert_biteq!(f128::from(i32::MAX), 2147483647.0);
     // FIXME(f16_f128): Uncomment these tests once the From<{u64,i64}> impls are added.
     // assert_eq!(f128::from(u64::MIN), 0.0);
     // assert_eq!(f128::from(42_u64), 42.0);