Auto merge of #149514 - adamgemmell:dev/adagem01/bootstrap-to-library, r=Kobzol

Move bootstrap configuration to library workspace

This creates a new "dist" profile in the standard library which contains configuration for the distributed std artifacts previously contained in bootstrap, in order for a future build-std implementation to use. bootstrap.toml settings continue to override these defaults, as would any RUSTFLAGS provided. I've left some cargo features driven by bootstrap for a future patch.

Unfortunately, profiles aren't expressive enough to express per-target overrides, so [this risc-v example](https://github.com/rust-lang/rust/blob/c8f22ca269a1f2653ac962fe2bc21105065fd6cd/src/bootstrap/src/core/build_steps/compile.rs#L692) was not able to be moved across. This could go in its own profile which Cargo would have to know to use, and then the panic-abort rustflags overrides would need duplicating again. Doesn't seem like a sustainable solution as a couple similar overrides would explode the number of lines here.

We could use a cargo config in the library workspace for this, but this then would have to be respected by Cargo's build-std implementation and I'm not yet sure about the tradeoffs there.

This patch also introduces a build probe to deal with the test crate's stability which is obviously not ideal, I'm open to other solutions here or can back that change out for now if anyone prefers.

cc @Mark-Simulacrum https://github.com/rust-lang/rfcs/pull/3874

library/Cargo.toml

@@ -54,6 +54,32 @@ rustflags = ["-Cpanic=abort"]
 [profile.release.package.panic_abort]
 rustflags = ["-Cpanic=abort"]
 
+# The "dist" profile is used by bootstrap for prebuilt libstd artifacts
+# These settings ensure that the prebuilt artifacts support a variety of features
+# in the user's profile.
+[profile.dist]
+inherits = "release"
+codegen-units = 1
+debug = 1 # "limited"
+rustflags = [
+    # `profile.lto=off` implies `-Cembed-bitcode=no`, but unconditionally embedding
+    # bitcode is necessary for when users enable LTO.
+    # Required until Cargo can re-build the standard library based on the value
+    # of `profile.lto` in the user's profile.
+    "-Cembed-bitcode=yes",
+    # Enable frame pointers
+    "-Zunstable-options",
+    "-Cforce-frame-pointers=non-leaf",
+]
+
+[profile.dist.package.panic_abort]
+rustflags = [
+    "-Cpanic=abort",
+    "-Cembed-bitcode=yes",
+    "-Zunstable-options",
+    "-Cforce-frame-pointers=non-leaf",
+]
+
 [patch.crates-io]
 # See comments in `library/rustc-std-workspace-core/README.md` for what's going on here
 rustc-std-workspace-core = { path = 'rustc-std-workspace-core' }

library/test/build.rs

@@ -0,0 +1,11 @@
+fn main() {
+    println!("cargo:rustc-check-cfg=cfg(enable_unstable_features)");
+
+    let rustc = std::env::var("RUSTC").unwrap_or_else(|_| "rustc".into());
+    let version = std::process::Command::new(rustc).arg("-vV").output().unwrap();
+    let stdout = String::from_utf8(version.stdout).unwrap();
+
+    if stdout.contains("nightly") || stdout.contains("dev") {
+        println!("cargo:rustc-cfg=enable_unstable_features");
+    }
+}

library/test/src/cli.rs

@@ -314,15 +314,14 @@ fn parse_opts_impl(matches: getopts::Matches) -> OptRes {
     Ok(test_opts)
 }
 
-// FIXME: Copied from librustc_ast until linkage errors are resolved. Issue #47566
 fn is_nightly() -> bool {
-    // Whether this is a feature-staged build, i.e., on the beta or stable channel
-    let disable_unstable_features =
-        option_env!("CFG_DISABLE_UNSTABLE_FEATURES").map(|s| s != "0").unwrap_or(false);
-    // Whether we should enable unstable features for bootstrapping
+    // Whether the current rustc version should allow unstable features
+    let enable_unstable_features = cfg!(enable_unstable_features);
+
+    // The runtime override for unstable features
     let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok();
 
-    bootstrap || !disable_unstable_features
+    bootstrap || enable_unstable_features
 }
 
 // Gets the CLI options associated with `report-time` feature.

src/bootstrap/src/core/build_steps/compile.rs

@@ -626,12 +626,6 @@ pub fn std_cargo(
         CompilerBuiltins::BuildRustOnly => "",
     };
 
-    // `libtest` uses this to know whether or not to support
-    // `-Zunstable-options`.
-    if !builder.unstable_features() {
-        cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1");
-    }
-
     for krate in crates {
         cargo.args(["-p", krate]);
     }
@@ -680,13 +674,6 @@ pub fn std_cargo(
         }
     }
 
-    // By default, rustc uses `-Cembed-bitcode=yes`, and Cargo overrides that
-    // with `-Cembed-bitcode=no` for non-LTO builds. However, libstd must be
-    // built with bitcode so that the produced rlibs can be used for both LTO
-    // builds (which use bitcode) and non-LTO builds (which use object code).
-    // So we override the override here!
-    cargo.rustflag("-Cembed-bitcode=yes");
-
     if builder.config.rust_lto == RustcLto::Off {
         cargo.rustflag("-Clto=off");
     }
@@ -701,11 +688,6 @@ pub fn std_cargo(
         cargo.rustflag("-Cforce-unwind-tables=yes");
     }
 
-    // Enable frame pointers by default for the library. Note that they are still controlled by a
-    // separate setting for the compiler.
-    cargo.rustflag("-Zunstable-options");
-    cargo.rustflag("-Cforce-frame-pointers=non-leaf");
-
     let html_root =
         format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),);
     cargo.rustflag(&html_root);

src/bootstrap/src/core/build_steps/dist.rs

@@ -1008,7 +1008,7 @@ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
         let src = builder
             .stage_out(compiler, Mode::Std)
             .join(target)
-            .join(builder.cargo_dir())
+            .join(builder.cargo_dir(Mode::Std))
             .join("deps")
             .join("save-analysis");
 

src/bootstrap/src/core/build_steps/test.rs

@@ -927,8 +927,9 @@ fn run(self, builder: &Builder<'_>) {
 
         cargo.env("RUSTC_TEST_SUITE", builder.rustc(build_compiler));
         cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(build_compiler));
-        let host_libs =
-            builder.stage_out(build_compiler, Mode::ToolRustcPrivate).join(builder.cargo_dir());
+        let host_libs = builder
+            .stage_out(build_compiler, Mode::ToolRustcPrivate)
+            .join(builder.cargo_dir(Mode::ToolRustcPrivate));
         cargo.env("HOST_LIBS", host_libs);
 
         // Build the standard library that the tests can use.

src/bootstrap/src/core/build_steps/tool.rs

@@ -133,7 +133,7 @@ fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
                 RustcLto::ThinLocal => None,
             };
             if let Some(lto) = lto {
-                cargo.env(cargo_profile_var("LTO", &builder.config), lto);
+                cargo.env(cargo_profile_var("LTO", &builder.config, self.mode), lto);
             }
         }
 

src/bootstrap/src/core/builder/cargo.rs

@@ -106,9 +106,9 @@ pub struct Cargo {
     rustdocflags: Rustflags,
     hostflags: HostFlags,
     allow_features: String,
-    release_build: bool,
     build_compiler_stage: u32,
     extra_rustflags: Vec<String>,
+    profile: Option<&'static str>,
 }
 
 impl Cargo {
@@ -137,7 +137,11 @@ pub fn new(
     }
 
     pub fn release_build(&mut self, release_build: bool) {
-        self.release_build = release_build;
+        self.profile = if release_build { Some("release") } else { None };
+    }
+
+    pub fn profile(&mut self, profile: &'static str) {
+        self.profile = Some(profile);
     }
 
     pub fn compiler(&self) -> Compiler {
@@ -407,8 +411,8 @@ fn configure_linker(&mut self, builder: &Builder<'_>) -> &mut Cargo {
 
 impl From<Cargo> for BootstrapCommand {
     fn from(mut cargo: Cargo) -> BootstrapCommand {
-        if cargo.release_build {
-            cargo.args.insert(0, "--release".into());
+        if let Some(profile) = cargo.profile {
+            cargo.args.insert(0, format!("--profile={profile}").into());
         }
 
         for arg in &cargo.extra_rustflags {
@@ -598,7 +602,7 @@ fn cargo(
             build_stamp::clear_if_dirty(self, &my_out, &rustdoc);
         }
 
-        let profile_var = |name: &str| cargo_profile_var(name, &self.config);
+        let profile_var = |name: &str| cargo_profile_var(name, &self.config, mode);
 
         // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config
         // needs to not accidentally link to libLLVM in stage0/lib.
@@ -660,23 +664,15 @@ fn cargo(
             rustflags.arg(sysroot_str);
         }
 
-        let use_new_symbol_mangling = match self.config.rust_new_symbol_mangling {
-            Some(setting) => {
-                // If an explicit setting is given, use that
-                setting
+        let use_new_symbol_mangling = self.config.rust_new_symbol_mangling.or_else(|| {
+            if mode != Mode::Std {
+                // The compiler and tools default to the new scheme
+                Some(true)
+            } else {
+                // std follows the flag's default, which per compiler-team#938 is v0 on nightly
+                None
             }
-            // Per compiler-team#938, v0 mangling is used on nightly
-            None if self.config.channel == "dev" || self.config.channel == "nightly" => true,
-            None => {
-                if mode == Mode::Std {
-                    // The standard library defaults to the legacy scheme
-                    false
-                } else {
-                    // The compiler and tools default to the new scheme
-                    true
-                }
-            }
-        };
+        });
 
         // By default, windows-rs depends on a native library that doesn't get copied into the
         // sysroot. Passing this cfg enables raw-dylib support instead, which makes the native
@@ -687,10 +683,12 @@ fn cargo(
             rustflags.arg("--cfg=windows_raw_dylib");
         }
 
-        if use_new_symbol_mangling {
-            rustflags.arg("-Csymbol-mangling-version=v0");
-        } else {
-            rustflags.arg("-Csymbol-mangling-version=legacy");
+        if let Some(usm) = use_new_symbol_mangling {
+            rustflags.arg(if usm {
+                "-Csymbol-mangling-version=v0"
+            } else {
+                "-Csymbol-mangling-version=legacy"
+            });
         }
 
         // Always enable move/copy annotations for profiler visibility (non-stage0 only).
@@ -1434,9 +1432,18 @@ fn cargo(
             .unwrap_or(&self.config.rust_rustflags)
             .clone();
 
-        let release_build = self.config.rust_optimize.is_release() &&
-            // cargo bench/install do not accept `--release` and miri doesn't want it
-            !matches!(cmd_kind, Kind::Bench | Kind::Install | Kind::Miri | Kind::MiriSetup | Kind::MiriTest);
+        let profile =
+            if matches!(cmd_kind, Kind::Bench | Kind::Miri | Kind::MiriSetup | Kind::MiriTest) {
+                // Use the default profile for bench/miri
+                None
+            } else {
+                match (mode, self.config.rust_optimize.is_release()) {
+                    // Some std configuration exists in its own profile
+                    (Mode::Std, _) => Some("dist"),
+                    (_, true) => Some("release"),
+                    (_, false) => Some("dev"),
+                }
+            };
 
         Cargo {
             command: cargo,
@@ -1448,14 +1455,19 @@ fn cargo(
             rustdocflags,
             hostflags,
             allow_features,
-            release_build,
             build_compiler_stage,
             extra_rustflags,
+            profile,
         }
     }
 }
 
-pub fn cargo_profile_var(name: &str, config: &Config) -> String {
-    let profile = if config.rust_optimize.is_release() { "RELEASE" } else { "DEV" };
+pub fn cargo_profile_var(name: &str, config: &Config, mode: Mode) -> String {
+    let profile = match (mode, config.rust_optimize.is_release()) {
+        // Some std configuration exists in its own profile
+        (Mode::Std, _) => "DIST",
+        (_, true) => "RELEASE",
+        (_, false) => "DEV",
+    };
     format!("CARGO_PROFILE_{profile}_{name}")
 }

src/bootstrap/src/lib.rs

@@ -898,8 +898,12 @@ fn rustc_features(&self, kind: Kind, target: TargetSelection, crates: &[String])
 
     /// Component directory that Cargo will produce output into (e.g.
     /// release/debug)
-    fn cargo_dir(&self) -> &'static str {
-        if self.config.rust_optimize.is_release() { "release" } else { "debug" }
+    fn cargo_dir(&self, mode: Mode) -> &'static str {
+        match (mode, self.config.rust_optimize.is_release()) {
+            (Mode::Std, _) => "dist",
+            (_, true) => "release",
+            (_, false) => "debug",
+        }
     }
 
     fn tools_dir(&self, build_compiler: Compiler) -> PathBuf {
@@ -956,7 +960,7 @@ fn staged_tool(build_compiler: Compiler) -> (Option<u32>, &'static str) {
     /// running a particular compiler, whether or not we're building the
     /// standard library, and targeting the specified architecture.
     fn cargo_out(&self, build_compiler: Compiler, mode: Mode, target: TargetSelection) -> PathBuf {
-        self.stage_out(build_compiler, mode).join(target).join(self.cargo_dir())
+        self.stage_out(build_compiler, mode).join(target).join(self.cargo_dir(mode))
     }
 
     /// Root output directory of LLVM for `target`