Mirrors/rust
1
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2026-05-28 20:16:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1341 from bjorn3/build_sysroot_cleanup

Browse Source 
Refactor sysroot building
This commit is contained in:
bjorn3
2023-01-15 20:36:09 +01:00
committed by GitHub
parent b31b74e8fb 0f4df8fb0e
commit 611c5184ce
5 changed files with 173 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/main.yml
-1
View File
@@ -77,7 +77,6 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
rustup target add x86_64-pc-windows-gnu
- name: Install AArch64 toolchain and qemu
if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'aarch64-unknown-linux-gnu'
build_system/bench.rs
+3 -2
View File
@@ -4,7 +4,7 @@
use super::path::{Dirs, RelPath};
use super::prepare::GitRepo;
use super::rustc_info::{get_file_name, get_wrapper_file_name};
use super::rustc_info::get_file_name;
use super::utils::{hyperfine_command, is_ci, spawn_and_wait, CargoProject, Compiler};
pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
@@ -51,7 +51,8 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
.unwrap();
eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
let cargo_clif = RelPath::DIST.to_path(dirs).join(get_wrapper_file_name("cargo-clif", "bin"));
let cargo_clif =
RelPath::DIST.to_path(dirs).join(get_file_name("cargo_clif", "bin").replace('_', "-"));
let manifest_path = SIMPLE_RAYTRACER.manifest_path(dirs);
let target_dir = SIMPLE_RAYTRACER.target_dir(dirs);
build_system/build_sysroot.rs
+169 -113
View File
@@ -1,18 +1,15 @@
use std::fs;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::process::{self, Command};
use super::path::{Dirs, RelPath};
use super::rustc_info::{
get_file_name, get_rustc_version, get_toolchain_name, get_wrapper_file_name,
};
use super::rustc_info::{get_file_name, get_rustc_version, get_toolchain_name};
use super::utils::{spawn_and_wait, try_hard_link, CargoProject, Compiler};
use super::SysrootKind;
static DIST_DIR: RelPath = RelPath::DIST;
static BIN_DIR: RelPath = RelPath::DIST.join("bin");
static LIB_DIR: RelPath = RelPath::DIST.join("lib");
static RUSTLIB_DIR: RelPath = LIB_DIR.join("rustlib");
pub(crate) fn build_sysroot(
dirs: &Dirs,
@@ -39,145 +36,169 @@ pub(crate) fn build_sysroot(
LIB_DIR
}
.to_path(dirs)
.join(get_file_name("rustc_codegen_cranelift", "dylib"));
.join(cg_clif_dylib_src.file_name().unwrap());
try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path);
// Build and copy rustc and cargo wrappers
let wrapper_base_name = get_file_name("____", "bin");
let toolchain_name = get_toolchain_name();
for wrapper in ["rustc-clif", "rustdoc-clif", "cargo-clif"] {
let wrapper_name = get_wrapper_file_name(wrapper, "bin");
let wrapper_name = wrapper_base_name.replace("____", wrapper);
let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc);
build_cargo_wrapper_cmd
.env("TOOLCHAIN_NAME", get_toolchain_name())
.env("TOOLCHAIN_NAME", toolchain_name.clone())
.arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
.arg("-o")
.arg(DIST_DIR.to_path(dirs).join(wrapper_name))
.arg("-g");
.arg("-Cstrip=debuginfo");
spawn_and_wait(build_cargo_wrapper_cmd);
}
let default_sysroot = super::rustc_info::get_default_sysroot(&bootstrap_host_compiler.rustc);
let host = build_sysroot_for_triple(
dirs,
channel,
bootstrap_host_compiler.clone(),
&cg_clif_dylib_path,
sysroot_kind,
);
host.install_into_sysroot(&DIST_DIR.to_path(dirs));
let host_rustlib_lib =
RUSTLIB_DIR.to_path(dirs).join(&bootstrap_host_compiler.triple).join("lib");
let target_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(&target_triple).join("lib");
fs::create_dir_all(&host_rustlib_lib).unwrap();
fs::create_dir_all(&target_rustlib_lib).unwrap();
if target_triple == "x86_64-pc-windows-gnu" {
if !default_sysroot.join("lib").join("rustlib").join(&target_triple).join("lib").exists() {
eprintln!(
"The x86_64-pc-windows-gnu target needs to be installed first before it is possible \
to compile a sysroot for it.",
);
process::exit(1);
}
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(&target_triple).join("lib"),
)
.unwrap()
{
let file = file.unwrap().path();
if file.extension().map_or(true, |ext| ext.to_str().unwrap() != "o") {
continue; // only copy object files
}
try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
}
}
match sysroot_kind {
SysrootKind::None => {} // Nothing to do
SysrootKind::Llvm => {
for file in fs::read_dir(
default_sysroot
.join("lib")
.join("rustlib")
.join(&bootstrap_host_compiler.triple)
.join("lib"),
)
.unwrap()
if !is_native {
build_sysroot_for_triple(
dirs,
channel,
{
let file = file.unwrap().path();
let file_name_str = file.file_name().unwrap().to_str().unwrap();
if (file_name_str.contains("rustc_")
&& !file_name_str.contains("rustc_std_workspace_")
&& !file_name_str.contains("rustc_demangle"))
|| file_name_str.contains("chalk")
|| file_name_str.contains("tracing")
|| file_name_str.contains("regex")
{
// These are large crates that are part of the rustc-dev component and are not
// necessary to run regular programs.
continue;
}
try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap()));
}
let mut bootstrap_target_compiler = bootstrap_host_compiler.clone();
bootstrap_target_compiler.triple = target_triple.clone();
bootstrap_target_compiler.set_cross_linker_and_runner();
bootstrap_target_compiler
},
&cg_clif_dylib_path,
sysroot_kind,
)
.install_into_sysroot(&DIST_DIR.to_path(dirs));
}
if !is_native {
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(&target_triple).join("lib"),
)
.unwrap()
{
let file = file.unwrap().path();
try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
}
}
}
SysrootKind::Clif => {
build_clif_sysroot_for_triple(
dirs,
channel,
bootstrap_host_compiler.clone(),
&cg_clif_dylib_path,
);
if !is_native {
build_clif_sysroot_for_triple(
dirs,
channel,
{
let mut bootstrap_target_compiler = bootstrap_host_compiler.clone();
bootstrap_target_compiler.triple = target_triple.clone();
bootstrap_target_compiler.set_cross_linker_and_runner();
bootstrap_target_compiler
},
&cg_clif_dylib_path,
);
}
// Copy std for the host to the lib dir. This is necessary for the jit mode to find
// libstd.
for file in fs::read_dir(host_rustlib_lib).unwrap() {
let file = file.unwrap().path();
let filename = file.file_name().unwrap().to_str().unwrap();
if filename.contains("std-") && !filename.contains(".rlib") {
try_hard_link(&file, LIB_DIR.to_path(dirs).join(file.file_name().unwrap()));
}
}
// Copy std for the host to the lib dir. This is necessary for the jit mode to find
// libstd.
for lib in host.libs {
let filename = lib.file_name().unwrap().to_str().unwrap();
if filename.contains("std-") && !filename.contains(".rlib") {
try_hard_link(&lib, LIB_DIR.to_path(dirs).join(lib.file_name().unwrap()));
}
}
let mut target_compiler = Compiler::clif_with_triple(&dirs, target_triple);
let mut target_compiler = {
let dirs: &Dirs = &dirs;
let rustc_clif =
RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustc-clif"));
let rustdoc_clif =
RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustdoc-clif"));
Compiler {
cargo: bootstrap_host_compiler.cargo.clone(),
rustc: rustc_clif.clone(),
rustdoc: rustdoc_clif.clone(),
rustflags: String::new(),
rustdocflags: String::new(),
triple: target_triple,
runner: vec![],
}
};
if !is_native {
target_compiler.set_cross_linker_and_runner();
}
target_compiler
}
struct SysrootTarget {
triple: String,
libs: Vec<PathBuf>,
}
impl SysrootTarget {
fn install_into_sysroot(&self, sysroot: &Path) {
if self.libs.is_empty() {
return;
}
let target_rustlib_lib = sysroot.join("lib").join("rustlib").join(&self.triple).join("lib");
fs::create_dir_all(&target_rustlib_lib).unwrap();
for lib in &self.libs {
try_hard_link(lib, target_rustlib_lib.join(lib.file_name().unwrap()));
}
}
}
pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot");
pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version");
pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src");
pub(crate) static STANDARD_LIBRARY: CargoProject =
CargoProject::new(&BUILD_SYSROOT, "build_sysroot");
pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup");
#[must_use]
fn build_sysroot_for_triple(
dirs: &Dirs,
channel: &str,
compiler: Compiler,
cg_clif_dylib_path: &Path,
sysroot_kind: SysrootKind,
) -> SysrootTarget {
match sysroot_kind {
SysrootKind::None => build_rtstartup(dirs, &compiler)
.unwrap_or(SysrootTarget { triple: compiler.triple, libs: vec![] }),
SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler),
SysrootKind::Clif => {
build_clif_sysroot_for_triple(dirs, channel, compiler, &cg_clif_dylib_path)
}
}
}
#[must_use]
fn build_llvm_sysroot_for_triple(compiler: Compiler) -> SysrootTarget {
let default_sysroot = super::rustc_info::get_default_sysroot(&compiler.rustc);
let mut target_libs = SysrootTarget { triple: compiler.triple, libs: vec![] };
for entry in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(&target_libs.triple).join("lib"),
)
.unwrap()
{
let entry = entry.unwrap();
if entry.file_type().unwrap().is_dir() {
continue;
}
let file = entry.path();
let file_name_str = file.file_name().unwrap().to_str().unwrap();
if (file_name_str.contains("rustc_")
&& !file_name_str.contains("rustc_std_workspace_")
&& !file_name_str.contains("rustc_demangle"))
|| file_name_str.contains("chalk")
|| file_name_str.contains("tracing")
|| file_name_str.contains("regex")
{
// These are large crates that are part of the rustc-dev component and are not
// necessary to run regular programs.
continue;
}
target_libs.libs.push(file);
}
target_libs
}
#[must_use]
fn build_clif_sysroot_for_triple(
dirs: &Dirs,
channel: &str,
mut compiler: Compiler,
cg_clif_dylib_path: &Path,
) {
) -> SysrootTarget {
match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) {
Err(e) => {
eprintln!("Failed to get rustc version for patched sysroot source: {}", e);
@@ -196,6 +217,14 @@ fn build_clif_sysroot_for_triple(
}
}
let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] };
if let Some(rtstartup_target_libs) = build_rtstartup(dirs, &compiler) {
rtstartup_target_libs.install_into_sysroot(&RTSTARTUP_SYSROOT.to_path(dirs));
target_libs.libs.extend(rtstartup_target_libs.libs);
}
let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(&compiler.triple).join(channel);
if !super::config::get_bool("keep_sysroot") {
@@ -209,7 +238,9 @@ fn build_clif_sysroot_for_triple(
// Build sysroot
let mut rustflags = " -Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap()));
rustflags.push_str(&format!(" --sysroot={}", DIST_DIR.to_path(dirs).to_str().unwrap()));
// Necessary for MinGW to find rsbegin.o and rsend.o
rustflags
.push_str(&format!(" --sysroot={}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap()));
if channel == "release" {
rustflags.push_str(" -Zmir-opt-level=3");
}
@@ -221,7 +252,6 @@ fn build_clif_sysroot_for_triple(
build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
spawn_and_wait(build_cmd);
// Copy all relevant files to the sysroot
for entry in fs::read_dir(build_dir.join("deps")).unwrap() {
let entry = entry.unwrap();
if let Some(ext) = entry.path().extension() {
@@ -231,9 +261,35 @@ fn build_clif_sysroot_for_triple(
} else {
continue;
};
try_hard_link(
entry.path(),
RUSTLIB_DIR.to_path(dirs).join(&compiler.triple).join("lib").join(entry.file_name()),
);
target_libs.libs.push(entry.path());
}
target_libs
}
fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option<SysrootTarget> {
if !compiler.triple.ends_with("windows-gnu") {
return None;
}
RTSTARTUP_SYSROOT.ensure_fresh(dirs);
let rtstartup_src = SYSROOT_SRC.to_path(dirs).join("library").join("rtstartup");
let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] };
for file in ["rsbegin", "rsend"] {
let obj = RTSTARTUP_SYSROOT.to_path(dirs).join(format!("{file}.o"));
let mut build_rtstartup_cmd = Command::new(&compiler.rustc);
build_rtstartup_cmd
.arg("--target")
.arg(&compiler.triple)
.arg("--emit=obj")
.arg("-o")
.arg(&obj)
.arg(rtstartup_src.join(format!("{file}.rs")));
spawn_and_wait(build_rtstartup_cmd);
target_libs.libs.push(obj.clone());
}
Some(target_libs)
}
build_system/rustc_info.rs
-9
View File
@@ -93,12 +93,3 @@ pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String {
assert!(file_name.contains(crate_name));
file_name
}
/// Similar to `get_file_name`, but converts any dashes (`-`) in the `crate_name` to
/// underscores (`_`). This is specially made for the rustc and cargo wrappers
/// which have a dash in the name, and that is not allowed in a crate name.
pub(crate) fn get_wrapper_file_name(crate_name: &str, crate_type: &str) -> String {
let crate_name = crate_name.replace('-', "_");
let wrapper_name = get_file_name(&crate_name, crate_type);
wrapper_name.replace('_', "-")
}
build_system/utils.rs
+1 -18
View File
@@ -5,7 +5,7 @@
use std::process::{self, Command, Stdio};
use super::path::{Dirs, RelPath};
use super::rustc_info::{get_cargo_path, get_rustc_path, get_rustdoc_path, get_wrapper_file_name};
use super::rustc_info::{get_cargo_path, get_rustc_path, get_rustdoc_path};
#[derive(Clone, Debug)]
pub(crate) struct Compiler {
@@ -31,23 +31,6 @@ pub(crate) fn bootstrap_with_triple(triple: String) -> Compiler {
}
}
pub(crate) fn clif_with_triple(dirs: &Dirs, triple: String) -> Compiler {
let rustc_clif =
RelPath::DIST.to_path(&dirs).join(get_wrapper_file_name("rustc-clif", "bin"));
let rustdoc_clif =
RelPath::DIST.to_path(&dirs).join(get_wrapper_file_name("rustdoc-clif", "bin"));
Compiler {
cargo: get_cargo_path(),
rustc: rustc_clif.clone(),
rustdoc: rustdoc_clif.clone(),
rustflags: String::new(),
rustdocflags: String::new(),
triple,
runner: vec![],
}
}
pub(crate) fn set_cross_linker_and_runner(&mut self) {
match self.triple.as_str() {
"aarch64-unknown-linux-gnu" => {