Mirrors/zig
1
0
Fork 0
mirror of https://codeberg.org/ziglang/zig.git synced 2026-04-27 19:09:47 +03:00
Code Issues Packages Projects Releases Wiki Activity

configure/make phase process separation sketch

Browse Source 
`zig build` CLI kicks off async task to compile optimized make runner
executable, does fetch, compiles configure process in debug mode, then
checks cache for the CLI options that affect configuration only. On hit,
skips building/running the configure script. On miss, runs it, saves
result in cache.

The cached artifact is a "configuration" file - a serialized build step
graph, which also includes unlazy package dependencies and additional
file system dependencies.

Next, awaits task for compiling optimized make runner executable, passes
configuration file to it. Make runner is responsible for the CLI after
that point.

For the use case of detecting when `git describe` needs to be rerun, we
can allow the configure process to manually add a file system mtime
dependencies, in this case it would be on `.git/index` and `.git/HEAD`.

This will enable two optimizations:

1. The bulk of the build system will not be rebuilt when user changes
   their configure script.

2. The user logic can be completely bypassed when the CLI options
   provided do not affect the configure phase - even if they affect the
   make phase.

Remaining tasks in the branch:

* some stuff in `zig build` CLI is `@panic("TODO")`.
* configure runner needs to implement serialization of build graph using
  std.zig.Configuration
* build runner needs to be transformed into make runner, consuming
  configuration file as input and deserializing the step graph.
* introduce depending only on a file's metadata and *not* its contents
  into the cache system, and add a std.Build API for using it.
This commit is contained in:
Andrew Kelley
2026-02-12 22:42:34 -08:00
parent b0f6889d12
commit a41ad3a702
8 changed files with 718 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compiler/configure_runner.zig
+336
View File
@@ -0,0 +1,336 @@
const builtin = @import("builtin");
const std = @import("std");
const Io = std.Io;
const assert = std.debug.assert;
const fmt = std.fmt;
const mem = std.mem;
const process = std.process;
const File = std.Io.File;
const Step = std.Build.Step;
const Watch = std.Build.Watch;
const WebServer = std.Build.WebServer;
const Allocator = std.mem.Allocator;
const fatal = std.process.fatal;
const Writer = std.Io.Writer;
const Color = std.zig.Color;
pub const root = @import("@build");
pub const dependencies = @import("@dependencies");
pub const std_options: std.Options = .{
.side_channels_mitigations = .none,
.http_disable_tls = true,
};
pub fn main(init: process.Init.Minimal) !void {
// The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not
// always the case. So, we do need a true gpa for some things.
var debug_gpa_state: std.heap.DebugAllocator(.{}) = .init;
defer _ = debug_gpa_state.deinit();
const gpa = debug_gpa_state.allocator();
var threaded: std.Io.Threaded = .init(gpa, .{
.environ = init.environ,
.argv0 = .init(init.args),
});
defer threaded.deinit();
const io = threaded.io();
// ...but we'll back our arena by `std.heap.page_allocator` for efficiency.
var arena_allocator: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
defer arena_allocator.deinit();
const arena = arena_allocator.allocator();
const args = try init.args.toSlice(arena);
// skip my own exe name
var arg_idx: usize = 1;
const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{});
const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{});
const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{});
const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{});
const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{});
const cwd: Io.Dir = .cwd();
const zig_lib_directory: std.Build.Cache.Directory = .{
.path = zig_lib_dir,
.handle = try cwd.openDir(io, zig_lib_dir, .{}),
};
const build_root_directory: std.Build.Cache.Directory = .{
.path = build_root,
.handle = try cwd.openDir(io, build_root, .{}),
};
const local_cache_directory: std.Build.Cache.Directory = .{
.path = cache_root,
.handle = try cwd.createDirPathOpen(io, cache_root, .{}),
};
const global_cache_directory: std.Build.Cache.Directory = .{
.path = global_cache_root,
.handle = try cwd.createDirPathOpen(io, global_cache_root, .{}),
};
var graph: std.Build.Graph = .{
.io = io,
.arena = arena,
.cache = .{
.io = io,
.gpa = gpa,
.manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}),
.cwd = try process.currentPathAlloc(io, arena),
},
.zig_exe = zig_exe,
.environ_map = try init.environ.createMap(arena),
.global_cache_root = global_cache_directory,
.zig_lib_directory = zig_lib_directory,
.host = .{
.query = .{},
.result = try std.zig.system.resolveTargetQuery(io, .{}),
},
.time_report = false,
};
graph.cache.addPrefix(.{ .path = null, .handle = cwd });
graph.cache.addPrefix(build_root_directory);
graph.cache.addPrefix(local_cache_directory);
graph.cache.addPrefix(global_cache_directory);
graph.cache.hash.addBytes(builtin.zig_version_string);
const builder = try std.Build.create(
&graph,
build_root_directory,
local_cache_directory,
dependencies.root_deps,
);
var error_style: ErrorStyle = .verbose;
var multiline_errors: MultilineErrors = .indent;
var color: Color = .auto;
if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| {
if (std.meta.stringToEnum(ErrorStyle, str)) |style| {
error_style = style;
}
}
if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| {
if (std.meta.stringToEnum(MultilineErrors, str)) |style| {
multiline_errors = style;
}
}
while (nextArg(args, &arg_idx)) |arg| {
if (mem.cutPrefix(u8, arg, "-D")) |option_contents| {
if (option_contents.len == 0)
fatalWithHint("expected option name after '-D'", .{});
if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| {
const option_name = option_contents[0..name_end];
const option_value = option_contents[name_end + 1 ..];
if (try builder.addUserInputOption(option_name, option_value))
fatal(" access the help menu with 'zig build -h'", .{});
} else {
if (try builder.addUserInputFlag(option_contents))
fatal(" access the help menu with 'zig build -h'", .{});
}
} else if (mem.eql(u8, arg, "--verbose")) {
builder.verbose = true;
} else if (mem.startsWith(u8, arg, "-fsys=")) {
const name = arg["-fsys=".len..];
graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM");
} else if (mem.startsWith(u8, arg, "-fno-sys=")) {
const name = arg["-fno-sys=".len..];
graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM");
} else if (mem.eql(u8, arg, "--release")) {
builder.release_mode = .any;
} else if (mem.startsWith(u8, arg, "--release=")) {
const text = arg["--release=".len..];
builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse {
fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{
arg, text,
});
};
} else if (mem.eql(u8, arg, "--search-prefix")) {
const search_prefix = nextArgOrFatal(args, &arg_idx);
builder.addSearchPrefix(search_prefix);
} else if (mem.eql(u8, arg, "--libc")) {
builder.libc_file = nextArgOrFatal(args, &arg_idx);
} else if (mem.eql(u8, arg, "--color")) {
const next_arg = nextArg(args, &arg_idx) orelse
fatalWithHint("expected [auto|on|off] after '{s}'", .{arg});
color = std.meta.stringToEnum(Color, next_arg) orelse {
fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{
arg, next_arg,
});
};
} else if (mem.eql(u8, arg, "--error-style")) {
const next_arg = nextArg(args, &arg_idx) orelse
fatalWithHint("expected style after '{s}'", .{arg});
error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse {
fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg });
};
} else if (mem.eql(u8, arg, "--multiline-errors")) {
const next_arg = nextArg(args, &arg_idx) orelse
fatalWithHint("expected style after '{s}'", .{arg});
multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse {
fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg });
};
} else if (mem.eql(u8, arg, "--seed")) {
const next_arg = nextArg(args, &arg_idx) orelse
fatalWithHint("expected u32 after '{s}'", .{arg});
graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| {
fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{
next_arg, @errorName(err),
});
};
} else if (mem.eql(u8, arg, "--build-id")) {
builder.build_id = .fast;
} else if (mem.startsWith(u8, arg, "--build-id=")) {
const style = arg["--build-id=".len..];
builder.build_id = std.zig.BuildId.parse(style) catch |err| {
fatal("unable to parse --build-id style '{s}': {s}", .{
style, @errorName(err),
});
};
} else if (mem.eql(u8, arg, "--debug-pkg-config")) {
builder.debug_pkg_config = true;
} else if (mem.eql(u8, arg, "--debug-rt")) {
graph.debug_compiler_runtime_libs = true;
} else if (mem.eql(u8, arg, "--debug-compile-errors")) {
builder.debug_compile_errors = true;
} else if (mem.eql(u8, arg, "--debug-incremental")) {
builder.debug_incremental = true;
} else if (mem.eql(u8, arg, "--system")) {
// The usage text shows another argument after this parameter
// but it is handled by the parent process. The build runner
// only sees this flag.
graph.system_package_mode = true;
} else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) {
// --glibc-runtimes was the old name of the flag; kept for compatibility for now.
builder.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx);
} else if (mem.eql(u8, arg, "--verbose-link")) {
builder.verbose_link = true;
} else if (mem.eql(u8, arg, "--verbose-air")) {
builder.verbose_air = true;
} else if (mem.eql(u8, arg, "--verbose-llvm-ir")) {
builder.verbose_llvm_ir = "-";
} else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) {
builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..];
} else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) {
builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..];
} else if (mem.eql(u8, arg, "--verbose-cimport")) {
builder.verbose_cimport = true;
} else if (mem.eql(u8, arg, "--verbose-cc")) {
builder.verbose_cc = true;
} else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) {
builder.verbose_llvm_cpu_features = true;
} else if (mem.eql(u8, arg, "-fincremental")) {
graph.incremental = true;
} else if (mem.eql(u8, arg, "-fno-incremental")) {
graph.incremental = false;
} else if (mem.eql(u8, arg, "-fwine")) {
builder.enable_wine = true;
} else if (mem.eql(u8, arg, "-fno-wine")) {
builder.enable_wine = false;
} else if (mem.eql(u8, arg, "-fqemu")) {
builder.enable_qemu = true;
} else if (mem.eql(u8, arg, "-fno-qemu")) {
builder.enable_qemu = false;
} else if (mem.eql(u8, arg, "-fwasmtime")) {
builder.enable_wasmtime = true;
} else if (mem.eql(u8, arg, "-fno-wasmtime")) {
builder.enable_wasmtime = false;
} else if (mem.eql(u8, arg, "-frosetta")) {
builder.enable_rosetta = true;
} else if (mem.eql(u8, arg, "-fno-rosetta")) {
builder.enable_rosetta = false;
} else if (mem.eql(u8, arg, "-fdarling")) {
builder.enable_darling = true;
} else if (mem.eql(u8, arg, "-fno-darling")) {
builder.enable_darling = false;
} else if (mem.eql(u8, arg, "-fallow-so-scripts")) {
graph.allow_so_scripts = true;
} else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) {
graph.allow_so_scripts = false;
} else if (mem.eql(u8, arg, "-freference-trace")) {
builder.reference_trace = 256;
} else if (mem.startsWith(u8, arg, "-freference-trace=")) {
const num = arg["-freference-trace=".len..];
builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| {
std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) });
process.exit(1);
};
} else if (mem.eql(u8, arg, "-fno-reference-trace")) {
builder.reference_trace = null;
} else if (mem.cutPrefix(u8, arg, "-j")) |text| {
const n = std.fmt.parseUnsigned(u32, text, 10) catch |err|
fatal("unable to parse jobs count '{s}': {t}", .{ text, err });
if (n < 1) fatal("number of jobs must be at least 1", .{});
threaded.setAsyncLimit(.limited(n));
} else if (mem.eql(u8, arg, "--")) {
builder.args = argsRest(args, arg_idx);
break;
} else {
fatalWithHint("unrecognized argument: '{s}'", .{arg});
}
}
const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map);
const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map);
graph.stderr_mode = switch (color) {
.auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE),
.on => .escape_codes,
.off => .no_color,
};
try builder.runBuild(root);
}
fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 {
if (idx.* >= args.len) return null;
defer idx.* += 1;
return args[idx.*];
}
fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 {
return nextArg(args, idx) orelse {
std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]});
process.exit(1);
};
}
fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 {
if (idx >= args.len) return null;
return args[idx..];
}
const ErrorStyle = enum {
verbose,
minimal,
verbose_clear,
minimal_clear,
fn verboseContext(s: ErrorStyle) bool {
return switch (s) {
.verbose, .verbose_clear => true,
.minimal, .minimal_clear => false,
};
}
fn clearOnUpdate(s: ErrorStyle) bool {
return switch (s) {
.verbose, .minimal => false,
.verbose_clear, .minimal_clear => true,
};
}
};
const MultilineErrors = enum { indent, newline, none };
const Summary = enum { all, new, failures, line, none };
fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn {
std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args);
process.exit(1);
}
lib/std/Build/Cache.zig
+6
View File
@@ -1024,6 +1024,12 @@ pub const Manifest = struct {
try self.populateFileHash(gop.key_ptr);
}
pub fn addPathPost(man: *Manifest, path: Path) !void {
_ = man;
_ = path;
@panic("TODO");
}
/// Like `addFilePost` but when the file contents have already been loaded from disk.
pub fn addFilePostContents(
self: *Manifest,
lib/std/zig.zig
+2
View File
@@ -11,6 +11,8 @@ const Writer = std.Io.Writer;
const tokenizer = @import("zig/tokenizer.zig");
/// The serialized output of configure phase ingested by make phase.
pub const Configuration = @import("zig/Configuration.zig");
pub const ErrorBundle = @import("zig/ErrorBundle.zig");
pub const Server = @import("zig/Server.zig");
pub const Client = @import("zig/Client.zig");
lib/std/zig/Configuration.zig
+83
View File
@@ -0,0 +1,83 @@
const Configuration = @This();
const std = @import("../std.zig");
const Io = std.Io;
const Allocator = std.mem.Allocator;
string_bytes: []u8,
steps: []Step,
path_deps_base: []Path.Base,
path_deps_sub: []String,
unlazy_deps: []String,
pub const Header = extern struct {
string_bytes_len: u32,
steps_len: u32,
path_deps_len: u32,
unlazy_deps_len: u32,
};
pub const Step = extern struct {
name: String,
};
pub const Path = extern struct {
base: Base,
sub: String,
pub const Base = enum(u8) {
cwd,
global_cache,
local_cache,
build_root,
};
pub fn toCachePath(path: Path, c: *const Configuration, arena: Allocator) std.Build.Cache.Path {
_ = c;
_ = arena;
_ = path;
@panic("TODO");
}
};
pub const String = enum(u32) {
_,
pub fn slice(index: String, c: *const Configuration) [:0]const u8 {
const start_slice = c.string_bytes[@intFromEnum(index)..];
return start_slice[0..std.mem.indexOfScalar(u8, start_slice, 0).? :0];
}
};
pub const LoadError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream};
pub fn load(arena: Allocator, io: Io, file: Io.File) LoadError!Configuration {
var buffer: [2000]u8 = undefined;
var fr = file.reader(io, &buffer);
const header = fr.interface.takeStruct(Header, .little) catch |err| switch (err) {
error.ReadFailed => return fr.err.?,
else => |e| return e,
};
var result: Configuration = .{
.string_bytes = try arena.alloc(u8, header.string_bytes_len),
.steps = try arena.alloc(Step, header.steps_len),
.path_deps_sub = try arena.alloc(String, header.path_deps_len),
.path_deps_base = try arena.alloc(Path.Base, header.path_deps_len),
.unlazy_deps = try arena.alloc(String, header.unlazy_deps_len),
};
var vecs = [_][]u8{
result.string_bytes,
@ptrCast(result.steps),
@ptrCast(result.path_deps_base),
@ptrCast(result.path_deps_sub),
@ptrCast(result.unlazy_deps),
};
fr.interface.readVecAll(&vecs) catch |err| switch (err) {
error.ReadFailed => return fr.err.?,
else => |e| return e,
};
return result;
}
lib/std/zig/LibCInstallation.zig
+16 -2
View File
@@ -12,6 +12,7 @@ const Target = std.Target;
const fs = std.fs;
const Allocator = std.mem.Allocator;
const Path = std.Build.Cache.Path;
const Cache = std.Build.Cache;
const log = std.log.scoped(.libc_installation);
const Environ = std.process.Environ;
@@ -969,7 +970,7 @@ pub fn resolveCrtPaths(
target: *const std.Target,
) error{ OutOfMemory, LibCInstallationMissingCrtDir }!CrtPaths {
const crt_dir_path: Path = .{
.root_dir = std.Build.Cache.Directory.cwd(),
.root_dir = Cache.Directory.cwd(),
.sub_path = lci.crt_dir orelse return error.LibCInstallationMissingCrtDir,
};
switch (target.os.tag) {
@@ -995,7 +996,7 @@ pub fn resolveCrtPaths(
},
.haiku => {
const gcc_dir_path: Path = .{
.root_dir = std.Build.Cache.Directory.cwd(),
.root_dir = Cache.Directory.cwd(),
.sub_path = lci.gcc_dir orelse return error.LibCInstallationMissingCrtDir,
};
return .{
@@ -1017,3 +1018,16 @@ pub fn resolveCrtPaths(
},
}
}
pub fn addToHash(opt_lci: ?*const LibCInstallation, hh: *Cache.HashHelper, abi: std.Target.Abi) void {
const lci = opt_lci orelse return hh.add(false);
hh.add(true);
hh.addOptionalBytes(lci.crt_dir);
switch (abi) {
.msvc, .itanium => {
hh.addOptionalBytes(lci.msvc_lib_dir);
hh.addOptionalBytes(lci.kernel32_lib_dir);
},
else => {},
}
}
src/Compilation.zig
+2 -12
View File
@@ -752,13 +752,10 @@ pub const Directories = struct {
else => []const u8,
},
environ_map: *const std.process.Environ.Map,
cwd: []const u8,
) Directories {
const wasi = builtin.target.os.tag == .wasi;
const cwd = introspect.getResolvedCwd(io, arena) catch |err| {
fatal("unable to get cwd: {t}", .{err});
};
const zig_lib: Cache.Directory = d: {
if (override_zig_lib) |path| break :d openUnresolved(arena, io, cwd, path, .@"zig lib");
if (wasi) break :d getPreopen(preopens, "/lib");
@@ -3527,14 +3524,7 @@ fn addNonIncrementalStuffToCacheManifest(
man.hash.addListOfBytes(opts.rpath_list);
man.hash.addListOfBytes(opts.symbol_wrap_set.keys());
if (comp.config.link_libc) {
man.hash.add(comp.libc_installation != null);
if (comp.libc_installation) |libc_installation| {
man.hash.addOptionalBytes(libc_installation.crt_dir);
if (target.abi == .msvc or target.abi == .itanium) {
man.hash.addOptionalBytes(libc_installation.msvc_lib_dir);
man.hash.addOptionalBytes(libc_installation.kernel32_lib_dir);
}
}
LibCInstallation.addToHash(comp.libc_installation, &man.hash, target.abi);
man.hash.addOptionalBytes(target.dynamic_linker.get());
}
man.hash.add(opts.repro);
src/main.zig
+269 -126
View File
@@ -3177,6 +3177,8 @@ fn buildOutputType(
else => process.executablePathAlloc(io, arena) catch |err| fatal("unable to find zig self exe path: {t}", .{err}),
};
const cwd_path = try introspect.getResolvedCwd(io, arena);
// This `init` calls `fatal` on error.
var dirs: Compilation.Directories = .init(
arena,
@@ -3193,6 +3195,7 @@ fn buildOutputType(
preopens,
self_exe_path,
environ_map,
cwd_path,
);
defer dirs.deinit(io);
@@ -4951,15 +4954,20 @@ test sanitizeExampleName {
try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project"));
}
fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, environ_map: *process.Environ.Map) !void {
dev.check(.build_command);
fn cmdBuild(
gpa: Allocator,
arena: Allocator,
io: Io,
args: []const []const u8,
environ_map: *process.Environ.Map,
) !void {
var build_file: ?[]const u8 = null;
var override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map);
var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
var override_build_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map);
var child_argv: std.ArrayList([]const u8) = .empty;
var override_make_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map);
var configure_argv: std.ArrayList([]const u8) = .empty;
var make_argv: std.ArrayList([]const u8) = .empty;
var forks: std.ArrayList(Fork) = .empty;
var reference_trace: ?u32 = null;
var debug_compile_errors = false;
@@ -4979,46 +4987,32 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
var debug_target: ?[]const u8 = null;
var debug_libc_paths_file: ?[]const u8 = null;
const argv_index_exe = child_argv.items.len;
_ = try child_argv.addOne(arena);
const argv_index_exe = configure_argv.items.len;
_ = try configure_argv.addOne(arena);
const self_exe_path = try process.executablePathAlloc(io, arena);
try child_argv.append(arena, self_exe_path);
try configure_argv.append(arena, self_exe_path);
const argv_index_zig_lib_dir = child_argv.items.len;
_ = try child_argv.addOne(arena);
const argv_index_zig_lib_dir = configure_argv.items.len;
_ = try configure_argv.addOne(arena);
const argv_index_build_file = child_argv.items.len;
_ = try child_argv.addOne(arena);
const argv_index_build_file = configure_argv.items.len;
_ = try configure_argv.addOne(arena);
const argv_index_cache_dir = child_argv.items.len;
_ = try child_argv.addOne(arena);
const argv_index_cache_dir = configure_argv.items.len;
_ = try configure_argv.addOne(arena);
const argv_index_global_cache_dir = child_argv.items.len;
_ = try child_argv.addOne(arena);
const argv_index_global_cache_dir = configure_argv.items.len;
_ = try configure_argv.addOne(arena);
try child_argv.appendSlice(arena, &.{
try configure_argv.appendSlice(arena, &.{
"--seed",
try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}),
});
const argv_index_seed = child_argv.items.len - 1;
const argv_index_seed = configure_argv.items.len - 1;
// This parent process needs a way to obtain results from the configuration
// phase of the child process. In the future, the make phase will be
// executed in a separate process than the configure phase, and we can then
// use stdout from the configuration phase for this purpose.
//
// However, currently, both phases are in the same process, and Run Step
// provides API for making the runned subprocesses inherit stdout and stderr
// which means these streams are not available for passing metadata back
// to the parent.
//
// Until make and configure phases are separated into different processes,
// the strategy is to choose a temporary file name ahead of time, and then
// read this file in the parent to obtain the results, in the case the child
// exits with code 3.
const results_tmp_file_nonce = std.fmt.hex(randInt(io, u64));
try child_argv.append(arena, "-Z" ++ results_tmp_file_nonce);
const argv_index_configuration_file = make_argv.items.len;
_ = try make_argv.addOne(arena);
var color: Color = .auto;
var n_jobs: ?u32 = null;
@@ -5041,7 +5035,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
} else if (mem.eql(u8, arg, "--build-runner")) {
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
i += 1;
override_build_runner = args[i];
override_make_runner = args[i];
continue;
} else if (mem.eql(u8, arg, "--cache-dir")) {
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
@@ -5080,7 +5074,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
i += 1;
system_pkg_dir_path = args[i];
try child_argv.append(arena, "--system");
try configure_argv.append(arena, "--system");
continue;
} else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| {
reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| {
@@ -5090,7 +5084,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
reference_trace = null;
} else if (mem.eql(u8, arg, "--debug-log")) {
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
try child_argv.appendSlice(arena, args[i .. i + 2]);
try make_argv.appendSlice(arena, args[i .. i + 2]);
i += 1;
if (!build_options.enable_logging) {
warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{});
@@ -5144,7 +5138,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
color = std.meta.stringToEnum(Color, args[i]) orelse {
fatal("expected [auto|on|off] after {s}, found '{s}'", .{ arg, args[i] });
};
try child_argv.appendSlice(arena, &.{ arg, args[i] });
try configure_argv.appendSlice(arena, &.{ arg, args[i] });
continue;
} else if (mem.cutPrefix(u8, arg, "-j")) |str| {
const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| {
@@ -5159,61 +5153,30 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
} else if (mem.eql(u8, arg, "--seed")) {
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
i += 1;
child_argv.items[argv_index_seed] = args[i];
configure_argv.items[argv_index_seed] = args[i];
continue;
} else if (mem.eql(u8, arg, "--")) {
// The rest of the args are supposed to get passed onto
// build runner's `build.args`
try child_argv.appendSlice(arena, args[i..]);
try configure_argv.appendSlice(arena, args[i..]);
break;
}
}
try child_argv.append(arena, arg);
try make_argv.append(arena, arg);
}
}
const root_prog_node = std.Progress.start(io, .{
.disable_printing = (color == .off),
.root_name = "Compile Build Script",
.root_name = "",
});
defer root_prog_node.end();
// Normally the build runner is compiled for the host target but here is
// some code to help when debugging edits to the build runner so that you
// can make sure it compiles successfully on other targets.
const resolved_target: Package.Module.ResolvedTarget = t: {
if (build_options.enable_debug_extensions) {
if (debug_target) |triple| {
const target_query = try std.Target.Query.parse(.{
.arch_os_abi = triple,
});
break :t .{
.result = std.zig.resolveTargetQueryOrFatal(io, target_query),
.is_native_os = false,
.is_native_abi = false,
.is_explicit_dynamic_linker = false,
};
}
}
break :t .{
.result = std.zig.resolveTargetQueryOrFatal(io, .{}),
.is_native_os = true,
.is_native_abi = true,
.is_explicit_dynamic_linker = false,
};
};
// Likewise, `--debug-libc` allows overriding the libc installation.
const libc_installation: ?*const LibCInstallation = lci: {
const paths_file = debug_libc_paths_file orelse break :lci null;
if (!build_options.enable_debug_extensions) unreachable;
const lci = try arena.create(LibCInstallation);
lci.* = try .parse(arena, io, paths_file, &resolved_target.result);
break :lci lci;
};
process.raiseFileDescriptorLimit();
const cwd_path = try introspect.getResolvedCwd(io, arena);
const cwd_path = introspect.getResolvedCwd(io, arena) catch |err|
fatal("failed to get current directory path: {t}", .{err});
const build_root = try findBuildRoot(arena, io, .{
.cwd_path = cwd_path,
.build_file = build_file,
@@ -5232,20 +5195,84 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
.empty,
self_exe_path,
environ_map,
cwd_path,
);
defer dirs.deinit(io);
child_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path;
child_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path;
child_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path;
child_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path;
const thread_limit = @min(
@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),
std.math.maxInt(Zcu.PerThread.IdBacking),
);
try setThreadLimit(arena, thread_limit);
// Kick off an optimized compilation of the make runner.
var make_runner_task = io.async(compileMakeRunner, .{ io, .{
.dirs = &dirs,
.optimize = .ReleaseSafe,
.parent_prog_node = root_prog_node,
} });
defer if (make_runner_task.cancel(io)) |mr| mr.deinit(io) else |_| {};
// Cache lookup for configure options. If we get a match, we can skip
// execution of the configure script. If not, we get the file path to pass
// to the configure process.
var local_cache: Cache = .{
.gpa = gpa,
.io = io,
.manifest_dir = try dirs.local_cache.handle.createDirPathOpen(io, "h", .{}),
.cwd = cwd_path,
};
local_cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() });
local_cache.addPrefix(dirs.zig_lib);
local_cache.addPrefix(dirs.local_cache);
local_cache.addPrefix(dirs.global_cache);
defer local_cache.manifest_dir.close(io);
var config_man = local_cache.obtain();
defer config_man.deinit();
config_man.hash.addBytes(build_options.version);
// Normally the build runner is compiled for the host target but here is
// some code to help when debugging edits to the build runner so that you
// can make sure it compiles successfully on other targets.
const resolved_target: Package.Module.ResolvedTarget = t: {
if (build_options.enable_debug_extensions) {
if (debug_target) |triple| {
const target_query = try std.Target.Query.parse(.{
.arch_os_abi = triple,
});
config_man.hash.addBytes(triple);
break :t .{
.result = std.zig.resolveTargetQueryOrFatal(io, target_query),
.is_native_os = false,
.is_native_abi = false,
.is_explicit_dynamic_linker = false,
};
}
}
break :t .{
.result = std.zig.resolveTargetQueryOrFatal(io, .{}),
.is_native_os = true,
.is_native_abi = true,
.is_explicit_dynamic_linker = false,
};
};
// Likewise, `--debug-libc` allows overriding the libc installation.
const libc_installation: ?*const LibCInstallation = lci: {
const paths_file = debug_libc_paths_file orelse break :lci null;
if (!build_options.enable_debug_extensions) unreachable;
const lci = try arena.create(LibCInstallation);
lci.* = try .parse(arena, io, paths_file, &resolved_target.result);
LibCInstallation.addToHash(lci, &config_man.hash, resolved_target.result.abi);
break :lci lci;
};
configure_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path;
configure_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path;
configure_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path;
configure_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path;
// Dummy http client that is not actually used when fetch_command is unsupported.
// Prevents bootstrap from depending on a bunch of unnecessary stuff.
var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct {
@@ -5282,11 +5309,11 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
// This loop is re-evaluated when the build script exits with an indication that it
// could not continue due to missing lazy dependencies.
while (true) {
const configuration_path: Path = cp: while (true) {
// We want to release all the locks before executing the child process, so we make a nice
// big block here to ensure the cleanup gets run when we extract out our argv.
{
const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_build_runner) |runner| .{
const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_make_runner) |runner| .{
.root = try .fromUnresolved(arena, dirs, &.{fs.path.dirname(runner) orelse "."}),
.root_src_path = fs.path.basename(runner),
} else .{
@@ -5515,6 +5542,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
config,
);
const compile_prog_node = root_prog_node.start("Compile Configure Script", 0);
defer compile_prog_node.end();
try root_mod.deps.put(arena, "@build", build_mod);
var create_diag: Compilation.CreateDiagnostic = undefined;
@@ -5546,7 +5576,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
};
defer comp.destroy();
updateModule(comp, color, root_prog_node) catch |err| switch (err) {
updateModule(comp, color, compile_prog_node) catch |err| switch (err) {
error.CompileErrorsReported => process.exit(2),
else => |e| return e,
};
@@ -5554,52 +5584,74 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
// Since incremental compilation isn't done yet, we use cache_mode = whole
// above, and thus the output file is already closed.
//try comp.makeBinFileExecutable();
child_argv.items[argv_index_exe] = try dirs.local_cache.join(arena, &.{
"o",
&Cache.binToHex(comp.digest.?),
comp.emit_bin.?,
});
const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?);
const exe_path: Path = .{
.root_dir = dirs.local_cache,
.sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }),
};
_ = try config_man.addFilePath(exe_path, null);
configure_argv.items[argv_index_exe] = try exe_path.toString(arena);
if (try config_man.hit()) {
const digest = config_man.final();
break :cp .{
.root_dir = dirs.local_cache,
.sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}),
};
}
}
if (!process.can_spawn) {
const cmd = try std.mem.join(arena, " ", child_argv.items);
const cmd = try std.mem.join(arena, " ", configure_argv.items);
fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd });
}
const rand_int = randInt(io, u64);
const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
const config_tmp_path: Path = .{
.root_dir = dirs.local_cache,
.sub_path = tmp_dir_sub_path,
};
const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile(
io,
config_tmp_path.sub_path,
.{ .read = true, .exclusive = true },
);
defer config_tmp_file.close(io);
switch (term: {
_ = try io.lockStderr(&.{}, .no_color);
defer io.unlockStderr();
const child_node = root_prog_node.start("Run Configure Script", 0);
defer child_node.end();
var child = std.process.spawn(io, .{
.argv = child_argv.items,
}) catch |err| fatal("failed to spawn build runner {s}: {t}", .{ child_argv.items[0], err });
.argv = configure_argv.items,
.stdout = .{ .file = config_tmp_file },
.progress_node = child_node,
}) catch |err| fatal("failed to spawn configure script {s}: {t}", .{ configure_argv.items[0], err });
defer child.kill(io);
break :term child.wait(io) catch |err|
fatal("failed to wait build runner {s}: {t}", .{ child_argv.items[0], err });
fatal("failed to wait configure script {s}: {t}", .{ configure_argv.items[0], err });
}) {
.exited => |code| {
if (code == 0) return cleanExit(io);
// Indicates that the build runner has reported compile errors
// and this parent process does not need to report any further
// diagnostics.
if (code == 2) process.exit(2);
if (code != 0) {
// Failure to produce the configuration file.
const cmd = try std.mem.join(arena, " ", configure_argv.items);
fatal("the following configure command failed with exit code {d}:\n{s}", .{ code, cmd });
}
// Even though the file is designed to be sent directly to make
// runner, we must load it now because:
// * If it contains additional file dependencies, we need to
// add them to `config_man` before obtaining the final digest.
// * If it contains a set of lazy packages that need to be
// fetched, we need to fetch those now and re-run configure.
var configuration = std.zig.Configuration.load(arena, io, config_tmp_file) catch |err|
fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err });
if (code == 3) {
if (!dev.env.supports(.fetch_command)) process.exit(3);
// Indicates the configure phase failed due to missing lazy
// dependencies and stdout contains the hashes of the ones
// that are missing.
const s = fs.path.sep_str;
const tmp_sub_path = "tmp" ++ s ++ results_tmp_file_nonce;
const stdout = dirs.local_cache.handle.readFileAlloc(io, tmp_sub_path, arena, .limited(50 * 1024 * 1024)) catch |err| {
fatal("unable to read results of configure phase from '{f}{s}': {t}", .{
dirs.local_cache, tmp_sub_path, err,
});
};
dirs.local_cache.handle.deleteFile(io, tmp_sub_path) catch {};
var it = mem.splitScalar(u8, stdout, '\n');
if (configuration.unlazy_deps.len != 0) {
if (!dev.env.supports(.fetch_command)) process.exit(1);
var any_errors = false;
while (it.next()) |hash| {
if (hash.len == 0) continue;
for (configuration.unlazy_deps) |hash_string| {
const hash = hash_string.slice(&configuration);
assert(hash.len != 0);
if (hash.len > Package.Hash.max_len) {
std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{
hash.len, hash,
@@ -5609,10 +5661,11 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
}
try unlazy_set.put(arena, .fromSlice(hash), {});
}
if (any_errors) process.exit(3);
if (any_errors) process.exit(1);
if (system_pkg_dir_path) |p| {
// In this mode, the system needs to provide these packages; they
// cannot be fetched by Zig.
const s = fs.path.sep_str;
for (unlazy_set.keys()) |*hash| {
std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{
p, hash.toSlice(),
@@ -5620,28 +5673,115 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
}
std.log.info("remote package fetching disabled due to --system mode", .{});
std.log.info("dependencies might be avoidable depending on build configuration", .{});
process.exit(3);
process.exit(1);
}
continue;
continue :cp;
}
const cmd = try std.mem.join(arena, " ", child_argv.items);
fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });
for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| {
const conf_path: std.zig.Configuration.Path = .{ .base = base, .sub = sub };
try config_man.addPathPost(conf_path.toCachePath(&configuration, arena));
}
const digest = config_man.final();
const final_path: Path = .{
.root_dir = dirs.local_cache,
.sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}),
};
Io.Dir.rename(
config_tmp_path.root_dir.handle,
config_tmp_path.sub_path,
final_path.root_dir.handle,
final_path.sub_path,
io,
) catch |err| {
fatal("failed to rename configuration file from {f} into {f}: {t}", .{
config_tmp_path, final_path, err,
});
};
config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err});
break :cp final_path;
},
.signal => |sig| {
const cmd = try std.mem.join(arena, " ", child_argv.items);
fatal("the following build command terminated with signal {t}:\n{s}", .{ sig, cmd });
const cmd = try std.mem.join(arena, " ", configure_argv.items);
fatal("the following configure command terminated with signal {t}:\n{s}", .{ sig, cmd });
},
.stopped => |sig| {
const cmd = try std.mem.join(arena, " ", child_argv.items);
const cmd = try std.mem.join(arena, " ", configure_argv.items);
fatal("the following build command stopped with signal {t}:\n{s}", .{ sig, cmd });
},
.unknown => {
const cmd = try std.mem.join(arena, " ", child_argv.items);
const cmd = try std.mem.join(arena, " ", configure_argv.items);
fatal("the following build command crashed:\n{s}", .{cmd});
},
}
};
{
// Release all file system locks just before running the maker process.
var configuration_lock = config_man.toOwnedLock();
defer configuration_lock.release(io);
const make_runner = make_runner_task.await(io) catch |err|
fatal("failed to compile maker: {t}", .{err});
defer make_runner.deinit(io);
make_argv.items[0] = try make_runner.exe_path.toString(arena);
make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena);
}
if (!process.can_spawn) {
const cmd = try std.mem.join(arena, " ", make_argv.items);
fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd });
}
switch (term: {
_ = try io.lockStderr(&.{}, .no_color);
defer io.unlockStderr();
var child = std.process.spawn(io, .{
.argv = make_argv.items,
}) catch |err| fatal("failed to spawn maker {s}: {t}", .{ make_argv.items[0], err });
defer child.kill(io);
break :term child.wait(io) catch |err|
fatal("failed to wait maker {s}: {t}", .{ make_argv.items[0], err });
}) {
.exited => |code| {
if (code == 0) return cleanExit(io);
const cmd = try std.mem.join(arena, " ", configure_argv.items);
fatal("the following maker command failed with exit code {d}:\n{s}", .{ code, cmd });
},
.signal => |sig| {
const cmd = try std.mem.join(arena, " ", configure_argv.items);
fatal("the following maker command terminated with signal {t}:\n{s}", .{ sig, cmd });
},
else => {
const cmd = try std.mem.join(arena, " ", configure_argv.items);
fatal("the following maker command crashed:\n{s}", .{cmd});
},
}
}
const MakeRunner = struct {
exe_path: Path,
const Options = struct {
dirs: *Compilation.Directories,
optimize: std.builtin.OptimizeMode,
parent_prog_node: std.Progress.Node,
};
fn deinit(mr: MakeRunner, io: Io) void {
_ = mr;
_ = io;
@panic("TODO");
}
};
fn compileMakeRunner(io: Io, options: MakeRunner.Options) !MakeRunner {
_ = io;
_ = options;
@panic("TODO");
}
const Fork = struct {
@@ -5767,6 +5907,8 @@ fn jitCmdInner(
const override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map);
const override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
const cwd_path = try introspect.getResolvedCwd(io, arena);
// This `init` calls `fatal` on error.
var dirs: Compilation.Directories = .init(
arena,
@@ -5777,6 +5919,7 @@ fn jitCmdInner(
preopens,
self_exe_path,
environ_map,
cwd_path,
);
defer dirs.deinit(io);
src/print_env.zig
+4
View File
@@ -8,6 +8,7 @@ const fatal = std.process.fatal;
const build_options = @import("build_options");
const Compilation = @import("Compilation.zig");
const introspect = @import("introspect.zig");
pub fn cmdEnv(
arena: Allocator,
@@ -28,6 +29,8 @@ pub fn cmdEnv(
},
};
const cwd_path = try introspect.getResolvedCwd(io, arena);
var dirs: Compilation.Directories = .init(
arena,
io,
@@ -37,6 +40,7 @@ pub fn cmdEnv(
preopens,
if (builtin.target.os.tag != .wasi) self_exe_path,
environ_map,
cwd_path,
);
defer dirs.deinit(io);