Mirrors/zig
1
0
Fork 0
mirror of https://codeberg.org/ziglang/zig.git synced 2026-05-30 12:56:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

maker: port Run step logic up to spawnChildAndCollect

Browse Source
This commit is contained in:
Andrew Kelley
2026-04-27 20:58:30 -07:00
parent 0d48cbb822
commit c8b583885d
8 changed files with 345 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
BRANCH_TODO
+1
View File
@@ -18,6 +18,7 @@
* https://codeberg.org/ziglang/zig/pulls/30762
## Followup Issues
* reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make
* link_eh_frame_hdr should be DefaultingBool
* make --foo, --no-foo CLI args uniform (make them -f args instead)
* install steps should provide generated files for installed things, then delete the run step hack
lib/compiler/Maker.zig
+12 -22
View File
@@ -153,17 +153,6 @@ pub fn main(init: process.Init.Minimal) !void {
var debounce_interval_ms: u16 = 50;
var webui_listen: ?Io.net.IpAddress = null;
var debug_pkg_config: bool = false;
// After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md,
// this will be the directory $glibc-build-dir/install/glibcs
// Given the example of the aarch64 target, this is the directory
// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`.
// Also works for dynamic musl.
var libc_runtimes_dir: ?[]const u8 = null;
var enable_wine = false;
var enable_qemu = false;
var enable_wasmtime = false;
var enable_darling = false;
var enable_rosetta = false;
var run_args: ?[]const []const u8 = null;
if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| {
@@ -314,7 +303,7 @@ pub fn main(init: process.Init.Minimal) !void {
fatal("unrecognized optimization mode: {s}", .{rest});
} else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) {
// --glibc-runtimes was the old name of the flag; kept for compatibility for now.
libc_runtimes_dir = nextArgOrFatal(args, &arg_idx);
graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx);
} else if (mem.eql(u8, arg, "--verbose")) {
graph.verbose = true;
} else if (mem.eql(u8, arg, "--verbose-air")) {
@@ -370,25 +359,25 @@ pub fn main(init: process.Init.Minimal) !void {
} else if (mem.eql(u8, arg, "-fno-incremental")) {
graph.incremental = false;
} else if (mem.eql(u8, arg, "-fwine")) {
enable_wine = true;
graph.enable_wine = true;
} else if (mem.eql(u8, arg, "-fno-wine")) {
enable_wine = false;
graph.enable_wine = false;
} else if (mem.eql(u8, arg, "-fqemu")) {
enable_qemu = true;
graph.enable_qemu = true;
} else if (mem.eql(u8, arg, "-fno-qemu")) {
enable_qemu = false;
graph.enable_qemu = false;
} else if (mem.eql(u8, arg, "-fwasmtime")) {
enable_wasmtime = true;
graph.enable_wasmtime = true;
} else if (mem.eql(u8, arg, "-fno-wasmtime")) {
enable_wasmtime = false;
graph.enable_wasmtime = false;
} else if (mem.eql(u8, arg, "-frosetta")) {
enable_rosetta = true;
graph.enable_rosetta = true;
} else if (mem.eql(u8, arg, "-fno-rosetta")) {
enable_rosetta = false;
graph.enable_rosetta = false;
} else if (mem.eql(u8, arg, "-fdarling")) {
enable_darling = true;
graph.enable_darling = true;
} else if (mem.eql(u8, arg, "-fno-darling")) {
enable_darling = false;
graph.enable_darling = false;
} else if (mem.eql(u8, arg, "-fallow-so-scripts")) {
graph.allow_so_scripts = true;
} else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) {
@@ -533,6 +522,7 @@ pub fn main(init: process.Init.Minimal) !void {
.bin = install_bin_path,
.include = install_include_path,
},
.steps = try arena.alloc(Step, scanned_config.configuration.steps.len),
.generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len),
.run_args = run_args,
lib/compiler/Maker/Graph.zig
+12
View File
@@ -52,6 +52,18 @@ error_limit: ?u32 = null,
/// a single step spawning a fixed number of processes this can be used.
max_jobs: ?u32 = null,
/// After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md,
/// this will be the directory $glibc-build-dir/install/glibcs
/// Given the example of the aarch64 target, this is the directory
/// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`.
/// Also works for dynamic musl.
libc_runtimes_dir: ?[]const u8 = null,
enable_wine: bool = false,
enable_qemu: bool = false,
enable_wasmtime: bool = false,
enable_darling: bool = false,
enable_rosetta: bool = false,
/// Intention of verbose is to print all sub-process command lines to stderr
/// before spawning them.
pub fn handleVerbose(
lib/compiler/Maker/Step.zig
+5 -8
View File
@@ -62,12 +62,11 @@ comptime {
// Common cache line size is 128. This check prevents accidentally crossing
// an additional cache line. In the future it might be nice to try to fit
// this struct in 128 bytes or less.
assert(@sizeOf(@This()) <= 128 * 3);
assert(@sizeOf(@This()) <= 128 * 4);
}
pub const Extended = union(enum) {
check_file: Todo,
check_object: Todo,
compile: Compile,
config_header: Todo,
fail: Todo,
@@ -87,7 +86,6 @@ pub const Extended = union(enum) {
pub fn init(tag: Configuration.Step.Tag) Extended {
return switch (tag) {
.check_file => .{ .check_file = .{} },
.check_object => .{ .check_object = .{} },
.compile => .{ .compile = .{} },
.config_header => .{ .config_header = .{} },
.fail => .{ .fail = .{} },
@@ -645,9 +643,8 @@ fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void {
/// Asserts that the caller has already populated `s.result_failed_command`.
pub inline fn handleChildProcUnsupported(s: *Step, maker: *Maker) FailError!void {
assert(s.result_failed_command != null);
if (!std.process.can_spawn) {
if (!std.process.can_spawn)
return s.fail(maker, "unable to spawn process: host cannot spawn child processes", .{});
}
}
/// Asserts that the caller has already populated `s.result_failed_command`.
@@ -708,10 +705,10 @@ fn failWithCacheError(
/// Prefer `writeManifestAndWatch` unless you already added watch inputs
/// separately from using the cache system.
pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void {
pub fn writeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void {
if (s.test_results.isSuccess()) {
man.writeManifest() catch |err| {
try s.addError("unable to write cache manifest: {t}", .{err});
try s.addError(maker, "unable to write cache manifest: {t}", .{err});
};
}
}
@@ -721,7 +718,7 @@ pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void {
///
/// Must be accompanied with `cacheHitAndWatch`.
pub fn writeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !void {
try writeManifest(s, man);
try writeManifest(s, maker, man);
try setWatchInputsFromManifest(s, maker, man);
}
lib/compiler/Maker/Step/Run.zig
+310 -226
View File
@@ -16,6 +16,7 @@ const allocPrint = std.fmt.allocPrint;
const Step = @import("../Step.zig");
const Maker = @import("../../Maker.zig");
const Fuzz = @import("../../Maker/Fuzz.zig");
/// If this is a Zig unit test binary, this tracks the names of the unit
/// tests that are also fuzz tests. Indexes cannot be used as they may
@@ -31,6 +32,8 @@ rebuilt_executable: ?Path = null,
argv: std.ArrayList([]const u8) = .empty,
/// Persisted to reuse memory on subsequent calls to `make`.
output_placeholders: std.ArrayList(IndexedOutput) = .empty,
/// Persisted to reuse memory on subsequent calls to `make`.
environ_map: std.process.Environ.Map = .{ .array_hash_map = .empty, .allocator = undefined },
pub fn make(
run: *Run,
@@ -67,6 +70,8 @@ pub fn make(
man.hash.add(conf_run.flags.color);
man.hash.add(conf_run.flags.disable_zig_progress);
var dep_file_count: usize = 0;
for (conf_run.args.slice) |arg_index| {
const arg = arg_index.get(conf);
try argv_list.ensureUnusedCapacity(gpa, 1);
@@ -157,6 +162,9 @@ pub fn make(
man.hash.addBytesZ(basename);
man.hash.addBytesZ(suffix);
man.hash.add(arg.flags.dep_file);
dep_file_count += @intFromBool(arg.flags.dep_file);
// Add a placeholder into the argument list because we need the
// manifest hash to be updated with all arguments before the
// object directory is computed.
@@ -220,145 +228,95 @@ pub fn make(
const has_side_effects = conf_run.flags.has_side_effects;
if (true) @panic("TODO");
if (!has_side_effects and try step.cacheHitAndWatch(maker, &man)) {
// cache hit, skip running command
// Cache hit; skip running command.
const digest = man.final();
try populateGeneratedPaths(
arena,
output_placeholders.items,
&conf_run,
cache_root,
&digest,
);
try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest);
try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest);
step.result_cached = true;
return;
}
const dep_output_file = conf_run.dep_output_file orelse {
// We already know the final output paths, use them directly.
const digest = if (has_side_effects)
man.hash.final()
else
man.final();
try populateGeneratedPaths(
arena,
output_placeholders.items,
&conf_run,
cache_root,
&digest,
);
if (dep_file_count == 0) {
// We already know the final output paths; use them directly.
const digest = if (has_side_effects) man.hash.final() else man.final();
const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest;
for (output_placeholders.items) |placeholder| {
const output_sub_path = graph.pathJoin(&.{ output_dir_path, placeholder.output.basename });
const output_sub_dir_path = switch (placeholder.tag) {
.output_file => Dir.path.dirname(output_sub_path).?,
.output_directory => output_sub_path,
else => unreachable,
};
cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| {
return step.fail(maker, "unable to make path '{f}{s}': {t}", .{
cache_root, output_sub_dir_path, err,
});
};
const arg_output_path = try convertPathArg(run_index, maker, .{
.root_dir = .cwd(),
.sub_path = placeholder.output.generated_file.getPath(),
});
argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0)
arg_output_path
else
try allocPrint(arena, "{s}{s}", .{ placeholder.output.prefix, arg_output_path });
}
try runCommand(run, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null);
if (!has_side_effects) try step.writeManifestAndWatch(&man);
try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest);
try populateGeneratedPathsCreateDirs(run, run_index, maker, output_dir_path);
try runCommand(run, run_index, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null);
if (!has_side_effects) try step.writeManifestAndWatch(maker, &man);
return;
};
}
// We do not know the final output paths yet, use temp paths to run the command.
// We do not know the final output paths yet; use temporary directory to run the command.
var rand_int: u64 = undefined;
io.random(@ptrCast(&rand_int));
const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int);
try populateGeneratedPathsCreateDirs(run, run_index, maker, tmp_dir_path);
try runCommand(run, run_index, maker, progress_node, argv_list.items, has_side_effects, tmp_dir_path, null);
for (output_placeholders.items) |placeholder| {
const output_components = .{ tmp_dir_path, placeholder.output.basename };
const output_sub_path = graph.pathJoin(&output_components);
const output_sub_dir_path = switch (placeholder.tag) {
.output_file => Dir.path.dirname(output_sub_path).?,
.output_directory => output_sub_path,
else => unreachable,
};
cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| {
return step.fail(maker, "unable to make path '{f}{s}': {t}", .{
cache_root, output_sub_dir_path, err,
});
};
const raw_output_path: Path = .{
.root_dir = cache_root,
.sub_path = graph.pathJoin(&output_components),
};
placeholder.output.generated_file.path = raw_output_path.toString(arena) catch @panic("OOM");
argv_list.items[placeholder.index] = try mem.concat(arena, u8, .{
placeholder.output.prefix,
try convertPathArg(run_index, maker, raw_output_path),
});
}
try runCommand(run, maker, progress_node, argv_list.items, has_side_effects, tmp_dir_path, null);
const dep_file_dir = Dir.cwd();
const dep_file_basename = dep_output_file.generated_file.getPath2(graph, step);
if (has_side_effects)
try man.addDepFile(dep_file_dir, dep_file_basename)
else
try man.addDepFilePost(dep_file_dir, dep_file_basename);
const digest = if (has_side_effects)
man.hash.final()
else
man.final();
const any_output = output_placeholders.items.len > 0 or
conf_run.captured_stdout != null or conf_run.captured_stderr != null;
// Rename into place
if (any_output) {
const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest;
cache_root.handle.rename(tmp_dir_path, cache_root.handle, o_sub_path, io) catch |err| switch (err) {
Dir.RenameError.DirNotEmpty => {
cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| {
return step.fail(maker, "unable to remove dir '{f}'{s}: {t}", .{
cache_root, tmp_dir_path, del_err,
});
};
cache_root.handle.rename(tmp_dir_path, cache_root.handle, o_sub_path, io) catch |retry_err| {
return step.fail(maker, "unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{
cache_root, tmp_dir_path, cache_root, o_sub_path, retry_err,
});
const arg = placeholder.arg_index.get(conf);
switch (arg.flags.tag) {
.output_file => if (arg.flags.dep_file) {
const generated_path = maker.generatedPath(arg.generated.value.?).*;
const result = if (has_side_effects)
man.addDepFile(generated_path.root_dir.handle, generated_path.sub_path)
else
man.addDepFilePost(generated_path.root_dir.handle, generated_path.sub_path);
result catch |err| switch (err) {
error.OutOfMemory, error.Canceled => |e| return e,
else => |e| return step.fail(maker, "failed adding to cache the file {f}: {t}", .{
generated_path, e,
}),
};
},
else => return step.fail(maker, "unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{
cache_root, tmp_dir_path, cache_root, o_sub_path, err,
.output_directory => continue,
else => unreachable,
}
}
const digest = if (has_side_effects) man.hash.final() else man.final();
const any_output = output_placeholders.items.len > 0 or
conf_run.captured_stdout.value != null or conf_run.captured_stderr.value != null;
if (any_output) {
// Rename into place.
const tmp_path: Path = .{ .root_dir = cache_root, .sub_path = tmp_dir_path };
const dst_path: Path = .{ .root_dir = cache_root, .sub_path = "o" ++ Dir.path.sep_str ++ &digest };
Dir.rename(
tmp_path.root_dir.handle,
tmp_path.sub_path,
dst_path.root_dir.handle,
dst_path.sub_path,
io,
) catch |err| switch (err) {
error.DirNotEmpty => {
dst_path.root_dir.handle.deleteTree(io, dst_path.sub_path) catch |del_err|
return step.fail(maker, "failed to remove tree {f}: {t}", .{ dst_path, del_err });
Dir.rename(
tmp_path.root_dir.handle,
tmp_path.sub_path,
dst_path.root_dir.handle,
dst_path.sub_path,
io,
) catch |retry_err| return step.fail(maker, "failed to rename directory {f} to {f}: {t}", .{
tmp_path, dst_path, retry_err,
});
},
else => return step.fail(maker, "failed to rename directory {f} to {f}: {t}", .{
tmp_path, dst_path, err,
}),
};
}
if (!has_side_effects) try step.writeManifestAndWatch(&man);
if (!has_side_effects) try step.writeManifestAndWatch(maker, &man);
try populateGeneratedPaths(
arena,
output_placeholders.items,
&conf_run,
cache_root,
&digest,
);
try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest);
try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest);
}
/// Reads stdout of a Zig test process until a termination condition is reached:
@@ -918,9 +876,12 @@ const FuzzTestRunner = struct {
}
fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void {
const fuzz = f.context.fuzz;
const maker = fuzz.maker;
const step = &f.run.step;
const b = step.owner;
const io = b.graph.io;
const graph = maker.graph;
const io = graph.io;
const cache_root = graph.local_cache_root;
if (f.coverage_id == null) return;
@@ -938,7 +899,7 @@ const FuzzTestRunner = struct {
}) {
const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in";
in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable;
in_f = b.cache_root.handle.openFile(io, in_name, .{
in_f = cache_root.handle.openFile(io, in_name, .{
.lock = .exclusive,
.lock_nonblocking = true,
}) catch |e| switch (e) {
@@ -946,7 +907,7 @@ const FuzzTestRunner = struct {
error.WouldBlock => continue, // Can not be from
// the crashed instance since it is still locked.
else => return step.fail("failed to open file '{f}{s}': {t}", .{
b.cache_root, in_name, e,
cache_root, in_name, e,
}),
};
@@ -955,7 +916,7 @@ const FuzzTestRunner = struct {
in_f.close(io);
switch (e) {
error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{
b.cache_root, in_name, in_r.err.?,
cache_root, in_name, in_r.err.?,
}),
error.EndOfStream => continue,
}
@@ -974,10 +935,10 @@ const FuzzTestRunner = struct {
// Save it to a seperate file
const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash";
const out = b.cache_root.handle.createFile(io, crash_name, .{
const out = cache_root.handle.createFile(io, crash_name, .{
.lock = .exclusive, // Multiple run steps could have found a crash at the same time
}) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{
b.cache_root, crash_name, e,
cache_root, crash_name, e,
});
defer out.close(io);
@@ -985,17 +946,17 @@ const FuzzTestRunner = struct {
var out_w = out.writerStreaming(io, &out_w_buf);
_ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) {
error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{
b.cache_root, in_name, in_r.err.?,
cache_root, in_name, in_r.err.?,
}),
error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{
b.cache_root, crash_name, out_w.err.?,
cache_root, crash_name, out_w.err.?,
}),
};
return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{
f.run.fuzz_tests.items[header.test_i],
fmtTerm(term),
b.cache_root,
cache_root,
crash_name,
});
}
@@ -1492,54 +1453,85 @@ pub fn rerunInFuzzMode(
const maker = fuzz.maker;
const graph = maker.graph;
const step = &run.step;
const b = step.owner;
const io = graph.io;
const arena = b.allocator;
var argv_list: std.ArrayList([]const u8) = .empty;
for (run.argv.items) |arg| {
switch (arg) {
.bytes => |bytes| {
try argv_list.append(arena, bytes);
},
.lazy_path => |file| {
const file_path = file.lazy_path.getPath3(b, step);
try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, convertPathArg(run_index, maker, file_path) }));
},
.decorated_directory => |dd| {
const file_path = dd.lazy_path.getPath3(b, step);
try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, convertPathArg(run_index, maker, file_path), dd.suffix }));
},
.file_content => |file_plp| {
const file_path = file_plp.lazy_path.getPath3(b, step);
const arena = graph.arena; // TODO don't leak into the process arena
const gpa = maker.gpa;
const conf = &maker.scanned_config.configuration;
const conf_step = run_index.ptr(conf);
const conf_run = conf_step.extended.get(conf.extra).run;
const argv_list = &run.argv;
var result: std.Io.Writer.Allocating = .init(arena);
errdefer result.deinit();
result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory;
argv_list.clearRetainingCapacity();
const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{});
defer file.close(io);
var buf: [1024]u8 = undefined;
var file_reader = file.reader(io, &buf);
_ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) {
error.ReadFailed => return file_reader.err.?,
error.WriteFailed => return error.OutOfMemory,
};
try argv_list.append(arena, result.written());
for (conf_run.args.slice) |arg_index| {
const arg = arg_index.get(conf);
try argv_list.ensureUnusedCapacity(gpa, 1);
switch (arg.flags.tag) {
.string => {
const prefix = arg.prefix.value.?.slice(conf);
argv_list.appendAssumeCapacity(prefix);
},
.artifact => |pa| {
const artifact = pa.artifact;
const file_path: []const u8 = p: {
if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?});
break :p artifact.installed_path orelse artifact.generated_bin.?.path.?;
};
try argv_list.append(arena, b.fmt("{s}{s}", .{
pa.prefix,
convertPathArg(run_index, maker, .{ .root_dir = .cwd(), .sub_path = file_path }),
.path_file => {
const prefix = if (arg.prefix.value) |p| p.slice(conf) else "";
const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index);
argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{
prefix, try convertPathArg(run_index, maker, file_path), suffix,
}));
},
.output_file, .output_directory => unreachable,
.path_directory => {
const prefix = if (arg.prefix.value) |p| p.slice(conf) else "";
const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index);
const resolved_arg = try mem.concat(arena, u8, &.{
prefix, try convertPathArg(run_index, maker, file_path), suffix,
});
argv_list.appendAssumeCapacity(resolved_arg);
},
.file_content => {
const prefix = if (arg.prefix.value) |p| p.slice(conf) else "";
const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index);
var result: std.Io.Writer.Allocating = .init(arena);
result.writer.writeAll(prefix) catch return error.OutOfMemory;
const file = file_path.root_dir.handle.openFile(io, file_path.sub_path, .{}) catch |err|
return step.fail(maker, "unable to open input file {f}: {t}", .{ file_path, err });
defer file.close(io);
var file_reader = file.reader(io, &.{});
_ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) {
error.ReadFailed => switch (file_reader.err.?) {
error.Canceled => |e| return e,
else => |e| return step.fail(maker, "failed to read from {f}: {t}", .{ file_path, e }),
},
error.WriteFailed => return error.OutOfMemory,
};
result.writer.writeAll(suffix) catch return error.OutOfMemory;
argv_list.appendAssumeCapacity(result.written());
},
.artifact => {
const prefix = if (arg.prefix.value) |p| p.slice(conf) else "";
const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
const producer_index = arg.producer.value.?;
const producer_step = producer_index.ptr(conf);
const producer = producer_step.extended.get(conf.extra).compile;
const producer_make_comp_step = maker.stepByIndex(producer_index);
const producer_make_comp = &producer_make_comp_step.extended.compile;
const file_path: Path = if (producer_index == conf_run.producer.value.?)
run.rebuilt_executable.?
else
producer_make_comp.installed_path orelse
maker.generatedPath(producer.generated_bin.value.?).*;
argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{
prefix, try convertPathArg(run_index, maker, file_path), suffix,
}));
},
.output_file => unreachable,
.output_directory => unreachable,
.cli_rest_positionals => unreachable,
}
}
@@ -1552,7 +1544,7 @@ pub fn rerunInFuzzMode(
var rand_int: u64 = undefined;
io.random(@ptrCast(&rand_int));
const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int);
try runCommand(run, maker, prog_node, argv_list.items, has_side_effects, tmp_dir_path, .{
try runCommand(run, run_index, maker, prog_node, argv_list.items, has_side_effects, tmp_dir_path, .{
.fuzz = fuzz,
});
}
@@ -1560,28 +1552,94 @@ pub fn rerunInFuzzMode(
const CapturedStdIo = void; // TODO get it from Configuration
fn populateGeneratedPaths(
arena: std.mem.Allocator,
maker: *Maker,
output_placeholders: []const IndexedOutput,
cache_root: Cache.Directory,
digest: *const Cache.HexDigest,
) !void {
const conf = &maker.scanned_config.configuration;
const graph = maker.graph;
const arena = graph.arena; // TODO don't leak into the process arena
for (output_placeholders) |placeholder| {
const arg = placeholder.arg_index.get(conf);
maker.generatedPath(arg.generated.value.?).* = .{
.root_dir = cache_root,
.sub_path = try Dir.path.join(arena, &.{
"o", digest, arg.basename.value.?.slice(conf),
}),
};
}
}
fn populateGeneratedPathsCreateDirs(
run: *Run,
run_index: Configuration.Step.Index,
maker: *Maker,
output_dir_path: []const u8,
) !void {
const step = maker.stepByIndex(run_index);
const conf = &maker.scanned_config.configuration;
const graph = maker.graph;
const io = graph.io;
const arena = graph.arena; // TODO don't leak into the process arena
const cache_root = graph.local_cache_root;
const argv = run.argv.items;
for (run.output_placeholders.items) |placeholder| {
const arg = placeholder.arg_index.get(conf);
const prefix = if (arg.prefix.value) |p| p.slice(conf) else "";
const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
const basename = arg.basename.value.?.slice(conf);
const generated_path: Path = .{
.root_dir = cache_root,
.sub_path = try Dir.path.join(arena, &.{ output_dir_path, basename }),
};
const create_path: Path = .{
.root_dir = cache_root,
.sub_path = switch (arg.flags.tag) {
.output_file => Dir.path.dirname(generated_path.sub_path).?,
.output_directory => generated_path.sub_path,
else => unreachable,
},
};
create_path.root_dir.handle.createDirPath(io, create_path.sub_path) catch |err|
return step.fail(maker, "unable to make path {f}: {t}", .{ create_path, err });
maker.generatedPath(arg.generated.value.?).* = generated_path;
const arg_output_path = try convertPathArg(run_index, maker, generated_path);
argv[placeholder.index] = try mem.concat(arena, u8, &.{ prefix, arg_output_path, suffix });
}
}
fn populateGeneratedStdIo(
maker: *Maker,
conf_run: *const Configuration.Step.Run,
cache_root: Cache.Directory,
digest: *const Cache.HexDigest,
) !void {
for (output_placeholders) |placeholder| {
placeholder.output.generated_file.path = try cache_root.join(arena, &.{
"o", digest, placeholder.output.basename,
});
}
const conf = &maker.scanned_config.configuration;
const graph = maker.graph;
const arena = graph.arena; // TODO don't leak into the process arena
if (conf_run.captured_stdout.value) |captured| {
captured.output.generated_file.path = try cache_root.join(arena, &.{
"o", digest, captured.output.basename,
});
maker.generatedPath(captured.generated_file).* = .{
.root_dir = cache_root,
.sub_path = try Dir.path.join(arena, &.{
"o", digest, captured.basename.slice(conf),
}),
};
}
if (conf_run.captured_stderr.value) |captured| {
captured.output.generated_file.path = try cache_root.join(arena, &.{
"o", digest, captured.output.basename,
});
maker.generatedPath(captured.generated_file).* = .{
.root_dir = cache_root,
.sub_path = try Dir.path.join(arena, &.{
"o", digest, captured.basename.slice(conf),
}),
};
}
}
@@ -1600,11 +1658,12 @@ fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTer
}
const FuzzContext = struct {
fuzz: *std.Build.Fuzz,
fuzz: *Fuzz,
};
fn runCommand(
run: *Run,
run_index: Configuration.Step.Index,
maker: *Maker,
progress_node: std.Progress.Node,
argv: []const []const u8,
@@ -1615,30 +1674,45 @@ fn runCommand(
const graph = maker.graph;
const arena = graph.arena; // TODO don't leak into process arena
const gpa = maker.gpa;
const step = &run.step;
const b = step.owner;
const step = maker.stepByIndex(run_index);
const io = graph.io;
const cache_root = graph.local_cache_root;
const conf = &maker.scanned_config.configuration;
const conf_step = run_index.ptr(conf);
const conf_run = conf_step.extended.get(conf.extra).run;
const environ_map = &run.environ_map;
const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit;
const cwd: process.Child.Cwd = if (conf_run.cwd.value) |lazy_cwd|
.{ .path = try maker.resolveLazyPathIndexAbs(arena, lazy_cwd, run_index) }
else
.inherit;
try step.handleChildProcUnsupported();
try Step.handleVerbose(step.owner, cwd, run.environ_map, argv);
const allow_skip = switch (run.stdio) {
.check, .zig_test => run.skip_foreign_checks,
const allow_skip = switch (conf_run.flags.stdio) {
.check, .zig_test => conf_run.flags.skip_foreign_checks,
else => false,
};
var interp_argv = std.array_list.Managed([]const u8).init(b.allocator);
defer interp_argv.deinit();
var interp_argv: std.ArrayList([]const u8) = .empty;
var environ_map: EnvMap = env: {
const orig = run.environ_map orelse &graph.environ_map;
break :env try orig.clone(gpa);
};
defer environ_map.deinit();
// `environ_map` is initialized with an undefined `allocator` field; lazily
// initialize it here.
environ_map.allocator = gpa;
// In either case we add to this mutatable data structure so that we can
// tweak the environment below.
environ_map.clearRetainingCapacity();
if (conf_run.environ_map.value) |env_map_index| {
const conf_env_map = env_map_index.get(conf);
for (conf_env_map.keys.slice(conf), conf_env_map.values.slice(conf)) |k, v| {
try environ_map.put(k.slice(conf), v.slice(conf));
}
} else {
try environ_map.putAll(&graph.environ_map);
}
try graph.handleVerbose(cwd, environ_map, argv);
const opt_generic_result = spawnChildAndCollect(run, maker, progress_node, argv, &environ_map, has_side_effects, fuzz_context) catch |err| term: {
if (true) @panic("TODO");
const opt_generic_result = spawnChildAndCollect(run_index, run, maker, progress_node, argv, &environ_map, has_side_effects, fuzz_context) catch |err| term: {
// InvalidExe: cpu arch mismatch
// FileNotFound: can happen with a wrong dynamic linker path
if (err == error.InvalidExe or err == error.FileNotFound) interpret: {
@@ -1660,7 +1734,7 @@ fn runCommand(
(root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic));
const other_target = exe.root_module.resolved_target.?.result;
switch (std.zig.system.getExternalExecutor(io, &graph.host.result, &other_target, .{
.qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null,
.qemu_fixes_dl = need_cross_libc and graph.libc_runtimes_dir != null,
.link_libc = exe.is_linking_libc,
})) {
.native, .rosetta => {
@@ -1668,7 +1742,7 @@ fn runCommand(
break :interpret;
},
.wine => |bin_name| {
if (b.enable_wine) {
if (graph.enable_wine) {
try interp_argv.append(bin_name);
try interp_argv.appendSlice(argv);
@@ -1682,21 +1756,21 @@ fn runCommand(
}
},
.qemu => |bin_name| {
if (b.enable_qemu) {
if (graph.enable_qemu) {
try interp_argv.append(bin_name);
if (need_cross_libc) {
if (b.libc_runtimes_dir) |dir| {
if (graph.libc_runtimes_dir) |dir| {
try interp_argv.append("-L");
try interp_argv.append(b.pathJoin(&.{
try interp_argv.append(try Dir.path.join(arena, &.{
dir,
try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple(
b.allocator,
arena,
root_target.cpu.arch,
root_target.os.tag,
root_target.abi,
) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple(
b.allocator,
arena,
root_target.cpu.arch,
root_target.abi,
) else unreachable,
@@ -1708,7 +1782,7 @@ fn runCommand(
} else return failForeign(run, "-fqemu", argv[0], exe);
},
.darling => |bin_name| {
if (b.enable_darling) {
if (graph.enable_darling) {
try interp_argv.append(bin_name);
try interp_argv.appendSlice(argv);
} else {
@@ -1716,7 +1790,7 @@ fn runCommand(
}
},
.wasmtime => |bin_name| {
if (b.enable_wasmtime) {
if (graph.enable_wasmtime) {
try interp_argv.append(bin_name);
try interp_argv.append("--dir=.");
// Wasmtime doeesn't inherit environment variables from the parent process
@@ -1743,8 +1817,8 @@ fn runCommand(
.bad_os_or_cpu => {
if (allow_skip) return error.MakeSkipped;
const host_name = try graph.host.result.zigTriple(b.allocator);
const foreign_name = try root_target.zigTriple(b.allocator);
const host_name = try graph.host.result.zigTriple(arena);
const foreign_name = try root_target.zigTriple(arena);
return step.fail(maker, "the host system ({s}) is unable to execute binaries from the target ({s})", .{
host_name, foreign_name,
@@ -1761,7 +1835,7 @@ fn runCommand(
step.result_failed_command = null;
try Step.handleVerbose(step.owner, cwd, run.environ_map, interp_argv.items);
break :term spawnChildAndCollect(run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| {
break :term spawnChildAndCollect(run_index, run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| {
if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped;
if (e == error.MakeFailed) return error.MakeFailed; // error already reported
return step.fail(maker, "unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e });
@@ -1800,14 +1874,14 @@ fn runCommand(
}) |stream| {
if (stream.captured) |captured| {
const output_components = .{ output_dir_path, captured.output.basename };
const output_path = try b.cache_root.join(arena, &output_components);
const output_path = try cache_root.join(arena, &output_components);
captured.output.generated_file.path = output_path;
const sub_path = b.pathJoin(&output_components);
const sub_path = try Dir.path.join(arena, &output_components);
const sub_path_dirname = Dir.path.dirname(sub_path).?;
b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| {
return step.fail(maker, "unable to make path '{f}{s}': {s}", .{
b.cache_root, sub_path_dirname, @errorName(err),
cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| {
return step.fail(maker, "unable to make path '{f}{s}': {t}", .{
cache_root, sub_path_dirname, err,
});
};
const data = switch (captured.trim_whitespace) {
@@ -1816,9 +1890,9 @@ fn runCommand(
.leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace),
.trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace),
};
b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| {
return step.fail(maker, "unable to write file '{f}{s}': {s}", .{
b.cache_root, sub_path, @errorName(err),
cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| {
return step.fail(maker, "unable to write file '{f}{s}': {t}", .{
cache_root, sub_path, err,
});
};
}
@@ -1912,6 +1986,7 @@ const EvalGenericResult = struct {
};
fn spawnChildAndCollect(
run_index: Configuration.Step.Index,
run: *Run,
maker: *Maker,
progress_node: std.Progress.Node,
@@ -1920,25 +1995,34 @@ fn spawnChildAndCollect(
has_side_effects: bool,
fuzz_context: ?FuzzContext,
) !?EvalGenericResult {
const b = run.step.owner;
const step = run.step;
const graph = maker.graph;
const gpa = maker.gpa;
const io = graph.io;
const arena = graph.arena; // TODO don't leak into process arena
const conf = &maker.scanned_config.configuration;
const conf_step = run_index.ptr(conf);
const conf_run = conf_step.extended.get(conf.extra).run;
if (fuzz_context != null) {
assert(!has_side_effects);
assert(run.stdio == .zig_test);
}
const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit;
const child_cwd: process.Child.Cwd = if (conf_run.cwd) |lazy_cwd|
.{ .path = try maker.resolveLazyPathIndexAbs(arena, lazy_cwd, run_index) }
else
.inherit;
// If an error occurs, it's caused by this command:
assert(run.step.result_failed_command == null);
run.step.result_failed_command = try Step.allocPrintCmd(gpa, child_cwd, .{
assert(step.result_failed_command == null);
step.result_failed_command = try Step.allocPrintCmd(gpa, child_cwd, .{
.child = environ_map,
.parent = &graph.environ_map,
}, argv);
try step.handleChildProcUnsupported(maker);
var spawn_options: process.SpawnOptions = .{
.argv = argv,
.cwd = child_cwd,
@@ -1973,7 +2057,7 @@ fn spawnChildAndCollect(
error.Canceled => |e| return e,
else => |e| e,
};
run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds);
step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds);
try result;
return null;
} else {
@@ -1993,7 +2077,7 @@ fn spawnChildAndCollect(
error.Canceled => |e| return e,
else => |e| e,
};
run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds);
step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds);
return try result;
}
}
lib/compiler/configurer.zig
+2 -3
View File
@@ -455,7 +455,7 @@ const Serialize = struct {
.producer = .{ .value = null },
.generated = .{ .value = null },
},
.output_file => |a| .{
.output_file, .output_file_dep => |a, tag| .{
.flags = .{
.tag = .output_file,
.prefix = a.prefix.len != 0,
@@ -464,7 +464,7 @@ const Serialize = struct {
.path = false,
.producer = false,
.generated = true,
.dep_file = false,
.dep_file = tag == .output_file_dep,
},
.prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null },
.suffix = .{ .value = null },
@@ -1023,7 +1023,6 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
break :e @enumFromInt(extra_index);
},
.check_file => @panic("TODO"),
.check_object => @panic("TODO"),
.config_header => @panic("TODO"),
.objcopy => @panic("TODO"),
.options => @panic("TODO"),
lib/std/Build/Configuration.zig
-11
View File
@@ -427,7 +427,6 @@ pub const Step = extern struct {
max_rss: MaxRss,
extended: Storage.Extended(Flags, union(Tag) {
check_file: CheckFile,
check_object: CheckObject,
compile: Compile,
config_header: ConfigHeader,
fail: Fail,
@@ -462,7 +461,6 @@ pub const Step = extern struct {
pub const Tag = enum(u5) {
check_file,
check_object,
compile,
config_header,
fail,
@@ -997,15 +995,6 @@ pub const Step = extern struct {
};
};
pub const CheckObject = struct {
flags: @This().Flags,
pub const Flags = packed struct(u32) {
tag: Tag = .check_object,
_: u27 = 0,
};
};
pub const ConfigHeader = struct {
flags: @This().Flags,
lib/std/Build/Step/Run.zig
+3 -9
View File
@@ -85,8 +85,6 @@ stdio_limit: std.Io.Limit,
captured_stdout: ?*CapturedStdIo,
captured_stderr: ?*CapturedStdIo,
dep_output_file: ?*Output,
has_side_effects: bool,
test_runner_mode: bool = false,
@@ -141,6 +139,7 @@ pub const Arg = union(enum) {
file_content: PrefixedLazyPath,
bytes: []const u8,
output_file: *Output,
output_file_dep: *Output,
output_directory: *Output,
/// The arguments passed after "--" on the "zig build" CLI.
cli_rest_positionals,
@@ -203,7 +202,6 @@ pub fn create(owner: *std.Build, name: []const u8) *Run {
.stdio_limit = .unlimited,
.captured_stdout = null,
.captured_stderr = null,
.dep_output_file = null,
.has_side_effects = false,
.producer = null,
};
@@ -476,12 +474,10 @@ pub fn addDepFileOutputArg(run: *Run, basename: []const u8) std.Build.LazyPath {
/// Add a prefixed path argument to a dep file (.d) for the child process to
/// write its discovered additional dependencies.
/// Only one dep file argument is allowed by instance.
pub fn addPrefixedDepFileOutputArg(run: *Run, prefix: []const u8, basename: []const u8) std.Build.LazyPath {
const b = run.step.owner;
const graph = b.graph;
const arena = graph.arena;
assert(run.dep_output_file == null);
const dep_file = arena.create(Output) catch @panic("OOM");
dep_file.* = .{
@@ -490,9 +486,7 @@ pub fn addPrefixedDepFileOutputArg(run: *Run, prefix: []const u8, basename: []co
.generated_file = graph.addGeneratedFile(&run.step),
};
run.dep_output_file = dep_file;
run.argv.append(arena, .{ .output_file = dep_file }) catch @panic("OOM");
run.argv.append(arena, .{ .output_file_dep = dep_file }) catch @panic("OOM");
return .{ .generated = .{ .index = dep_file.generated_file } };
}
@@ -544,7 +538,7 @@ pub fn addPathDir(run: *Run, search_path: []const u8) void {
.decorated_directory => false,
.file_content => unreachable, // not allowed as first arg
.bytes => |bytes| std.mem.endsWith(u8, bytes, ".exe"),
.output_file, .output_directory => false,
.output_file, .output_file_dep, .output_directory => false,
};
const key = if (use_wine) "WINEPATH" else "PATH";
const prev_path = environ_map.get(key);