diff --git a/BRANCH_TODO b/BRANCH_TODO index 51e8ad28c9..7b28f34c27 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -18,6 +18,7 @@ * https://codeberg.org/ziglang/zig/pulls/30762 ## Followup Issues +* reduce the size of Maker.Step.Extended (make Run smaller) probably by using an arena per make * link_eh_frame_hdr should be DefaultingBool * make --foo, --no-foo CLI args uniform (make them -f args instead) * install steps should provide generated files for installed things, then delete the run step hack diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 2b8b163447..0ba2cfbbfc 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -153,17 +153,6 @@ pub fn main(init: process.Init.Minimal) !void { var debounce_interval_ms: u16 = 50; var webui_listen: ?Io.net.IpAddress = null; var debug_pkg_config: bool = false; - // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, - // this will be the directory $glibc-build-dir/install/glibcs - // Given the example of the aarch64 target, this is the directory - // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. - // Also works for dynamic musl. - var libc_runtimes_dir: ?[]const u8 = null; - var enable_wine = false; - var enable_qemu = false; - var enable_wasmtime = false; - var enable_darling = false; - var enable_rosetta = false; var run_args: ?[]const []const u8 = null; if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { @@ -314,7 +303,7 @@ pub fn main(init: process.Init.Minimal) !void { fatal("unrecognized optimization mode: {s}", .{rest}); } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); + graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); } else if (mem.eql(u8, arg, "--verbose")) { graph.verbose = true; } else if (mem.eql(u8, arg, "--verbose-air")) { @@ -370,25 +359,25 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "-fno-incremental")) { graph.incremental = false; } else if (mem.eql(u8, arg, "-fwine")) { - enable_wine = true; + graph.enable_wine = true; } else if (mem.eql(u8, arg, "-fno-wine")) { - enable_wine = false; + graph.enable_wine = false; } else if (mem.eql(u8, arg, "-fqemu")) { - enable_qemu = true; + graph.enable_qemu = true; } else if (mem.eql(u8, arg, "-fno-qemu")) { - enable_qemu = false; + graph.enable_qemu = false; } else if (mem.eql(u8, arg, "-fwasmtime")) { - enable_wasmtime = true; + graph.enable_wasmtime = true; } else if (mem.eql(u8, arg, "-fno-wasmtime")) { - enable_wasmtime = false; + graph.enable_wasmtime = false; } else if (mem.eql(u8, arg, "-frosetta")) { - enable_rosetta = true; + graph.enable_rosetta = true; } else if (mem.eql(u8, arg, "-fno-rosetta")) { - enable_rosetta = false; + graph.enable_rosetta = false; } else if (mem.eql(u8, arg, "-fdarling")) { - enable_darling = true; + graph.enable_darling = true; } else if (mem.eql(u8, arg, "-fno-darling")) { - enable_darling = false; + graph.enable_darling = false; } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { graph.allow_so_scripts = true; } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { @@ -533,6 +522,7 @@ pub fn main(init: process.Init.Minimal) !void { .bin = install_bin_path, .include = install_include_path, }, + .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), .generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len), .run_args = run_args, diff --git a/lib/compiler/Maker/Graph.zig b/lib/compiler/Maker/Graph.zig index 50dba7f931..cb96300349 100644 --- a/lib/compiler/Maker/Graph.zig +++ b/lib/compiler/Maker/Graph.zig @@ -52,6 +52,18 @@ error_limit: ?u32 = null, /// a single step spawning a fixed number of processes this can be used. max_jobs: ?u32 = null, +/// After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, +/// this will be the directory $glibc-build-dir/install/glibcs +/// Given the example of the aarch64 target, this is the directory +/// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. +/// Also works for dynamic musl. +libc_runtimes_dir: ?[]const u8 = null, +enable_wine: bool = false, +enable_qemu: bool = false, +enable_wasmtime: bool = false, +enable_darling: bool = false, +enable_rosetta: bool = false, + /// Intention of verbose is to print all sub-process command lines to stderr /// before spawning them. pub fn handleVerbose( diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index c1c9c1f2cc..8651ec8667 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -62,12 +62,11 @@ comptime { // Common cache line size is 128. This check prevents accidentally crossing // an additional cache line. In the future it might be nice to try to fit // this struct in 128 bytes or less. - assert(@sizeOf(@This()) <= 128 * 3); + assert(@sizeOf(@This()) <= 128 * 4); } pub const Extended = union(enum) { check_file: Todo, - check_object: Todo, compile: Compile, config_header: Todo, fail: Todo, @@ -87,7 +86,6 @@ pub const Extended = union(enum) { pub fn init(tag: Configuration.Step.Tag) Extended { return switch (tag) { .check_file => .{ .check_file = .{} }, - .check_object => .{ .check_object = .{} }, .compile => .{ .compile = .{} }, .config_header => .{ .config_header = .{} }, .fail => .{ .fail = .{} }, @@ -645,9 +643,8 @@ fn sendMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag) !void { /// Asserts that the caller has already populated `s.result_failed_command`. pub inline fn handleChildProcUnsupported(s: *Step, maker: *Maker) FailError!void { assert(s.result_failed_command != null); - if (!std.process.can_spawn) { + if (!std.process.can_spawn) return s.fail(maker, "unable to spawn process: host cannot spawn child processes", .{}); - } } /// Asserts that the caller has already populated `s.result_failed_command`. @@ -708,10 +705,10 @@ fn failWithCacheError( /// Prefer `writeManifestAndWatch` unless you already added watch inputs /// separately from using the cache system. -pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void { +pub fn writeManifest(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { if (s.test_results.isSuccess()) { man.writeManifest() catch |err| { - try s.addError("unable to write cache manifest: {t}", .{err}); + try s.addError(maker, "unable to write cache manifest: {t}", .{err}); }; } } @@ -721,7 +718,7 @@ pub fn writeManifest(s: *Step, man: *Cache.Manifest) !void { /// /// Must be accompanied with `cacheHitAndWatch`. pub fn writeManifestAndWatch(s: *Step, maker: *Maker, man: *Cache.Manifest) !void { - try writeManifest(s, man); + try writeManifest(s, maker, man); try setWatchInputsFromManifest(s, maker, man); } diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 9dc0b0fecf..4d6d491f16 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -16,6 +16,7 @@ const allocPrint = std.fmt.allocPrint; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); +const Fuzz = @import("../../Maker/Fuzz.zig"); /// If this is a Zig unit test binary, this tracks the names of the unit /// tests that are also fuzz tests. Indexes cannot be used as they may @@ -31,6 +32,8 @@ rebuilt_executable: ?Path = null, argv: std.ArrayList([]const u8) = .empty, /// Persisted to reuse memory on subsequent calls to `make`. output_placeholders: std.ArrayList(IndexedOutput) = .empty, +/// Persisted to reuse memory on subsequent calls to `make`. +environ_map: std.process.Environ.Map = .{ .array_hash_map = .empty, .allocator = undefined }, pub fn make( run: *Run, @@ -67,6 +70,8 @@ pub fn make( man.hash.add(conf_run.flags.color); man.hash.add(conf_run.flags.disable_zig_progress); + var dep_file_count: usize = 0; + for (conf_run.args.slice) |arg_index| { const arg = arg_index.get(conf); try argv_list.ensureUnusedCapacity(gpa, 1); @@ -157,6 +162,9 @@ pub fn make( man.hash.addBytesZ(basename); man.hash.addBytesZ(suffix); + man.hash.add(arg.flags.dep_file); + dep_file_count += @intFromBool(arg.flags.dep_file); + // Add a placeholder into the argument list because we need the // manifest hash to be updated with all arguments before the // object directory is computed. @@ -220,145 +228,95 @@ pub fn make( const has_side_effects = conf_run.flags.has_side_effects; - if (true) @panic("TODO"); - if (!has_side_effects and try step.cacheHitAndWatch(maker, &man)) { - // cache hit, skip running command + // Cache hit; skip running command. const digest = man.final(); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - &conf_run, - cache_root, - &digest, - ); - + try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); + try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest); step.result_cached = true; return; } - const dep_output_file = conf_run.dep_output_file orelse { - // We already know the final output paths, use them directly. - const digest = if (has_side_effects) - man.hash.final() - else - man.final(); - - try populateGeneratedPaths( - arena, - output_placeholders.items, - &conf_run, - cache_root, - &digest, - ); - + if (dep_file_count == 0) { + // We already know the final output paths; use them directly. + const digest = if (has_side_effects) man.hash.final() else man.final(); const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; - for (output_placeholders.items) |placeholder| { - const output_sub_path = graph.pathJoin(&.{ output_dir_path, placeholder.output.basename }); - const output_sub_dir_path = switch (placeholder.tag) { - .output_file => Dir.path.dirname(output_sub_path).?, - .output_directory => output_sub_path, - else => unreachable, - }; - cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ - cache_root, output_sub_dir_path, err, - }); - }; - const arg_output_path = try convertPathArg(run_index, maker, .{ - .root_dir = .cwd(), - .sub_path = placeholder.output.generated_file.getPath(), - }); - argv_list.items[placeholder.index] = if (placeholder.output.prefix.len == 0) - arg_output_path - else - try allocPrint(arena, "{s}{s}", .{ placeholder.output.prefix, arg_output_path }); - } - - try runCommand(run, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); - if (!has_side_effects) try step.writeManifestAndWatch(&man); + try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); + try populateGeneratedPathsCreateDirs(run, run_index, maker, output_dir_path); + try runCommand(run, run_index, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); + if (!has_side_effects) try step.writeManifestAndWatch(maker, &man); return; - }; + } - // We do not know the final output paths yet, use temp paths to run the command. + // We do not know the final output paths yet; use temporary directory to run the command. var rand_int: u64 = undefined; io.random(@ptrCast(&rand_int)); const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + try populateGeneratedPathsCreateDirs(run, run_index, maker, tmp_dir_path); + try runCommand(run, run_index, maker, progress_node, argv_list.items, has_side_effects, tmp_dir_path, null); + for (output_placeholders.items) |placeholder| { - const output_components = .{ tmp_dir_path, placeholder.output.basename }; - const output_sub_path = graph.pathJoin(&output_components); - const output_sub_dir_path = switch (placeholder.tag) { - .output_file => Dir.path.dirname(output_sub_path).?, - .output_directory => output_sub_path, - else => unreachable, - }; - cache_root.handle.createDirPath(io, output_sub_dir_path) catch |err| { - return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ - cache_root, output_sub_dir_path, err, - }); - }; - const raw_output_path: Path = .{ - .root_dir = cache_root, - .sub_path = graph.pathJoin(&output_components), - }; - placeholder.output.generated_file.path = raw_output_path.toString(arena) catch @panic("OOM"); - argv_list.items[placeholder.index] = try mem.concat(arena, u8, .{ - placeholder.output.prefix, - try convertPathArg(run_index, maker, raw_output_path), - }); - } - - try runCommand(run, maker, progress_node, argv_list.items, has_side_effects, tmp_dir_path, null); - - const dep_file_dir = Dir.cwd(); - const dep_file_basename = dep_output_file.generated_file.getPath2(graph, step); - if (has_side_effects) - try man.addDepFile(dep_file_dir, dep_file_basename) - else - try man.addDepFilePost(dep_file_dir, dep_file_basename); - - const digest = if (has_side_effects) - man.hash.final() - else - man.final(); - - const any_output = output_placeholders.items.len > 0 or - conf_run.captured_stdout != null or conf_run.captured_stderr != null; - - // Rename into place - if (any_output) { - const o_sub_path = "o" ++ Dir.path.sep_str ++ &digest; - - cache_root.handle.rename(tmp_dir_path, cache_root.handle, o_sub_path, io) catch |err| switch (err) { - Dir.RenameError.DirNotEmpty => { - cache_root.handle.deleteTree(io, o_sub_path) catch |del_err| { - return step.fail(maker, "unable to remove dir '{f}'{s}: {t}", .{ - cache_root, tmp_dir_path, del_err, - }); - }; - cache_root.handle.rename(tmp_dir_path, cache_root.handle, o_sub_path, io) catch |retry_err| { - return step.fail(maker, "unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - cache_root, tmp_dir_path, cache_root, o_sub_path, retry_err, - }); + const arg = placeholder.arg_index.get(conf); + switch (arg.flags.tag) { + .output_file => if (arg.flags.dep_file) { + const generated_path = maker.generatedPath(arg.generated.value.?).*; + const result = if (has_side_effects) + man.addDepFile(generated_path.root_dir.handle, generated_path.sub_path) + else + man.addDepFilePost(generated_path.root_dir.handle, generated_path.sub_path); + result catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + else => |e| return step.fail(maker, "failed adding to cache the file {f}: {t}", .{ + generated_path, e, + }), }; }, - else => return step.fail(maker, "unable to rename dir '{f}{s}' to '{f}{s}': {t}", .{ - cache_root, tmp_dir_path, cache_root, o_sub_path, err, + .output_directory => continue, + else => unreachable, + } + } + + const digest = if (has_side_effects) man.hash.final() else man.final(); + + const any_output = output_placeholders.items.len > 0 or + conf_run.captured_stdout.value != null or conf_run.captured_stderr.value != null; + + if (any_output) { + // Rename into place. + const tmp_path: Path = .{ .root_dir = cache_root, .sub_path = tmp_dir_path }; + const dst_path: Path = .{ .root_dir = cache_root, .sub_path = "o" ++ Dir.path.sep_str ++ &digest }; + Dir.rename( + tmp_path.root_dir.handle, + tmp_path.sub_path, + dst_path.root_dir.handle, + dst_path.sub_path, + io, + ) catch |err| switch (err) { + error.DirNotEmpty => { + dst_path.root_dir.handle.deleteTree(io, dst_path.sub_path) catch |del_err| + return step.fail(maker, "failed to remove tree {f}: {t}", .{ dst_path, del_err }); + + Dir.rename( + tmp_path.root_dir.handle, + tmp_path.sub_path, + dst_path.root_dir.handle, + dst_path.sub_path, + io, + ) catch |retry_err| return step.fail(maker, "failed to rename directory {f} to {f}: {t}", .{ + tmp_path, dst_path, retry_err, + }); + }, + else => return step.fail(maker, "failed to rename directory {f} to {f}: {t}", .{ + tmp_path, dst_path, err, }), }; } - if (!has_side_effects) try step.writeManifestAndWatch(&man); + if (!has_side_effects) try step.writeManifestAndWatch(maker, &man); - try populateGeneratedPaths( - arena, - output_placeholders.items, - &conf_run, - cache_root, - &digest, - ); + try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); + try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest); } /// Reads stdout of a Zig test process until a termination condition is reached: @@ -918,9 +876,12 @@ const FuzzTestRunner = struct { } fn saveCrash(f: *FuzzTestRunner, id: u32, term: process.Child.Term) !void { + const fuzz = f.context.fuzz; + const maker = fuzz.maker; const step = &f.run.step; - const b = step.owner; - const io = b.graph.io; + const graph = maker.graph; + const io = graph.io; + const cache_root = graph.local_cache_root; if (f.coverage_id == null) return; @@ -938,7 +899,7 @@ const FuzzTestRunner = struct { }) { const name_prefix = "f" ++ Io.Dir.path.sep_str ++ "in"; in_name = std.fmt.bufPrint(&in_name_buf, name_prefix ++ "{x}", .{i}) catch unreachable; - in_f = b.cache_root.handle.openFile(io, in_name, .{ + in_f = cache_root.handle.openFile(io, in_name, .{ .lock = .exclusive, .lock_nonblocking = true, }) catch |e| switch (e) { @@ -946,7 +907,7 @@ const FuzzTestRunner = struct { error.WouldBlock => continue, // Can not be from // the crashed instance since it is still locked. else => return step.fail("failed to open file '{f}{s}': {t}", .{ - b.cache_root, in_name, e, + cache_root, in_name, e, }), }; @@ -955,7 +916,7 @@ const FuzzTestRunner = struct { in_f.close(io); switch (e) { error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ - b.cache_root, in_name, in_r.err.?, + cache_root, in_name, in_r.err.?, }), error.EndOfStream => continue, } @@ -974,10 +935,10 @@ const FuzzTestRunner = struct { // Save it to a seperate file const crash_name = "f" ++ Io.Dir.path.sep_str ++ "crash"; - const out = b.cache_root.handle.createFile(io, crash_name, .{ + const out = cache_root.handle.createFile(io, crash_name, .{ .lock = .exclusive, // Multiple run steps could have found a crash at the same time }) catch |e| return step.fail("failed to create file '{f}{s}': {t}", .{ - b.cache_root, crash_name, e, + cache_root, crash_name, e, }); defer out.close(io); @@ -985,17 +946,17 @@ const FuzzTestRunner = struct { var out_w = out.writerStreaming(io, &out_w_buf); _ = out_w.interface.sendFileAll(&in_r, .limited(header.len)) catch |e| switch (e) { error.ReadFailed => return step.fail("failed to read file '{f}{s}': {t}", .{ - b.cache_root, in_name, in_r.err.?, + cache_root, in_name, in_r.err.?, }), error.WriteFailed => return step.fail("failed to write file '{f}{s}': {t}", .{ - b.cache_root, crash_name, out_w.err.?, + cache_root, crash_name, out_w.err.?, }), }; return f.run.step.fail("test '{s}' {f}; input saved to '{f}{s}'", .{ f.run.fuzz_tests.items[header.test_i], fmtTerm(term), - b.cache_root, + cache_root, crash_name, }); } @@ -1492,54 +1453,85 @@ pub fn rerunInFuzzMode( const maker = fuzz.maker; const graph = maker.graph; const step = &run.step; - const b = step.owner; const io = graph.io; - const arena = b.allocator; - var argv_list: std.ArrayList([]const u8) = .empty; - for (run.argv.items) |arg| { - switch (arg) { - .bytes => |bytes| { - try argv_list.append(arena, bytes); - }, - .lazy_path => |file| { - const file_path = file.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, convertPathArg(run_index, maker, file_path) })); - }, - .decorated_directory => |dd| { - const file_path = dd.lazy_path.getPath3(b, step); - try argv_list.append(arena, b.fmt("{s}{s}{s}", .{ dd.prefix, convertPathArg(run_index, maker, file_path), dd.suffix })); - }, - .file_content => |file_plp| { - const file_path = file_plp.lazy_path.getPath3(b, step); + const arena = graph.arena; // TODO don't leak into the process arena + const gpa = maker.gpa; + const conf = &maker.scanned_config.configuration; + const conf_step = run_index.ptr(conf); + const conf_run = conf_step.extended.get(conf.extra).run; + const argv_list = &run.argv; - var result: std.Io.Writer.Allocating = .init(arena); - errdefer result.deinit(); - result.writer.writeAll(file_plp.prefix) catch return error.OutOfMemory; + argv_list.clearRetainingCapacity(); - const file = try file_path.root_dir.handle.openFile(io, file_path.subPathOrDot(), .{}); - defer file.close(io); - - var buf: [1024]u8 = undefined; - var file_reader = file.reader(io, &buf); - _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, - error.WriteFailed => return error.OutOfMemory, - }; - - try argv_list.append(arena, result.written()); + for (conf_run.args.slice) |arg_index| { + const arg = arg_index.get(conf); + try argv_list.ensureUnusedCapacity(gpa, 1); + switch (arg.flags.tag) { + .string => { + const prefix = arg.prefix.value.?.slice(conf); + argv_list.appendAssumeCapacity(prefix); }, - .artifact => |pa| { - const artifact = pa.artifact; - const file_path: []const u8 = p: { - if (artifact == run.producer.?) break :p b.fmt("{f}", .{run.rebuilt_executable.?}); - break :p artifact.installed_path orelse artifact.generated_bin.?.path.?; - }; - try argv_list.append(arena, b.fmt("{s}{s}", .{ - pa.prefix, - convertPathArg(run_index, maker, .{ .root_dir = .cwd(), .sub_path = file_path }), + .path_file => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); + argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ + prefix, try convertPathArg(run_index, maker, file_path), suffix, })); }, - .output_file, .output_directory => unreachable, + .path_directory => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); + const resolved_arg = try mem.concat(arena, u8, &.{ + prefix, try convertPathArg(run_index, maker, file_path), suffix, + }); + argv_list.appendAssumeCapacity(resolved_arg); + }, + .file_content => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const file_path = try maker.resolveLazyPathIndex(arena, arg.path.value.?, run_index); + + var result: std.Io.Writer.Allocating = .init(arena); + result.writer.writeAll(prefix) catch return error.OutOfMemory; + + const file = file_path.root_dir.handle.openFile(io, file_path.sub_path, .{}) catch |err| + return step.fail(maker, "unable to open input file {f}: {t}", .{ file_path, err }); + defer file.close(io); + + var file_reader = file.reader(io, &.{}); + _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { + error.ReadFailed => switch (file_reader.err.?) { + error.Canceled => |e| return e, + else => |e| return step.fail(maker, "failed to read from {f}: {t}", .{ file_path, e }), + }, + error.WriteFailed => return error.OutOfMemory, + }; + result.writer.writeAll(suffix) catch return error.OutOfMemory; + + argv_list.appendAssumeCapacity(result.written()); + }, + .artifact => { + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const producer_index = arg.producer.value.?; + const producer_step = producer_index.ptr(conf); + const producer = producer_step.extended.get(conf.extra).compile; + const producer_make_comp_step = maker.stepByIndex(producer_index); + const producer_make_comp = &producer_make_comp_step.extended.compile; + const file_path: Path = if (producer_index == conf_run.producer.value.?) + run.rebuilt_executable.? + else + producer_make_comp.installed_path orelse + maker.generatedPath(producer.generated_bin.value.?).*; + argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ + prefix, try convertPathArg(run_index, maker, file_path), suffix, + })); + }, + .output_file => unreachable, + .output_directory => unreachable, + .cli_rest_positionals => unreachable, } } @@ -1552,7 +1544,7 @@ pub fn rerunInFuzzMode( var rand_int: u64 = undefined; io.random(@ptrCast(&rand_int)); const tmp_dir_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); - try runCommand(run, maker, prog_node, argv_list.items, has_side_effects, tmp_dir_path, .{ + try runCommand(run, run_index, maker, prog_node, argv_list.items, has_side_effects, tmp_dir_path, .{ .fuzz = fuzz, }); } @@ -1560,28 +1552,94 @@ pub fn rerunInFuzzMode( const CapturedStdIo = void; // TODO get it from Configuration fn populateGeneratedPaths( - arena: std.mem.Allocator, + maker: *Maker, output_placeholders: []const IndexedOutput, + cache_root: Cache.Directory, + digest: *const Cache.HexDigest, +) !void { + const conf = &maker.scanned_config.configuration; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena + + for (output_placeholders) |placeholder| { + const arg = placeholder.arg_index.get(conf); + maker.generatedPath(arg.generated.value.?).* = .{ + .root_dir = cache_root, + .sub_path = try Dir.path.join(arena, &.{ + "o", digest, arg.basename.value.?.slice(conf), + }), + }; + } +} + +fn populateGeneratedPathsCreateDirs( + run: *Run, + run_index: Configuration.Step.Index, + maker: *Maker, + output_dir_path: []const u8, +) !void { + const step = maker.stepByIndex(run_index); + const conf = &maker.scanned_config.configuration; + const graph = maker.graph; + const io = graph.io; + const arena = graph.arena; // TODO don't leak into the process arena + const cache_root = graph.local_cache_root; + const argv = run.argv.items; + + for (run.output_placeholders.items) |placeholder| { + const arg = placeholder.arg_index.get(conf); + const prefix = if (arg.prefix.value) |p| p.slice(conf) else ""; + const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; + const basename = arg.basename.value.?.slice(conf); + + const generated_path: Path = .{ + .root_dir = cache_root, + .sub_path = try Dir.path.join(arena, &.{ output_dir_path, basename }), + }; + const create_path: Path = .{ + .root_dir = cache_root, + .sub_path = switch (arg.flags.tag) { + .output_file => Dir.path.dirname(generated_path.sub_path).?, + .output_directory => generated_path.sub_path, + else => unreachable, + }, + }; + create_path.root_dir.handle.createDirPath(io, create_path.sub_path) catch |err| + return step.fail(maker, "unable to make path {f}: {t}", .{ create_path, err }); + + maker.generatedPath(arg.generated.value.?).* = generated_path; + + const arg_output_path = try convertPathArg(run_index, maker, generated_path); + argv[placeholder.index] = try mem.concat(arena, u8, &.{ prefix, arg_output_path, suffix }); + } +} + +fn populateGeneratedStdIo( + maker: *Maker, conf_run: *const Configuration.Step.Run, cache_root: Cache.Directory, digest: *const Cache.HexDigest, ) !void { - for (output_placeholders) |placeholder| { - placeholder.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, placeholder.output.basename, - }); - } + const conf = &maker.scanned_config.configuration; + const graph = maker.graph; + const arena = graph.arena; // TODO don't leak into the process arena if (conf_run.captured_stdout.value) |captured| { - captured.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, captured.output.basename, - }); + maker.generatedPath(captured.generated_file).* = .{ + .root_dir = cache_root, + .sub_path = try Dir.path.join(arena, &.{ + "o", digest, captured.basename.slice(conf), + }), + }; } if (conf_run.captured_stderr.value) |captured| { - captured.output.generated_file.path = try cache_root.join(arena, &.{ - "o", digest, captured.output.basename, - }); + maker.generatedPath(captured.generated_file).* = .{ + .root_dir = cache_root, + .sub_path = try Dir.path.join(arena, &.{ + "o", digest, captured.basename.slice(conf), + }), + }; } } @@ -1600,11 +1658,12 @@ fn fmtTerm(term: ?process.Child.Term) std.fmt.Alt(?process.Child.Term, formatTer } const FuzzContext = struct { - fuzz: *std.Build.Fuzz, + fuzz: *Fuzz, }; fn runCommand( run: *Run, + run_index: Configuration.Step.Index, maker: *Maker, progress_node: std.Progress.Node, argv: []const []const u8, @@ -1615,30 +1674,45 @@ fn runCommand( const graph = maker.graph; const arena = graph.arena; // TODO don't leak into process arena const gpa = maker.gpa; - const step = &run.step; - const b = step.owner; + const step = maker.stepByIndex(run_index); const io = graph.io; + const cache_root = graph.local_cache_root; + const conf = &maker.scanned_config.configuration; + const conf_step = run_index.ptr(conf); + const conf_run = conf_step.extended.get(conf.extra).run; + const environ_map = &run.environ_map; - const cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, step) } else .inherit; + const cwd: process.Child.Cwd = if (conf_run.cwd.value) |lazy_cwd| + .{ .path = try maker.resolveLazyPathIndexAbs(arena, lazy_cwd, run_index) } + else + .inherit; - try step.handleChildProcUnsupported(); - try Step.handleVerbose(step.owner, cwd, run.environ_map, argv); - - const allow_skip = switch (run.stdio) { - .check, .zig_test => run.skip_foreign_checks, + const allow_skip = switch (conf_run.flags.stdio) { + .check, .zig_test => conf_run.flags.skip_foreign_checks, else => false, }; - var interp_argv = std.array_list.Managed([]const u8).init(b.allocator); - defer interp_argv.deinit(); + var interp_argv: std.ArrayList([]const u8) = .empty; - var environ_map: EnvMap = env: { - const orig = run.environ_map orelse &graph.environ_map; - break :env try orig.clone(gpa); - }; - defer environ_map.deinit(); + // `environ_map` is initialized with an undefined `allocator` field; lazily + // initialize it here. + environ_map.allocator = gpa; + // In either case we add to this mutatable data structure so that we can + // tweak the environment below. + environ_map.clearRetainingCapacity(); + if (conf_run.environ_map.value) |env_map_index| { + const conf_env_map = env_map_index.get(conf); + for (conf_env_map.keys.slice(conf), conf_env_map.values.slice(conf)) |k, v| { + try environ_map.put(k.slice(conf), v.slice(conf)); + } + } else { + try environ_map.putAll(&graph.environ_map); + } + try graph.handleVerbose(cwd, environ_map, argv); - const opt_generic_result = spawnChildAndCollect(run, maker, progress_node, argv, &environ_map, has_side_effects, fuzz_context) catch |err| term: { + if (true) @panic("TODO"); + + const opt_generic_result = spawnChildAndCollect(run_index, run, maker, progress_node, argv, &environ_map, has_side_effects, fuzz_context) catch |err| term: { // InvalidExe: cpu arch mismatch // FileNotFound: can happen with a wrong dynamic linker path if (err == error.InvalidExe or err == error.FileNotFound) interpret: { @@ -1660,7 +1734,7 @@ fn runCommand( (root_target.isGnuLibC() or (root_target.isMuslLibC() and exe.linkage == .dynamic)); const other_target = exe.root_module.resolved_target.?.result; switch (std.zig.system.getExternalExecutor(io, &graph.host.result, &other_target, .{ - .qemu_fixes_dl = need_cross_libc and b.libc_runtimes_dir != null, + .qemu_fixes_dl = need_cross_libc and graph.libc_runtimes_dir != null, .link_libc = exe.is_linking_libc, })) { .native, .rosetta => { @@ -1668,7 +1742,7 @@ fn runCommand( break :interpret; }, .wine => |bin_name| { - if (b.enable_wine) { + if (graph.enable_wine) { try interp_argv.append(bin_name); try interp_argv.appendSlice(argv); @@ -1682,21 +1756,21 @@ fn runCommand( } }, .qemu => |bin_name| { - if (b.enable_qemu) { + if (graph.enable_qemu) { try interp_argv.append(bin_name); if (need_cross_libc) { - if (b.libc_runtimes_dir) |dir| { + if (graph.libc_runtimes_dir) |dir| { try interp_argv.append("-L"); - try interp_argv.append(b.pathJoin(&.{ + try interp_argv.append(try Dir.path.join(arena, &.{ dir, try if (root_target.isGnuLibC()) std.zig.target.glibcRuntimeTriple( - b.allocator, + arena, root_target.cpu.arch, root_target.os.tag, root_target.abi, ) else if (root_target.isMuslLibC()) std.zig.target.muslRuntimeTriple( - b.allocator, + arena, root_target.cpu.arch, root_target.abi, ) else unreachable, @@ -1708,7 +1782,7 @@ fn runCommand( } else return failForeign(run, "-fqemu", argv[0], exe); }, .darling => |bin_name| { - if (b.enable_darling) { + if (graph.enable_darling) { try interp_argv.append(bin_name); try interp_argv.appendSlice(argv); } else { @@ -1716,7 +1790,7 @@ fn runCommand( } }, .wasmtime => |bin_name| { - if (b.enable_wasmtime) { + if (graph.enable_wasmtime) { try interp_argv.append(bin_name); try interp_argv.append("--dir=."); // Wasmtime doeesn't inherit environment variables from the parent process @@ -1743,8 +1817,8 @@ fn runCommand( .bad_os_or_cpu => { if (allow_skip) return error.MakeSkipped; - const host_name = try graph.host.result.zigTriple(b.allocator); - const foreign_name = try root_target.zigTriple(b.allocator); + const host_name = try graph.host.result.zigTriple(arena); + const foreign_name = try root_target.zigTriple(arena); return step.fail(maker, "the host system ({s}) is unable to execute binaries from the target ({s})", .{ host_name, foreign_name, @@ -1761,7 +1835,7 @@ fn runCommand( step.result_failed_command = null; try Step.handleVerbose(step.owner, cwd, run.environ_map, interp_argv.items); - break :term spawnChildAndCollect(run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| { + break :term spawnChildAndCollect(run_index, run, maker, progress_node, interp_argv.items, &environ_map, has_side_effects, fuzz_context) catch |e| { if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped; if (e == error.MakeFailed) return error.MakeFailed; // error already reported return step.fail(maker, "unable to spawn interpreter {s}: {t}", .{ interp_argv.items[0], e }); @@ -1800,14 +1874,14 @@ fn runCommand( }) |stream| { if (stream.captured) |captured| { const output_components = .{ output_dir_path, captured.output.basename }; - const output_path = try b.cache_root.join(arena, &output_components); + const output_path = try cache_root.join(arena, &output_components); captured.output.generated_file.path = output_path; - const sub_path = b.pathJoin(&output_components); + const sub_path = try Dir.path.join(arena, &output_components); const sub_path_dirname = Dir.path.dirname(sub_path).?; - b.cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { - return step.fail(maker, "unable to make path '{f}{s}': {s}", .{ - b.cache_root, sub_path_dirname, @errorName(err), + cache_root.handle.createDirPath(io, sub_path_dirname) catch |err| { + return step.fail(maker, "unable to make path '{f}{s}': {t}", .{ + cache_root, sub_path_dirname, err, }); }; const data = switch (captured.trim_whitespace) { @@ -1816,9 +1890,9 @@ fn runCommand( .leading => mem.trimStart(u8, stream.bytes.?, &std.ascii.whitespace), .trailing => mem.trimEnd(u8, stream.bytes.?, &std.ascii.whitespace), }; - b.cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { - return step.fail(maker, "unable to write file '{f}{s}': {s}", .{ - b.cache_root, sub_path, @errorName(err), + cache_root.handle.writeFile(io, .{ .sub_path = sub_path, .data = data }) catch |err| { + return step.fail(maker, "unable to write file '{f}{s}': {t}", .{ + cache_root, sub_path, err, }); }; } @@ -1912,6 +1986,7 @@ const EvalGenericResult = struct { }; fn spawnChildAndCollect( + run_index: Configuration.Step.Index, run: *Run, maker: *Maker, progress_node: std.Progress.Node, @@ -1920,25 +1995,34 @@ fn spawnChildAndCollect( has_side_effects: bool, fuzz_context: ?FuzzContext, ) !?EvalGenericResult { - const b = run.step.owner; + const step = run.step; const graph = maker.graph; const gpa = maker.gpa; const io = graph.io; + const arena = graph.arena; // TODO don't leak into process arena + const conf = &maker.scanned_config.configuration; + const conf_step = run_index.ptr(conf); + const conf_run = conf_step.extended.get(conf.extra).run; if (fuzz_context != null) { assert(!has_side_effects); assert(run.stdio == .zig_test); } - const child_cwd: process.Child.Cwd = if (run.cwd) |lazy_cwd| .{ .path = lazy_cwd.getPath2(b, &run.step) } else .inherit; + const child_cwd: process.Child.Cwd = if (conf_run.cwd) |lazy_cwd| + .{ .path = try maker.resolveLazyPathIndexAbs(arena, lazy_cwd, run_index) } + else + .inherit; // If an error occurs, it's caused by this command: - assert(run.step.result_failed_command == null); - run.step.result_failed_command = try Step.allocPrintCmd(gpa, child_cwd, .{ + assert(step.result_failed_command == null); + step.result_failed_command = try Step.allocPrintCmd(gpa, child_cwd, .{ .child = environ_map, .parent = &graph.environ_map, }, argv); + try step.handleChildProcUnsupported(maker); + var spawn_options: process.SpawnOptions = .{ .argv = argv, .cwd = child_cwd, @@ -1973,7 +2057,7 @@ fn spawnChildAndCollect( error.Canceled => |e| return e, else => |e| e, }; - run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); + step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); try result; return null; } else { @@ -1993,7 +2077,7 @@ fn spawnChildAndCollect( error.Canceled => |e| return e, else => |e| e, }; - run.step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); + step.result_duration_ns = @intCast(started.untilNow(io).raw.nanoseconds); return try result; } } diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index 2feac82180..7815198bca 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -455,7 +455,7 @@ const Serialize = struct { .producer = .{ .value = null }, .generated = .{ .value = null }, }, - .output_file => |a| .{ + .output_file, .output_file_dep => |a, tag| .{ .flags = .{ .tag = .output_file, .prefix = a.prefix.len != 0, @@ -464,7 +464,7 @@ const Serialize = struct { .path = false, .producer = false, .generated = true, - .dep_file = false, + .dep_file = tag == .output_file_dep, }, .prefix = .{ .value = if (a.prefix.len != 0) try wc.addString(a.prefix) else null }, .suffix = .{ .value = null }, @@ -1023,7 +1023,6 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { break :e @enumFromInt(extra_index); }, .check_file => @panic("TODO"), - .check_object => @panic("TODO"), .config_header => @panic("TODO"), .objcopy => @panic("TODO"), .options => @panic("TODO"), diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index cd45617de6..4c81bdb9ee 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -427,7 +427,6 @@ pub const Step = extern struct { max_rss: MaxRss, extended: Storage.Extended(Flags, union(Tag) { check_file: CheckFile, - check_object: CheckObject, compile: Compile, config_header: ConfigHeader, fail: Fail, @@ -462,7 +461,6 @@ pub const Step = extern struct { pub const Tag = enum(u5) { check_file, - check_object, compile, config_header, fail, @@ -997,15 +995,6 @@ pub const Step = extern struct { }; }; - pub const CheckObject = struct { - flags: @This().Flags, - - pub const Flags = packed struct(u32) { - tag: Tag = .check_object, - _: u27 = 0, - }; - }; - pub const ConfigHeader = struct { flags: @This().Flags, diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index bfb540a0ee..c91d8334fe 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -85,8 +85,6 @@ stdio_limit: std.Io.Limit, captured_stdout: ?*CapturedStdIo, captured_stderr: ?*CapturedStdIo, -dep_output_file: ?*Output, - has_side_effects: bool, test_runner_mode: bool = false, @@ -141,6 +139,7 @@ pub const Arg = union(enum) { file_content: PrefixedLazyPath, bytes: []const u8, output_file: *Output, + output_file_dep: *Output, output_directory: *Output, /// The arguments passed after "--" on the "zig build" CLI. cli_rest_positionals, @@ -203,7 +202,6 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { .stdio_limit = .unlimited, .captured_stdout = null, .captured_stderr = null, - .dep_output_file = null, .has_side_effects = false, .producer = null, }; @@ -476,12 +474,10 @@ pub fn addDepFileOutputArg(run: *Run, basename: []const u8) std.Build.LazyPath { /// Add a prefixed path argument to a dep file (.d) for the child process to /// write its discovered additional dependencies. -/// Only one dep file argument is allowed by instance. pub fn addPrefixedDepFileOutputArg(run: *Run, prefix: []const u8, basename: []const u8) std.Build.LazyPath { const b = run.step.owner; const graph = b.graph; const arena = graph.arena; - assert(run.dep_output_file == null); const dep_file = arena.create(Output) catch @panic("OOM"); dep_file.* = .{ @@ -490,9 +486,7 @@ pub fn addPrefixedDepFileOutputArg(run: *Run, prefix: []const u8, basename: []co .generated_file = graph.addGeneratedFile(&run.step), }; - run.dep_output_file = dep_file; - - run.argv.append(arena, .{ .output_file = dep_file }) catch @panic("OOM"); + run.argv.append(arena, .{ .output_file_dep = dep_file }) catch @panic("OOM"); return .{ .generated = .{ .index = dep_file.generated_file } }; } @@ -544,7 +538,7 @@ pub fn addPathDir(run: *Run, search_path: []const u8) void { .decorated_directory => false, .file_content => unreachable, // not allowed as first arg .bytes => |bytes| std.mem.endsWith(u8, bytes, ".exe"), - .output_file, .output_directory => false, + .output_file, .output_file_dep, .output_directory => false, }; const key = if (use_wine) "WINEPATH" else "PATH"; const prev_path = environ_map.get(key);