diff --git a/lib/compiler/configure_runner.zig b/lib/compiler/configure_runner.zig new file mode 100644 index 0000000000..b1eafdb9ae --- /dev/null +++ b/lib/compiler/configure_runner.zig @@ -0,0 +1,336 @@ +const builtin = @import("builtin"); + +const std = @import("std"); +const Io = std.Io; +const assert = std.debug.assert; +const fmt = std.fmt; +const mem = std.mem; +const process = std.process; +const File = std.Io.File; +const Step = std.Build.Step; +const Watch = std.Build.Watch; +const WebServer = std.Build.WebServer; +const Allocator = std.mem.Allocator; +const fatal = std.process.fatal; +const Writer = std.Io.Writer; +const Color = std.zig.Color; + +pub const root = @import("@build"); +pub const dependencies = @import("@dependencies"); + +pub const std_options: std.Options = .{ + .side_channels_mitigations = .none, + .http_disable_tls = true, +}; + +pub fn main(init: process.Init.Minimal) !void { + // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not + // always the case. So, we do need a true gpa for some things. + var debug_gpa_state: std.heap.DebugAllocator(.{}) = .init; + defer _ = debug_gpa_state.deinit(); + const gpa = debug_gpa_state.allocator(); + + var threaded: std.Io.Threaded = .init(gpa, .{ + .environ = init.environ, + .argv0 = .init(init.args), + }); + defer threaded.deinit(); + const io = threaded.io(); + + // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. + var arena_allocator: std.heap.ArenaAllocator = .init(std.heap.page_allocator); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); + + const args = try init.args.toSlice(arena); + + // skip my own exe name + var arg_idx: usize = 1; + + const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); + const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); + const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); + const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); + const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); + + const cwd: Io.Dir = .cwd(); + + const zig_lib_directory: std.Build.Cache.Directory = .{ + .path = zig_lib_dir, + .handle = try cwd.openDir(io, zig_lib_dir, .{}), + }; + + const build_root_directory: std.Build.Cache.Directory = .{ + .path = build_root, + .handle = try cwd.openDir(io, build_root, .{}), + }; + + const local_cache_directory: std.Build.Cache.Directory = .{ + .path = cache_root, + .handle = try cwd.createDirPathOpen(io, cache_root, .{}), + }; + + const global_cache_directory: std.Build.Cache.Directory = .{ + .path = global_cache_root, + .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), + }; + + var graph: std.Build.Graph = .{ + .io = io, + .arena = arena, + .cache = .{ + .io = io, + .gpa = gpa, + .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), + .cwd = try process.currentPathAlloc(io, arena), + }, + .zig_exe = zig_exe, + .environ_map = try init.environ.createMap(arena), + .global_cache_root = global_cache_directory, + .zig_lib_directory = zig_lib_directory, + .host = .{ + .query = .{}, + .result = try std.zig.system.resolveTargetQuery(io, .{}), + }, + .time_report = false, + }; + + graph.cache.addPrefix(.{ .path = null, .handle = cwd }); + graph.cache.addPrefix(build_root_directory); + graph.cache.addPrefix(local_cache_directory); + graph.cache.addPrefix(global_cache_directory); + graph.cache.hash.addBytes(builtin.zig_version_string); + + const builder = try std.Build.create( + &graph, + build_root_directory, + local_cache_directory, + dependencies.root_deps, + ); + + var error_style: ErrorStyle = .verbose; + var multiline_errors: MultilineErrors = .indent; + var color: Color = .auto; + + if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { + if (std.meta.stringToEnum(ErrorStyle, str)) |style| { + error_style = style; + } + } + + if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { + if (std.meta.stringToEnum(MultilineErrors, str)) |style| { + multiline_errors = style; + } + } + + while (nextArg(args, &arg_idx)) |arg| { + if (mem.cutPrefix(u8, arg, "-D")) |option_contents| { + if (option_contents.len == 0) + fatalWithHint("expected option name after '-D'", .{}); + if (mem.indexOfScalar(u8, option_contents, '=')) |name_end| { + const option_name = option_contents[0..name_end]; + const option_value = option_contents[name_end + 1 ..]; + if (try builder.addUserInputOption(option_name, option_value)) + fatal(" access the help menu with 'zig build -h'", .{}); + } else { + if (try builder.addUserInputFlag(option_contents)) + fatal(" access the help menu with 'zig build -h'", .{}); + } + } else if (mem.eql(u8, arg, "--verbose")) { + builder.verbose = true; + } else if (mem.startsWith(u8, arg, "-fsys=")) { + const name = arg["-fsys=".len..]; + graph.system_library_options.put(arena, name, .user_enabled) catch @panic("OOM"); + } else if (mem.startsWith(u8, arg, "-fno-sys=")) { + const name = arg["-fno-sys=".len..]; + graph.system_library_options.put(arena, name, .user_disabled) catch @panic("OOM"); + } else if (mem.eql(u8, arg, "--release")) { + builder.release_mode = .any; + } else if (mem.startsWith(u8, arg, "--release=")) { + const text = arg["--release=".len..]; + builder.release_mode = std.meta.stringToEnum(std.Build.ReleaseMode, text) orelse { + fatalWithHint("expected [off|any|fast|safe|small] in '{s}', found '{s}'", .{ + arg, text, + }); + }; + } else if (mem.eql(u8, arg, "--search-prefix")) { + const search_prefix = nextArgOrFatal(args, &arg_idx); + builder.addSearchPrefix(search_prefix); + } else if (mem.eql(u8, arg, "--libc")) { + builder.libc_file = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--color")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); + color = std.meta.stringToEnum(Color, next_arg) orelse { + fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ + arg, next_arg, + }); + }; + } else if (mem.eql(u8, arg, "--error-style")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected style after '{s}'", .{arg}); + error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { + fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + }; + } else if (mem.eql(u8, arg, "--multiline-errors")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected style after '{s}'", .{arg}); + multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { + fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + }; + } else if (mem.eql(u8, arg, "--seed")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u32 after '{s}'", .{arg}); + graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { + fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ + next_arg, @errorName(err), + }); + }; + } else if (mem.eql(u8, arg, "--build-id")) { + builder.build_id = .fast; + } else if (mem.startsWith(u8, arg, "--build-id=")) { + const style = arg["--build-id=".len..]; + builder.build_id = std.zig.BuildId.parse(style) catch |err| { + fatal("unable to parse --build-id style '{s}': {s}", .{ + style, @errorName(err), + }); + }; + } else if (mem.eql(u8, arg, "--debug-pkg-config")) { + builder.debug_pkg_config = true; + } else if (mem.eql(u8, arg, "--debug-rt")) { + graph.debug_compiler_runtime_libs = true; + } else if (mem.eql(u8, arg, "--debug-compile-errors")) { + builder.debug_compile_errors = true; + } else if (mem.eql(u8, arg, "--debug-incremental")) { + builder.debug_incremental = true; + } else if (mem.eql(u8, arg, "--system")) { + // The usage text shows another argument after this parameter + // but it is handled by the parent process. The build runner + // only sees this flag. + graph.system_package_mode = true; + } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { + // --glibc-runtimes was the old name of the flag; kept for compatibility for now. + builder.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--verbose-link")) { + builder.verbose_link = true; + } else if (mem.eql(u8, arg, "--verbose-air")) { + builder.verbose_air = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { + builder.verbose_llvm_ir = "-"; + } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { + builder.verbose_llvm_ir = arg["--verbose-llvm-ir=".len..]; + } else if (mem.startsWith(u8, arg, "--verbose-llvm-bc=")) { + builder.verbose_llvm_bc = arg["--verbose-llvm-bc=".len..]; + } else if (mem.eql(u8, arg, "--verbose-cimport")) { + builder.verbose_cimport = true; + } else if (mem.eql(u8, arg, "--verbose-cc")) { + builder.verbose_cc = true; + } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { + builder.verbose_llvm_cpu_features = true; + } else if (mem.eql(u8, arg, "-fincremental")) { + graph.incremental = true; + } else if (mem.eql(u8, arg, "-fno-incremental")) { + graph.incremental = false; + } else if (mem.eql(u8, arg, "-fwine")) { + builder.enable_wine = true; + } else if (mem.eql(u8, arg, "-fno-wine")) { + builder.enable_wine = false; + } else if (mem.eql(u8, arg, "-fqemu")) { + builder.enable_qemu = true; + } else if (mem.eql(u8, arg, "-fno-qemu")) { + builder.enable_qemu = false; + } else if (mem.eql(u8, arg, "-fwasmtime")) { + builder.enable_wasmtime = true; + } else if (mem.eql(u8, arg, "-fno-wasmtime")) { + builder.enable_wasmtime = false; + } else if (mem.eql(u8, arg, "-frosetta")) { + builder.enable_rosetta = true; + } else if (mem.eql(u8, arg, "-fno-rosetta")) { + builder.enable_rosetta = false; + } else if (mem.eql(u8, arg, "-fdarling")) { + builder.enable_darling = true; + } else if (mem.eql(u8, arg, "-fno-darling")) { + builder.enable_darling = false; + } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { + graph.allow_so_scripts = true; + } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { + graph.allow_so_scripts = false; + } else if (mem.eql(u8, arg, "-freference-trace")) { + builder.reference_trace = 256; + } else if (mem.startsWith(u8, arg, "-freference-trace=")) { + const num = arg["-freference-trace=".len..]; + builder.reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { + std.debug.print("unable to parse reference_trace count '{s}': {s}", .{ num, @errorName(err) }); + process.exit(1); + }; + } else if (mem.eql(u8, arg, "-fno-reference-trace")) { + builder.reference_trace = null; + } else if (mem.cutPrefix(u8, arg, "-j")) |text| { + const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| + fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); + if (n < 1) fatal("number of jobs must be at least 1", .{}); + threaded.setAsyncLimit(.limited(n)); + } else if (mem.eql(u8, arg, "--")) { + builder.args = argsRest(args, arg_idx); + break; + } else { + fatalWithHint("unrecognized argument: '{s}'", .{arg}); + } + } + + const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); + const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); + + graph.stderr_mode = switch (color) { + .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), + .on => .escape_codes, + .off => .no_color, + }; + + try builder.runBuild(root); +} + +fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { + if (idx.* >= args.len) return null; + defer idx.* += 1; + return args[idx.*]; +} + +fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { + return nextArg(args, idx) orelse { + std.debug.print("expected argument after '{s}'\n access the help menu with 'zig build -h'\n", .{args[idx.* - 1]}); + process.exit(1); + }; +} + +fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { + if (idx >= args.len) return null; + return args[idx..]; +} + +const ErrorStyle = enum { + verbose, + minimal, + verbose_clear, + minimal_clear, + fn verboseContext(s: ErrorStyle) bool { + return switch (s) { + .verbose, .verbose_clear => true, + .minimal, .minimal_clear => false, + }; + } + fn clearOnUpdate(s: ErrorStyle) bool { + return switch (s) { + .verbose, .minimal => false, + .verbose_clear, .minimal_clear => true, + }; + } +}; +const MultilineErrors = enum { indent, newline, none }; +const Summary = enum { all, new, failures, line, none }; + +fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { + std.debug.print(f ++ "\n access the help menu with 'zig build -h'\n", args); + process.exit(1); +} diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 0cf9dde531..0919219fc3 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1024,6 +1024,12 @@ pub const Manifest = struct { try self.populateFileHash(gop.key_ptr); } + pub fn addPathPost(man: *Manifest, path: Path) !void { + _ = man; + _ = path; + @panic("TODO"); + } + /// Like `addFilePost` but when the file contents have already been loaded from disk. pub fn addFilePostContents( self: *Manifest, diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 5c609495cb..55337b6378 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -11,6 +11,8 @@ const Writer = std.Io.Writer; const tokenizer = @import("zig/tokenizer.zig"); +/// The serialized output of configure phase ingested by make phase. +pub const Configuration = @import("zig/Configuration.zig"); pub const ErrorBundle = @import("zig/ErrorBundle.zig"); pub const Server = @import("zig/Server.zig"); pub const Client = @import("zig/Client.zig"); diff --git a/lib/std/zig/Configuration.zig b/lib/std/zig/Configuration.zig new file mode 100644 index 0000000000..86bf82fa3d --- /dev/null +++ b/lib/std/zig/Configuration.zig @@ -0,0 +1,83 @@ +const Configuration = @This(); + +const std = @import("../std.zig"); +const Io = std.Io; +const Allocator = std.mem.Allocator; + +string_bytes: []u8, +steps: []Step, +path_deps_base: []Path.Base, +path_deps_sub: []String, +unlazy_deps: []String, + +pub const Header = extern struct { + string_bytes_len: u32, + steps_len: u32, + path_deps_len: u32, + unlazy_deps_len: u32, +}; + +pub const Step = extern struct { + name: String, +}; + +pub const Path = extern struct { + base: Base, + sub: String, + + pub const Base = enum(u8) { + cwd, + global_cache, + local_cache, + build_root, + }; + + pub fn toCachePath(path: Path, c: *const Configuration, arena: Allocator) std.Build.Cache.Path { + _ = c; + _ = arena; + _ = path; + @panic("TODO"); + } +}; + +pub const String = enum(u32) { + _, + + pub fn slice(index: String, c: *const Configuration) [:0]const u8 { + const start_slice = c.string_bytes[@intFromEnum(index)..]; + return start_slice[0..std.mem.indexOfScalar(u8, start_slice, 0).? :0]; + } +}; + +pub const LoadError = Io.File.Reader.Error || Allocator.Error || error{EndOfStream}; + +pub fn load(arena: Allocator, io: Io, file: Io.File) LoadError!Configuration { + var buffer: [2000]u8 = undefined; + var fr = file.reader(io, &buffer); + const header = fr.interface.takeStruct(Header, .little) catch |err| switch (err) { + error.ReadFailed => return fr.err.?, + else => |e| return e, + }; + + var result: Configuration = .{ + .string_bytes = try arena.alloc(u8, header.string_bytes_len), + .steps = try arena.alloc(Step, header.steps_len), + .path_deps_sub = try arena.alloc(String, header.path_deps_len), + .path_deps_base = try arena.alloc(Path.Base, header.path_deps_len), + .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len), + }; + + var vecs = [_][]u8{ + result.string_bytes, + @ptrCast(result.steps), + @ptrCast(result.path_deps_base), + @ptrCast(result.path_deps_sub), + @ptrCast(result.unlazy_deps), + }; + fr.interface.readVecAll(&vecs) catch |err| switch (err) { + error.ReadFailed => return fr.err.?, + else => |e| return e, + }; + + return result; +} diff --git a/lib/std/zig/LibCInstallation.zig b/lib/std/zig/LibCInstallation.zig index 1009d34012..1e5e079656 100644 --- a/lib/std/zig/LibCInstallation.zig +++ b/lib/std/zig/LibCInstallation.zig @@ -12,6 +12,7 @@ const Target = std.Target; const fs = std.fs; const Allocator = std.mem.Allocator; const Path = std.Build.Cache.Path; +const Cache = std.Build.Cache; const log = std.log.scoped(.libc_installation); const Environ = std.process.Environ; @@ -969,7 +970,7 @@ pub fn resolveCrtPaths( target: *const std.Target, ) error{ OutOfMemory, LibCInstallationMissingCrtDir }!CrtPaths { const crt_dir_path: Path = .{ - .root_dir = std.Build.Cache.Directory.cwd(), + .root_dir = Cache.Directory.cwd(), .sub_path = lci.crt_dir orelse return error.LibCInstallationMissingCrtDir, }; switch (target.os.tag) { @@ -995,7 +996,7 @@ pub fn resolveCrtPaths( }, .haiku => { const gcc_dir_path: Path = .{ - .root_dir = std.Build.Cache.Directory.cwd(), + .root_dir = Cache.Directory.cwd(), .sub_path = lci.gcc_dir orelse return error.LibCInstallationMissingCrtDir, }; return .{ @@ -1017,3 +1018,16 @@ pub fn resolveCrtPaths( }, } } + +pub fn addToHash(opt_lci: ?*const LibCInstallation, hh: *Cache.HashHelper, abi: std.Target.Abi) void { + const lci = opt_lci orelse return hh.add(false); + hh.add(true); + hh.addOptionalBytes(lci.crt_dir); + switch (abi) { + .msvc, .itanium => { + hh.addOptionalBytes(lci.msvc_lib_dir); + hh.addOptionalBytes(lci.kernel32_lib_dir); + }, + else => {}, + } +} diff --git a/src/Compilation.zig b/src/Compilation.zig index c4e3aec9aa..0e6d5410cc 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -752,13 +752,10 @@ pub const Directories = struct { else => []const u8, }, environ_map: *const std.process.Environ.Map, + cwd: []const u8, ) Directories { const wasi = builtin.target.os.tag == .wasi; - const cwd = introspect.getResolvedCwd(io, arena) catch |err| { - fatal("unable to get cwd: {t}", .{err}); - }; - const zig_lib: Cache.Directory = d: { if (override_zig_lib) |path| break :d openUnresolved(arena, io, cwd, path, .@"zig lib"); if (wasi) break :d getPreopen(preopens, "/lib"); @@ -3527,14 +3524,7 @@ fn addNonIncrementalStuffToCacheManifest( man.hash.addListOfBytes(opts.rpath_list); man.hash.addListOfBytes(opts.symbol_wrap_set.keys()); if (comp.config.link_libc) { - man.hash.add(comp.libc_installation != null); - if (comp.libc_installation) |libc_installation| { - man.hash.addOptionalBytes(libc_installation.crt_dir); - if (target.abi == .msvc or target.abi == .itanium) { - man.hash.addOptionalBytes(libc_installation.msvc_lib_dir); - man.hash.addOptionalBytes(libc_installation.kernel32_lib_dir); - } - } + LibCInstallation.addToHash(comp.libc_installation, &man.hash, target.abi); man.hash.addOptionalBytes(target.dynamic_linker.get()); } man.hash.add(opts.repro); diff --git a/src/main.zig b/src/main.zig index a43eb44334..d8c320d61e 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3177,6 +3177,8 @@ fn buildOutputType( else => process.executablePathAlloc(io, arena) catch |err| fatal("unable to find zig self exe path: {t}", .{err}), }; + const cwd_path = try introspect.getResolvedCwd(io, arena); + // This `init` calls `fatal` on error. var dirs: Compilation.Directories = .init( arena, @@ -3193,6 +3195,7 @@ fn buildOutputType( preopens, self_exe_path, environ_map, + cwd_path, ); defer dirs.deinit(io); @@ -4951,15 +4954,20 @@ test sanitizeExampleName { try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project")); } -fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, environ_map: *process.Environ.Map) !void { - dev.check(.build_command); - +fn cmdBuild( + gpa: Allocator, + arena: Allocator, + io: Io, + args: []const []const u8, + environ_map: *process.Environ.Map, +) !void { var build_file: ?[]const u8 = null; var override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map); var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); - var override_build_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map); - var child_argv: std.ArrayList([]const u8) = .empty; + var override_make_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map); + var configure_argv: std.ArrayList([]const u8) = .empty; + var make_argv: std.ArrayList([]const u8) = .empty; var forks: std.ArrayList(Fork) = .empty; var reference_trace: ?u32 = null; var debug_compile_errors = false; @@ -4979,46 +4987,32 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, var debug_target: ?[]const u8 = null; var debug_libc_paths_file: ?[]const u8 = null; - const argv_index_exe = child_argv.items.len; - _ = try child_argv.addOne(arena); + const argv_index_exe = configure_argv.items.len; + _ = try configure_argv.addOne(arena); const self_exe_path = try process.executablePathAlloc(io, arena); - try child_argv.append(arena, self_exe_path); + try configure_argv.append(arena, self_exe_path); - const argv_index_zig_lib_dir = child_argv.items.len; - _ = try child_argv.addOne(arena); + const argv_index_zig_lib_dir = configure_argv.items.len; + _ = try configure_argv.addOne(arena); - const argv_index_build_file = child_argv.items.len; - _ = try child_argv.addOne(arena); + const argv_index_build_file = configure_argv.items.len; + _ = try configure_argv.addOne(arena); - const argv_index_cache_dir = child_argv.items.len; - _ = try child_argv.addOne(arena); + const argv_index_cache_dir = configure_argv.items.len; + _ = try configure_argv.addOne(arena); - const argv_index_global_cache_dir = child_argv.items.len; - _ = try child_argv.addOne(arena); + const argv_index_global_cache_dir = configure_argv.items.len; + _ = try configure_argv.addOne(arena); - try child_argv.appendSlice(arena, &.{ + try configure_argv.appendSlice(arena, &.{ "--seed", try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}), }); - const argv_index_seed = child_argv.items.len - 1; + const argv_index_seed = configure_argv.items.len - 1; - // This parent process needs a way to obtain results from the configuration - // phase of the child process. In the future, the make phase will be - // executed in a separate process than the configure phase, and we can then - // use stdout from the configuration phase for this purpose. - // - // However, currently, both phases are in the same process, and Run Step - // provides API for making the runned subprocesses inherit stdout and stderr - // which means these streams are not available for passing metadata back - // to the parent. - // - // Until make and configure phases are separated into different processes, - // the strategy is to choose a temporary file name ahead of time, and then - // read this file in the parent to obtain the results, in the case the child - // exits with code 3. - const results_tmp_file_nonce = std.fmt.hex(randInt(io, u64)); - try child_argv.append(arena, "-Z" ++ results_tmp_file_nonce); + const argv_index_configuration_file = make_argv.items.len; + _ = try make_argv.addOne(arena); var color: Color = .auto; var n_jobs: ?u32 = null; @@ -5041,7 +5035,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } else if (mem.eql(u8, arg, "--build-runner")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; - override_build_runner = args[i]; + override_make_runner = args[i]; continue; } else if (mem.eql(u8, arg, "--cache-dir")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); @@ -5080,7 +5074,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; system_pkg_dir_path = args[i]; - try child_argv.append(arena, "--system"); + try configure_argv.append(arena, "--system"); continue; } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| { reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { @@ -5090,7 +5084,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, reference_trace = null; } else if (mem.eql(u8, arg, "--debug-log")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); - try child_argv.appendSlice(arena, args[i .. i + 2]); + try make_argv.appendSlice(arena, args[i .. i + 2]); i += 1; if (!build_options.enable_logging) { warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{}); @@ -5144,7 +5138,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, color = std.meta.stringToEnum(Color, args[i]) orelse { fatal("expected [auto|on|off] after {s}, found '{s}'", .{ arg, args[i] }); }; - try child_argv.appendSlice(arena, &.{ arg, args[i] }); + try configure_argv.appendSlice(arena, &.{ arg, args[i] }); continue; } else if (mem.cutPrefix(u8, arg, "-j")) |str| { const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| { @@ -5159,61 +5153,30 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } else if (mem.eql(u8, arg, "--seed")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; - child_argv.items[argv_index_seed] = args[i]; + configure_argv.items[argv_index_seed] = args[i]; continue; } else if (mem.eql(u8, arg, "--")) { // The rest of the args are supposed to get passed onto // build runner's `build.args` - try child_argv.appendSlice(arena, args[i..]); + try configure_argv.appendSlice(arena, args[i..]); break; } } - try child_argv.append(arena, arg); + try make_argv.append(arena, arg); } } const root_prog_node = std.Progress.start(io, .{ .disable_printing = (color == .off), - .root_name = "Compile Build Script", + .root_name = "", }); defer root_prog_node.end(); - // Normally the build runner is compiled for the host target but here is - // some code to help when debugging edits to the build runner so that you - // can make sure it compiles successfully on other targets. - const resolved_target: Package.Module.ResolvedTarget = t: { - if (build_options.enable_debug_extensions) { - if (debug_target) |triple| { - const target_query = try std.Target.Query.parse(.{ - .arch_os_abi = triple, - }); - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(io, target_query), - .is_native_os = false, - .is_native_abi = false, - .is_explicit_dynamic_linker = false, - }; - } - } - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(io, .{}), - .is_native_os = true, - .is_native_abi = true, - .is_explicit_dynamic_linker = false, - }; - }; - // Likewise, `--debug-libc` allows overriding the libc installation. - const libc_installation: ?*const LibCInstallation = lci: { - const paths_file = debug_libc_paths_file orelse break :lci null; - if (!build_options.enable_debug_extensions) unreachable; - const lci = try arena.create(LibCInstallation); - lci.* = try .parse(arena, io, paths_file, &resolved_target.result); - break :lci lci; - }; - process.raiseFileDescriptorLimit(); - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = introspect.getResolvedCwd(io, arena) catch |err| + fatal("failed to get current directory path: {t}", .{err}); + const build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path, .build_file = build_file, @@ -5232,20 +5195,84 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, .empty, self_exe_path, environ_map, + cwd_path, ); defer dirs.deinit(io); - child_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; - child_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; - child_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; - child_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; - const thread_limit = @min( @max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking), ); try setThreadLimit(arena, thread_limit); + // Kick off an optimized compilation of the make runner. + var make_runner_task = io.async(compileMakeRunner, .{ io, .{ + .dirs = &dirs, + .optimize = .ReleaseSafe, + .parent_prog_node = root_prog_node, + } }); + defer if (make_runner_task.cancel(io)) |mr| mr.deinit(io) else |_| {}; + + // Cache lookup for configure options. If we get a match, we can skip + // execution of the configure script. If not, we get the file path to pass + // to the configure process. + var local_cache: Cache = .{ + .gpa = gpa, + .io = io, + .manifest_dir = try dirs.local_cache.handle.createDirPathOpen(io, "h", .{}), + .cwd = cwd_path, + }; + local_cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); + local_cache.addPrefix(dirs.zig_lib); + local_cache.addPrefix(dirs.local_cache); + local_cache.addPrefix(dirs.global_cache); + defer local_cache.manifest_dir.close(io); + + var config_man = local_cache.obtain(); + defer config_man.deinit(); + config_man.hash.addBytes(build_options.version); + + // Normally the build runner is compiled for the host target but here is + // some code to help when debugging edits to the build runner so that you + // can make sure it compiles successfully on other targets. + const resolved_target: Package.Module.ResolvedTarget = t: { + if (build_options.enable_debug_extensions) { + if (debug_target) |triple| { + const target_query = try std.Target.Query.parse(.{ + .arch_os_abi = triple, + }); + config_man.hash.addBytes(triple); + break :t .{ + .result = std.zig.resolveTargetQueryOrFatal(io, target_query), + .is_native_os = false, + .is_native_abi = false, + .is_explicit_dynamic_linker = false, + }; + } + } + break :t .{ + .result = std.zig.resolveTargetQueryOrFatal(io, .{}), + .is_native_os = true, + .is_native_abi = true, + .is_explicit_dynamic_linker = false, + }; + }; + + // Likewise, `--debug-libc` allows overriding the libc installation. + const libc_installation: ?*const LibCInstallation = lci: { + const paths_file = debug_libc_paths_file orelse break :lci null; + if (!build_options.enable_debug_extensions) unreachable; + const lci = try arena.create(LibCInstallation); + lci.* = try .parse(arena, io, paths_file, &resolved_target.result); + LibCInstallation.addToHash(lci, &config_man.hash, resolved_target.result.abi); + break :lci lci; + }; + + configure_argv.items[argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; + configure_argv.items[argv_index_build_file] = build_root.directory.path orelse cwd_path; + configure_argv.items[argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; + configure_argv.items[argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; + // Dummy http client that is not actually used when fetch_command is unsupported. // Prevents bootstrap from depending on a bunch of unnecessary stuff. var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct { @@ -5282,11 +5309,11 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, // This loop is re-evaluated when the build script exits with an indication that it // could not continue due to missing lazy dependencies. - while (true) { + const configuration_path: Path = cp: while (true) { // We want to release all the locks before executing the child process, so we make a nice // big block here to ensure the cleanup gets run when we extract out our argv. { - const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_build_runner) |runner| .{ + const main_mod_paths: Package.Module.CreateOptions.Paths = if (override_make_runner) |runner| .{ .root = try .fromUnresolved(arena, dirs, &.{fs.path.dirname(runner) orelse "."}), .root_src_path = fs.path.basename(runner), } else .{ @@ -5515,6 +5542,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, config, ); + const compile_prog_node = root_prog_node.start("Compile Configure Script", 0); + defer compile_prog_node.end(); + try root_mod.deps.put(arena, "@build", build_mod); var create_diag: Compilation.CreateDiagnostic = undefined; @@ -5546,7 +5576,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, }; defer comp.destroy(); - updateModule(comp, color, root_prog_node) catch |err| switch (err) { + updateModule(comp, color, compile_prog_node) catch |err| switch (err) { error.CompileErrorsReported => process.exit(2), else => |e| return e, }; @@ -5554,52 +5584,74 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, // Since incremental compilation isn't done yet, we use cache_mode = whole // above, and thus the output file is already closed. //try comp.makeBinFileExecutable(); - child_argv.items[argv_index_exe] = try dirs.local_cache.join(arena, &.{ - "o", - &Cache.binToHex(comp.digest.?), - comp.emit_bin.?, - }); + const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?); + const exe_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }), + }; + _ = try config_man.addFilePath(exe_path, null); + configure_argv.items[argv_index_exe] = try exe_path.toString(arena); + + if (try config_man.hit()) { + const digest = config_man.final(); + break :cp .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), + }; + } } if (!process.can_spawn) { - const cmd = try std.mem.join(arena, " ", child_argv.items); + const cmd = try std.mem.join(arena, " ", configure_argv.items); fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); } + + const rand_int = randInt(io, u64); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); + const config_tmp_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = tmp_dir_sub_path, + }; + const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( + io, + config_tmp_path.sub_path, + .{ .read = true, .exclusive = true }, + ); + defer config_tmp_file.close(io); + switch (term: { - _ = try io.lockStderr(&.{}, .no_color); - defer io.unlockStderr(); + const child_node = root_prog_node.start("Run Configure Script", 0); + defer child_node.end(); var child = std.process.spawn(io, .{ - .argv = child_argv.items, - }) catch |err| fatal("failed to spawn build runner {s}: {t}", .{ child_argv.items[0], err }); + .argv = configure_argv.items, + .stdout = .{ .file = config_tmp_file }, + .progress_node = child_node, + }) catch |err| fatal("failed to spawn configure script {s}: {t}", .{ configure_argv.items[0], err }); defer child.kill(io); break :term child.wait(io) catch |err| - fatal("failed to wait build runner {s}: {t}", .{ child_argv.items[0], err }); + fatal("failed to wait configure script {s}: {t}", .{ configure_argv.items[0], err }); }) { .exited => |code| { - if (code == 0) return cleanExit(io); - // Indicates that the build runner has reported compile errors - // and this parent process does not need to report any further - // diagnostics. - if (code == 2) process.exit(2); + if (code != 0) { + // Failure to produce the configuration file. + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following configure command failed with exit code {d}:\n{s}", .{ code, cmd }); + } + // Even though the file is designed to be sent directly to make + // runner, we must load it now because: + // * If it contains additional file dependencies, we need to + // add them to `config_man` before obtaining the final digest. + // * If it contains a set of lazy packages that need to be + // fetched, we need to fetch those now and re-run configure. + var configuration = std.zig.Configuration.load(arena, io, config_tmp_file) catch |err| + fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); - if (code == 3) { - if (!dev.env.supports(.fetch_command)) process.exit(3); - // Indicates the configure phase failed due to missing lazy - // dependencies and stdout contains the hashes of the ones - // that are missing. - const s = fs.path.sep_str; - const tmp_sub_path = "tmp" ++ s ++ results_tmp_file_nonce; - const stdout = dirs.local_cache.handle.readFileAlloc(io, tmp_sub_path, arena, .limited(50 * 1024 * 1024)) catch |err| { - fatal("unable to read results of configure phase from '{f}{s}': {t}", .{ - dirs.local_cache, tmp_sub_path, err, - }); - }; - dirs.local_cache.handle.deleteFile(io, tmp_sub_path) catch {}; - - var it = mem.splitScalar(u8, stdout, '\n'); + if (configuration.unlazy_deps.len != 0) { + if (!dev.env.supports(.fetch_command)) process.exit(1); var any_errors = false; - while (it.next()) |hash| { - if (hash.len == 0) continue; + for (configuration.unlazy_deps) |hash_string| { + const hash = hash_string.slice(&configuration); + assert(hash.len != 0); if (hash.len > Package.Hash.max_len) { std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ hash.len, hash, @@ -5609,10 +5661,11 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } try unlazy_set.put(arena, .fromSlice(hash), {}); } - if (any_errors) process.exit(3); + if (any_errors) process.exit(1); if (system_pkg_dir_path) |p| { // In this mode, the system needs to provide these packages; they // cannot be fetched by Zig. + const s = fs.path.sep_str; for (unlazy_set.keys()) |*hash| { std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice(), @@ -5620,28 +5673,115 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } std.log.info("remote package fetching disabled due to --system mode", .{}); std.log.info("dependencies might be avoidable depending on build configuration", .{}); - process.exit(3); + process.exit(1); } - continue; + continue :cp; } - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd }); + for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { + const conf_path: std.zig.Configuration.Path = .{ .base = base, .sub = sub }; + try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); + } + + const digest = config_man.final(); + const final_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try std.fmt.allocPrint(arena, "o/{s}", .{&digest}), + }; + Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + ) catch |err| { + fatal("failed to rename configuration file from {f} into {f}: {t}", .{ + config_tmp_path, final_path, err, + }); + }; + config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); + + break :cp final_path; }, .signal => |sig| { - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following build command terminated with signal {t}:\n{s}", .{ sig, cmd }); + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following configure command terminated with signal {t}:\n{s}", .{ sig, cmd }); }, .stopped => |sig| { - const cmd = try std.mem.join(arena, " ", child_argv.items); + const cmd = try std.mem.join(arena, " ", configure_argv.items); fatal("the following build command stopped with signal {t}:\n{s}", .{ sig, cmd }); }, .unknown => { - const cmd = try std.mem.join(arena, " ", child_argv.items); + const cmd = try std.mem.join(arena, " ", configure_argv.items); fatal("the following build command crashed:\n{s}", .{cmd}); }, } + }; + + { + // Release all file system locks just before running the maker process. + var configuration_lock = config_man.toOwnedLock(); + defer configuration_lock.release(io); + + const make_runner = make_runner_task.await(io) catch |err| + fatal("failed to compile maker: {t}", .{err}); + defer make_runner.deinit(io); + + make_argv.items[0] = try make_runner.exe_path.toString(arena); + make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); } + + if (!process.can_spawn) { + const cmd = try std.mem.join(arena, " ", make_argv.items); + fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); + } + + switch (term: { + _ = try io.lockStderr(&.{}, .no_color); + defer io.unlockStderr(); + var child = std.process.spawn(io, .{ + .argv = make_argv.items, + }) catch |err| fatal("failed to spawn maker {s}: {t}", .{ make_argv.items[0], err }); + defer child.kill(io); + break :term child.wait(io) catch |err| + fatal("failed to wait maker {s}: {t}", .{ make_argv.items[0], err }); + }) { + .exited => |code| { + if (code == 0) return cleanExit(io); + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following maker command failed with exit code {d}:\n{s}", .{ code, cmd }); + }, + .signal => |sig| { + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following maker command terminated with signal {t}:\n{s}", .{ sig, cmd }); + }, + else => { + const cmd = try std.mem.join(arena, " ", configure_argv.items); + fatal("the following maker command crashed:\n{s}", .{cmd}); + }, + } +} + +const MakeRunner = struct { + exe_path: Path, + + const Options = struct { + dirs: *Compilation.Directories, + optimize: std.builtin.OptimizeMode, + parent_prog_node: std.Progress.Node, + }; + + fn deinit(mr: MakeRunner, io: Io) void { + _ = mr; + _ = io; + @panic("TODO"); + } +}; + +fn compileMakeRunner(io: Io, options: MakeRunner.Options) !MakeRunner { + _ = io; + _ = options; + @panic("TODO"); } const Fork = struct { @@ -5767,6 +5907,8 @@ fn jitCmdInner( const override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map); const override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); + const cwd_path = try introspect.getResolvedCwd(io, arena); + // This `init` calls `fatal` on error. var dirs: Compilation.Directories = .init( arena, @@ -5777,6 +5919,7 @@ fn jitCmdInner( preopens, self_exe_path, environ_map, + cwd_path, ); defer dirs.deinit(io); diff --git a/src/print_env.zig b/src/print_env.zig index 163648321e..ec7eadd709 100644 --- a/src/print_env.zig +++ b/src/print_env.zig @@ -8,6 +8,7 @@ const fatal = std.process.fatal; const build_options = @import("build_options"); const Compilation = @import("Compilation.zig"); +const introspect = @import("introspect.zig"); pub fn cmdEnv( arena: Allocator, @@ -28,6 +29,8 @@ pub fn cmdEnv( }, }; + const cwd_path = try introspect.getResolvedCwd(io, arena); + var dirs: Compilation.Directories = .init( arena, io, @@ -37,6 +40,7 @@ pub fn cmdEnv( preopens, if (builtin.target.os.tag != .wasi) self_exe_path, environ_map, + cwd_path, ); defer dirs.deinit(io);