configurer: serialize all data from run steps

This commit is contained in:
Andrew Kelley
2026-03-19 02:24:33 -07:00
parent 8d964600cd
commit b4d6f44704
3 changed files with 354 additions and 53 deletions
+2
View File
@@ -1,3 +1,5 @@
* make more stuff use IndexType
* make addExtra return Index using reflection
* remove Cache from configurer
* implement the build options
* don't forget to add -listen arg back
+226 -18
View File
@@ -351,6 +351,169 @@ const Serialize = struct {
})));
}
fn addEnvironMap(s: *Serialize, opt_map: ?*std.process.Environ.Map) !?Configuration.EnvironMap.Index {
const wc = s.wc;
const map = opt_map orelse return null;
return @enumFromInt(try wc.addDeduped(@as(Configuration.EnvironMap, .{
.keys = try wc.addStringList(map.array_hash_map.keys()),
.values = try wc.addStringList(map.array_hash_map.values()),
})));
}
fn initArgsList(s: *Serialize, args: []const Step.Run.Arg) ![]const Configuration.Step.Run.Arg.Index {
const wc = s.wc;
const result = try s.arena.alloc(Configuration.Step.Run.Arg.Index, args.len);
for (result, args) |*dest, src| {
dest.* = @enumFromInt(try wc.addExtra(@as(Configuration.Step.Run.Arg, switch (src) {
.artifact => |a| .{
.flags = .{
.tag = .artifact,
.prefix = a.prefix.len != 0,
.suffix = false,
.basename = false,
.path = false,
.producer = true,
.generated = false,
.dep_file = false,
},
.prefix = .{ .value = try s.addOptionalString(a.prefix) },
.suffix = .{ .value = null },
.basename = .{ .value = null },
.path = .{ .value = null },
.producer = .{ .value = stepIndex(s, &a.artifact.step) },
.generated = .{ .value = null },
},
.lazy_path => |a| .{
.flags = .{
.tag = .path_file,
.prefix = a.prefix.len != 0,
.suffix = false,
.basename = false,
.path = true,
.producer = false,
.generated = false,
.dep_file = false,
},
.prefix = .{ .value = try s.addOptionalString(a.prefix) },
.suffix = .{ .value = null },
.basename = .{ .value = null },
.path = .{ .value = try addLazyPath(s, a.lazy_path) },
.producer = .{ .value = null },
.generated = .{ .value = null },
},
.decorated_directory => |a| .{
.flags = .{
.tag = .path_directory,
.prefix = a.prefix.len != 0,
.suffix = a.suffix.len != 0,
.basename = false,
.path = true,
.producer = false,
.generated = false,
.dep_file = false,
},
.prefix = .{ .value = try addOptionalString(s, a.prefix) },
.suffix = .{ .value = try addOptionalString(s, a.suffix) },
.basename = .{ .value = null },
.path = .{ .value = try addLazyPath(s, a.lazy_path) },
.producer = .{ .value = null },
.generated = .{ .value = null },
},
.file_content => |a| .{
.flags = .{
.tag = .file_content,
.prefix = a.prefix.len != 0,
.suffix = false,
.basename = false,
.path = true,
.producer = false,
.generated = false,
.dep_file = false,
},
.prefix = .{ .value = try addOptionalString(s, a.prefix) },
.suffix = .{ .value = null },
.basename = .{ .value = null },
.path = .{ .value = try addLazyPath(s, a.lazy_path) },
.producer = .{ .value = null },
.generated = .{ .value = null },
},
.bytes => |a| .{
.flags = .{
.tag = .string,
.prefix = true,
.suffix = false,
.basename = false,
.path = false,
.producer = false,
.generated = false,
.dep_file = false,
},
.prefix = .{ .value = try addOptionalString(s, a) },
.suffix = .{ .value = null },
.basename = .{ .value = null },
.path = .{ .value = null },
.producer = .{ .value = null },
.generated = .{ .value = null },
},
.output_file => |a| .{
.flags = .{
.tag = .output_file,
.prefix = a.prefix.len != 0,
.suffix = false,
.basename = a.basename.len != 0,
.path = false,
.producer = false,
.generated = true,
.dep_file = false,
},
.prefix = .{ .value = try addOptionalString(s, a.prefix) },
.suffix = .{ .value = null },
.basename = .{ .value = try addOptionalString(s, a.basename) },
.path = .{ .value = null },
.producer = .{ .value = null },
.generated = .{ .value = a.generated_file },
},
.output_directory => |a| .{
.flags = .{
.tag = .output_directory,
.prefix = a.prefix.len != 0,
.suffix = false,
.basename = a.basename.len != 0,
.path = false,
.producer = false,
.generated = true,
.dep_file = false,
},
.prefix = .{ .value = try addOptionalString(s, a.prefix) },
.suffix = .{ .value = null },
.basename = .{ .value = try addOptionalString(s, a.basename) },
.path = .{ .value = null },
.producer = .{ .value = null },
.generated = .{ .value = a.generated_file },
},
.cli_rest_positionals => .{
.flags = .{
.tag = .cli_rest_positionals,
.prefix = false,
.suffix = false,
.basename = false,
.path = false,
.producer = false,
.generated = false,
.dep_file = false,
},
.prefix = .{ .value = null },
.suffix = .{ .value = null },
.basename = .{ .value = null },
.path = .{ .value = null },
.producer = .{ .value = null },
.generated = .{ .value = null },
},
})));
}
return result;
}
fn initLazyPathList(s: *Serialize, list: []const std.Build.LazyPath) ![]const Configuration.LazyPath.Index {
const result = try s.arena.alloc(Configuration.LazyPath.Index, list.len);
for (result, list) |*dest, src| dest.* = try addLazyPath(s, src);
@@ -768,16 +931,33 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
.update_source_files => @panic("TODO"),
.run => e: {
const run: *Step.Run = @fieldParentPtr("step", step);
const captured_stdout: Configuration.OptionalString = if (run.captured_stdout) |cs|
.init(try wc.addString(cs.output.basename))
else
.none;
const captured_stderr: Configuration.OptionalString = if (run.captured_stderr) |cs|
.init(try wc.addString(cs.output.basename))
else
.none;
var expect_stderr_exact: ?Configuration.Bytes = null;
var expect_stdout_exact: ?Configuration.Bytes = null;
var expect_stderr_match: std.ArrayList(Configuration.Bytes) = .empty;
var expect_stdout_match: std.ArrayList(Configuration.Bytes) = .empty;
var expect_term: ?struct {
status: Configuration.Step.Run.ExpectTermStatus,
value: u32,
} = null;
switch (run.stdio) {
.check => |checks| for (checks.items) |check| switch (check) {
.expect_stderr_exact => |bytes| expect_stderr_exact = try wc.addBytes(bytes),
.expect_stdout_exact => |bytes| expect_stdout_exact = try wc.addBytes(bytes),
.expect_stderr_match => |bytes| {
try expect_stderr_match.append(arena, try wc.addBytes(bytes));
},
.expect_stdout_match => |bytes| {
try expect_stdout_match.append(arena, try wc.addBytes(bytes));
},
.expect_term => |t| expect_term = switch (t) {
.exited => |x| .{ .status = .exited, .value = x },
.signal => |x| .{ .status = .signal, .value = @intFromEnum(x) },
.stopped => |x| .{ .status = .stopped, .value = x },
.unknown => |x| .{ .status = .unknown, .value = x },
},
},
else => {},
}
const extra_index = try wc.addExtra(@as(Configuration.Step.Run, .{
.flags = .{
@@ -802,16 +982,44 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
.stderr_trim_whitespace = if (run.captured_stderr) |cs| cs.trim_whitespace else .none,
.stdio_limit = run.stdio_limit != .unlimited,
.producer = run.producer != null,
.cwd = run.cwd != null,
.captured_stdout = run.captured_stdout != null,
.captured_stderr = run.captured_stderr != null,
.environ_map = run.environ_map != null,
},
.file_inputs_len = @intCast(run.file_inputs.items.len),
.args_len = @intCast(run.argv.items.len),
.cwd = try s.addOptionalLazyPathEnum(run.cwd),
.captured_stdout = captured_stdout,
.captured_stderr = captured_stderr,
.flags2 = .{
.expect_stderr_exact = expect_stderr_exact != null,
.expect_stdout_exact = expect_stdout_exact != null,
.expect_stderr_match = expect_stderr_match.items.len != 0,
.expect_stdout_match = expect_stdout_match.items.len != 0,
.expect_term = expect_term != null,
.expect_term_status = if (expect_term) |t| t.status else .exited,
},
.file_inputs = .{ .slice = try s.initLazyPathList(run.file_inputs.items) },
.args = .{ .slice = try s.initArgsList(run.argv.items) },
.cwd = .{ .value = try s.addOptionalLazyPath(run.cwd) },
.captured_stdout = .{ .value = if (run.captured_stdout) |cs| .{
.basename = try wc.addString(cs.output.basename),
.generated_file = cs.output.generated_file,
} else null },
.captured_stderr = .{ .value = if (run.captured_stderr) |cs| .{
.basename = try wc.addString(cs.output.basename),
.generated_file = cs.output.generated_file,
} else null },
.environ_map = .{ .value = try s.addEnvironMap(run.environ_map) },
.expect_term_value = .{ .value = if (expect_term) |t| t.value else null },
.stdio_limit = .{ .value = run.stdio_limit.toInt() },
.producer = .{ .value = if (run.producer) |cs| s.stepIndex(&cs.step) else null },
.expect_stderr_exact = .{ .value = if (expect_stderr_exact) |bytes| bytes else null },
.expect_stdout_exact = .{ .value = if (expect_stdout_exact) |bytes| bytes else null },
.expect_stderr_match = .{ .slice = expect_stderr_match.items },
.expect_stdout_match = .{ .slice = expect_stdout_match.items },
.stdin = .{ .u = switch (run.stdin) {
.none => .none,
.bytes => |bytes| .{ .bytes = try wc.addBytes(bytes) },
.lazy_path => |lp| .{ .lazy_path = try s.addLazyPath(lp) },
} },
}));
log.err("TODO serialize the trailing Run step data", .{});
break :e @enumFromInt(extra_index);
},
.check_file => @panic("TODO"),
+126 -35
View File
@@ -188,6 +188,18 @@ pub const Wip = struct {
return .init(try addString(wip, bytes orelse return .none));
}
pub fn addStringList(wip: *Wip, list: []const []const u8) Allocator.Error!StringList {
_ = wip;
_ = list;
@panic("TODO");
}
pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes {
_ = wip;
_ = bytes;
@panic("TODO");
}
pub fn addSemVer(wip: *Wip, sv: std.SemanticVersion) Allocator.Error!String {
var buffer: [256]u8 = undefined;
var writer: std.Io.Writer = .fixed(&buffer);
@@ -500,52 +512,65 @@ pub const Step = extern struct {
};
};
/// Trailing:
/// * LazyPath.Index for each file_inputs_len
/// * Arg for each args_len
/// * environ_map if corresponding flag is set
/// * stdin: Bytes, // if StdIn.bytes is chosen
/// * stdin: LazyPath.Index, // if StdIn.lazy_path is chosen
/// * checks: Checks, // if StdIo.check is chosen
/// * stdio_limit: u64, // if stdio_limit is set
/// * producer: Step.Index, // if producer is set. always compile step
pub const Run = struct {
flags: @This().Flags,
file_inputs_len: u32,
args_len: u32,
cwd: LazyPath.OptionalIndex,
captured_stdout: OptionalString, // basename
captured_stderr: OptionalString, // basename
flags2: Flags2,
args: Storage.LengthPrefixedList(Arg.Index),
cwd: Storage.FlagOptional(.flags, .cwd, LazyPath.Index),
captured_stdout: Storage.FlagOptional(.flags, .captured_stdout, CapturedStream),
captured_stderr: Storage.FlagOptional(.flags, .captured_stderr, CapturedStream),
file_inputs: Storage.LengthPrefixedList(LazyPath.Index),
stdio_limit: Storage.FlagOptional(.flags, .stdio_limit, u64),
/// Always a compile step.
producer: Storage.FlagOptional(.flags, .producer, Step.Index),
/// First half is keys, second half is values.
environ_map: Storage.FlagOptional(.flags, .environ_map, EnvironMap.Index),
stdin: Storage.FlagUnion(.flags, .stdin, StdIn),
expect_stderr_exact: Storage.FlagOptional(.flags2, .expect_stderr_exact, Bytes),
expect_stdout_exact: Storage.FlagOptional(.flags2, .expect_stdout_exact, Bytes),
expect_stderr_match: Storage.FlagLengthPrefixedList(.flags2, .expect_stderr_match, Bytes),
expect_stdout_match: Storage.FlagLengthPrefixedList(.flags2, .expect_stdout_match, Bytes),
expect_term_value: Storage.FlagOptional(.flags2, .expect_term, u32),
pub const CapturedStream = extern struct {
generated_file: GeneratedFileIndex,
basename: String,
};
/// Trailing:
/// * String if prefix set
/// * String if suffix set
/// * String if basename set
/// * Step.Index which is always a compile step if tag is artifact
/// * LazyPath.Index if tag is path_file, path_directory, or file_content
pub const Arg = struct {
flags: Arg.Flags,
flags: @This().Flags,
prefix: Storage.FlagOptional(.flags, .prefix, String),
suffix: Storage.FlagOptional(.flags, .suffix, String),
basename: Storage.FlagOptional(.flags, .basename, String),
path: Storage.FlagOptional(.flags, .path, LazyPath.Index),
/// Always a compile step.
producer: Storage.FlagOptional(.flags, .producer, Step.Index),
generated: Storage.FlagOptional(.flags, .generated, GeneratedFileIndex),
pub const Flags = packed struct(u32) {
tag: Arg.Tag,
prefix: bool,
suffix: bool,
basename: bool,
/// Implies Tag is output_file
path: bool,
producer: bool,
generated: bool,
dep_file: bool,
_: u20 = 0,
_: u22 = 0,
};
pub const Tag = enum(u8) {
pub const Tag = enum(u3) {
artifact,
path_file,
path_directory,
string,
file_content,
bytes,
output_file,
output_directory,
cli_rest_positionals,
};
pub const Index = IndexType(@This());
};
pub const Color = enum(u4) {
@@ -562,26 +587,47 @@ pub const Step = extern struct {
manual,
};
pub const StdIn = enum(u2) { none, bytes, lazy_path };
pub const StdIn = union(@This().Tag) {
none: void,
bytes: Bytes,
lazy_path: LazyPath.Index,
pub const Tag = enum(u2) { none, bytes, lazy_path };
};
pub const TrimWhitespace = enum(u2) { none, all, leading, trailing };
pub const StdIo = enum(u2) { infer_from_args, inherit, check, zig_test };
pub const ExpectTermStatus = enum(u2) { exited, signal, stopped, unknown };
pub const Flags = packed struct(u32) {
tag: Tag = .run,
disable_zig_progress: bool,
skip_foreign_checks: bool,
failing_to_execute_foreign_is_an_error: bool,
has_side_effects: bool,
test_runner_mode: bool,
color: Color,
stdin: StdIn,
stdin: StdIn.Tag,
stdio: StdIo,
stdout_trim_whitespace: TrimWhitespace,
stderr_trim_whitespace: TrimWhitespace,
stdio_limit: bool,
producer: bool,
_: u8 = 0,
cwd: bool,
captured_stdout: bool,
captured_stderr: bool,
environ_map: bool,
_: u4 = 0,
};
pub const Flags2 = packed struct(u32) {
expect_stderr_exact: bool,
expect_stdout_exact: bool,
expect_stderr_match: bool,
expect_stdout_match: bool,
expect_term: bool,
expect_term_status: ExpectTermStatus,
_: u25 = 0,
};
};
@@ -1395,17 +1441,37 @@ pub const Deps = struct {
};
};
pub const EnvironMap = struct {
keys: StringList,
values: StringList,
pub const Index = IndexType(@This());
};
/// Points into `extra`, where the first element is count of strings, following
/// elements is `String` per count.
///
/// Stored identically to `Deps`.
pub const StringList = enum(u32) {
_,
pub fn slice(this: @This(), c: *const Configuration) []const String {
const len = c.extra[@intFromEnum(this)];
return @ptrCast(c.extra[@intFromEnum(this) + 1 ..][0..len]);
}
};
pub const OptionalStringList = enum(u32) {
none = max_u32,
_,
pub fn slice(osl: OptionalStringList, c: *const Configuration) ?[]const String {
const len = c.extra[@intFromEnum(osl)];
return @ptrCast(c.extra[@intFromEnum(osl) + 1 ..][0..len]);
pub fn unwrap(this: @This()) ?StringList {
if (this == .none) return null;
return @enumFromInt(@intFromEnum(this));
}
pub fn slice(this: @This(), c: *const Configuration) ?[]const String {
return (unwrap(this) orelse return null).slice(c);
}
};
@@ -1499,6 +1565,13 @@ pub const String = enum(u32) {
}
};
/// Arbitrary sequence of bytes that may contain null bytes.
pub const Bytes = extern struct {
/// Points into `string_bytes`.
index: u32,
len: u32,
};
pub const DefaultingBool = enum(u2) {
false,
true,
@@ -2359,7 +2432,11 @@ pub const Storage = enum {
},
},
},
.@"extern" => comptime unreachable,
.@"extern" => {
const n = @divExact(@sizeOf(Field), @sizeOf(u32));
defer i.* += n;
return @bitCast(buffer[i.*..][0..n].*);
},
},
else => comptime unreachable,
}
@@ -2404,7 +2481,7 @@ pub const Storage = enum {
inline else => |v| extraFieldLen(v),
},
},
.@"extern" => comptime unreachable,
.@"extern" => @divExact(@sizeOf(Field), @sizeOf(u32)),
},
else => @compileError("bad type: " ++ @typeName(Field)),
};
@@ -2520,13 +2597,27 @@ pub const Storage = enum {
},
},
},
.@"extern" => comptime unreachable,
.@"extern" => {
const n = @divExact(@sizeOf(Field), @sizeOf(u32));
buffer[i..][0..n].* = @bitCast(value);
return n;
},
},
else => @compileError("bad field type: " ++ @typeName(Field)),
}
}
};
fn IndexType(comptime T: type) type {
return enum(u32) {
_,
pub fn get(this: @This(), c: *const Configuration) T {
return extraData(c, T, @intFromEnum(this));
}
};
}
pub fn extraData(c: *const Configuration, comptime T: type, index: usize) T {
var i: usize = index;
return Storage.data(c.extra, &i, T);