std.Build: port UpdateSourceFiles step to new system

This commit is contained in:
Andrew Kelley
2026-05-06 23:17:34 -07:00
parent d3ec255a1f
commit affe5ed867
11 changed files with 293 additions and 209 deletions
+2
View File
@@ -28,6 +28,8 @@
* no more "artifact arg" to run step. if you want to run the post-install binary, get the lazy path
from the install step.
* fmt step: import zig fmt code directly rather than child proc
* UpdateSourceFiles: introduce Group
* WriteFiles: introduce Group
## Already Filed Followup Issues
* build system fmt step with check=false does not acquire a write lock on source files #35204
+4 -3
View File
@@ -22,6 +22,7 @@ pub const Compile = @import("Step/Compile.zig");
pub const Run = @import("Step/Run.zig");
pub const InstallArtifact = @import("Step/InstallArtifact.zig");
pub const InstallFile = @import("Step/InstallFile.zig");
pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig");
/// Avoid false sharing.
_: void align(std.atomic.cache_line) = {},
@@ -75,13 +76,13 @@ pub const Extended = union(enum) {
install_artifact: InstallArtifact,
install_dir: Todo,
install_file: InstallFile,
objcopy: Todo,
obj_copy: Todo,
options: Todo,
remove_dir: Todo,
run: Run,
top_level: TopLevel,
translate_c: Todo,
update_source_files: Todo,
update_source_files: UpdateSourceFiles,
write_file: Todo,
pub fn init(tag: Configuration.Step.Tag) Extended {
@@ -95,7 +96,7 @@ pub const Extended = union(enum) {
.install_artifact => .{ .install_artifact = .{} },
.install_dir => .{ .install_dir = .{} },
.install_file => .{ .install_file = .{} },
.objcopy => .{ .objcopy = .{} },
.obj_copy => .{ .obj_copy = .{} },
.options => .{ .options = .{} },
.remove_dir => .{ .remove_dir = .{} },
.run => .{ .run = .{} },
+142
View File
@@ -0,0 +1,142 @@
const ObjCopy = @This();
const std = @import("std");
const Io = std.Io;
const allocPrint = std.fmt.allocPrint;
const Configuration = std.Build.Configuration;
const Step = @import("../Step.zig");
const Maker = @import("../../Maker.zig");
pub fn make(
obj_copy: *ObjCopy,
step_index: Configuration.Step.Index,
maker: *Maker,
progress_node: std.Progress.Node,
) Step.ExtendedMakeError!void {
_ = obj_copy;
const graph = maker.graph;
const arena = maker.graph.arena; // TODO don't leak into process arena
const io = graph.io;
const step = maker.stepByIndex(step_index);
const conf = &maker.scanned_config.configuration;
const conf_step = step_index.ptr(conf);
const conf_oc = conf_step.extended.get(conf.extra).obj_copy;
const cache_root = graph.local_cache_root;
try step.singleUnchangingWatchInput(maker, arena, conf_oc.input_file);
var man = graph.cache.obtain();
defer man.deinit();
const src_path = try maker.resolveLazyPathIndex(arena, conf_oc.input_file, step_index);
_ = try man.addFilePath(src_path, null);
man.hash.addOptionalBytes(conf_oc.only_section);
man.hash.addOptional(conf_oc.pad_to);
man.hash.addOptional(conf_oc.format);
man.hash.add(conf_oc.compress_debug);
man.hash.add(conf_oc.strip);
man.hash.add(conf_oc.output_file_debug != null);
if (try step.cacheHit(&man)) {
// Cache hit, skip subprocess execution.
const digest = man.final();
conf_oc.output_file.path = try cache_root.join(arena, &.{
"o", &digest, conf_oc.basename,
});
if (conf_oc.output_file_debug) |*file| {
file.path = try cache_root.join(arena, &.{
"o", &digest, try allocPrint(arena, "{s}.debug", .{conf_oc.basename}),
});
}
return;
}
const digest = man.final();
const cache_path = "o" ++ Io.Dir.path.sep_str ++ digest;
const full_dest_path = try cache_root.join(arena, &.{ cache_path, conf_oc.basename });
const full_dest_path_debug = try cache_root.join(arena, &.{
cache_path, try allocPrint(arena, "{s}.debug", .{conf_oc.basename}),
});
cache_root.handle.createDirPath(io, cache_path) catch |err|
return step.fail("unable to make path {s}: {t}", .{ cache_path, err });
var argv: std.ArrayList([]const u8) = .empty;
try argv.ensureUnusedCapacity(arena, 11);
argv.addManyAsArrayAssumeCapacity(2).* = .{ graph.zig_exe, "objcopy" };
if (conf_oc.only_section) |only_section|
argv.addManyAsArrayAssumeCapacity(2).* = .{ "-j", only_section };
switch (conf_oc.strip) {
.none => {},
.debug => argv.appendAssumeCapacity("--strip-debug"),
.debug_and_symbols => argv.appendAssumeCapacity("--strip-all"),
}
if (conf_oc.pad_to) |pad_to| {
argv.addManyAsArrayAssumeCapacity(2).* = .{
"--pad-to", try allocPrint(arena, "{d}", .{pad_to}),
};
}
if (conf_oc.format) |format| {
argv.addManyAsArrayAssumeCapacity(2).* = .{
"-O",
switch (format) {
.bin => "binary",
.hex => "hex",
.elf => "elf",
},
};
}
if (conf_oc.compress_debug)
argv.appendAssumeCapacity("--compress-debug-sections");
if (conf_oc.output_file_debug != null)
argv.appendAssumeCapacity(try allocPrint(arena, "--extract-to={s}", .{full_dest_path_debug}));
try argv.ensureUnusedCapacity(arena, 9);
if (conf_oc.add_section) |section| {
argv.appendAssumeCapacity("--add-section");
argv.appendAssumeCapacity(try allocPrint(arena, "{s}={f}", .{
section.section_name, try maker.resolveLazyPathIndex(arena, section.file_path, step_index),
}));
}
if (conf_oc.set_section_alignment) |set_align| {
argv.appendAssumeCapacity("--set-section-alignment");
argv.appendAssumeCapacity(try allocPrint(arena, "{s}={d}", .{ set_align.section_name, set_align.alignment }));
}
if (conf_oc.set_section_flags) |set_flags| {
const f = set_flags.flags;
// trailing comma is allowed
argv.appendAssumeCapacity("--set-section-flags");
argv.appendAssumeCapacity(try allocPrint(arena, "{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{
set_flags.section_name,
if (f.alloc) "alloc," else "",
if (f.contents) "contents," else "",
if (f.load) "load," else "",
if (f.readonly) "readonly," else "",
if (f.code) "code," else "",
if (f.exclude) "exclude," else "",
if (f.large) "large," else "",
if (f.merge) "merge," else "",
if (f.strings) "strings," else "",
}));
}
argv.appendAssumeCapacity(src_path);
argv.appendAssumeCapacity(full_dest_path);
argv.appendAssumeCapacity("--listen=-");
_ = try Step.evalZigProcess(step_index, maker, argv.items, progress_node, false);
conf_oc.output_file.path = full_dest_path;
if (conf_oc.output_file_debug) |*file| file.path = full_dest_path_debug;
try man.writeManifest();
}
@@ -0,0 +1,87 @@
const UpdateSourceFiles = @This();
const std = @import("std");
const Io = std.Io;
const Path = std.Build.Cache.Path;
const allocPrint = std.fmt.allocPrint;
const Configuration = std.Build.Configuration;
const Step = @import("../Step.zig");
const Maker = @import("../../Maker.zig");
pub fn make(
usf: *UpdateSourceFiles,
step_index: Configuration.Step.Index,
maker: *Maker,
progress_node: std.Progress.Node,
) Step.ExtendedMakeError!void {
_ = usf;
const graph = maker.graph;
const arena = maker.graph.arena; // TODO don't leak into process arena
const io = graph.io;
const step = maker.stepByIndex(step_index);
const conf = &maker.scanned_config.configuration;
const conf_step = step_index.ptr(conf);
const conf_usf = conf_step.extended.get(conf.extra).update_source_files;
const build_root = graph.build_root_directory;
if (conf_step.owner != .root)
return step.fail(maker, "non-root package attempted to update its source files", .{});
var any_miss = false;
progress_node.setEstimatedTotalItems(conf_usf.embeds.slice.len + conf_usf.copies.slice.len);
for (conf_usf.embeds.slice) |*embed| {
const dest_path: Path = .{
.root_dir = build_root,
.sub_path = embed.dest_path.slice(conf),
};
if (Io.Dir.path.dirname(dest_path.sub_path)) |dirname| {
const dirname_path: Path = .{
.root_dir = build_root,
.sub_path = dirname,
};
dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err|
return step.fail(maker, "failed to create path {f}: {t}", .{ dirname_path, err });
}
dest_path.root_dir.handle.writeFile(io, .{
.sub_path = dest_path.sub_path,
.data = embed.bytes.slice(conf),
}) catch |err| return step.fail(maker, "failed to write file {f}: {t}", .{ dest_path, err });
any_miss = true;
progress_node.completeOne();
}
for (conf_usf.copies.slice) |*copy| {
const dest_path: Path = .{
.root_dir = build_root,
.sub_path = copy.dest_path.slice(conf),
};
if (Io.Dir.path.dirname(dest_path.sub_path)) |dirname| {
const dirname_path: Path = .{
.root_dir = build_root,
.sub_path = dirname,
};
dirname_path.root_dir.handle.createDirPath(io, dirname_path.sub_path) catch |err|
return step.fail(maker, "failed to create path {f}: {t}", .{ dirname_path, err });
}
const src_lazy_path = copy.src_path.get(conf);
const source_path = try maker.resolveLazyPath(arena, src_lazy_path, step_index);
if (!step.inputs.populated()) try step.addWatchInput(maker, arena, src_lazy_path);
const prev_status = source_path.root_dir.handle.updateFile(
io,
source_path.sub_path,
dest_path.root_dir.handle,
dest_path.sub_path,
.{},
) catch |err| return step.fail(maker, "unable to update file from {f} to {f}: {t}", .{
source_path, dest_path, err,
});
any_miss = any_miss or prev_status == .stale;
progress_node.completeOne();
}
step.result_cached = !any_miss;
}
+1 -1
View File
@@ -967,7 +967,7 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
},
.check_file => @panic("TODO"),
.config_header => @panic("TODO"),
.objcopy => @panic("TODO"),
.obj_copy => @panic("TODO"),
.options => @panic("TODO"),
},
});
+21 -5
View File
@@ -456,7 +456,7 @@ pub const Step = extern struct {
install_artifact: InstallArtifact,
install_dir: InstallDir,
install_file: InstallFile,
objcopy: Objcopy,
obj_copy: ObjCopy,
options: Options,
remove_dir: RemoveDir,
run: Run,
@@ -491,7 +491,7 @@ pub const Step = extern struct {
install_artifact,
install_dir,
install_file,
objcopy,
obj_copy,
options,
remove_dir,
run,
@@ -1100,11 +1100,11 @@ pub const Step = extern struct {
};
};
pub const Objcopy = struct {
pub const ObjCopy = struct {
flags: @This().Flags,
pub const Flags = packed struct(u32) {
tag: Tag = .objcopy,
tag: Tag = .obj_copy,
_: u27 = 0,
};
};
@@ -1147,10 +1147,26 @@ pub const Step = extern struct {
pub const UpdateSourceFiles = struct {
flags: @This().Flags,
embeds: Storage.FlagLengthPrefixedList(.flags, .embeds, Embed),
copies: Storage.FlagLengthPrefixedList(.flags, .copies, Copy),
pub const Embed = extern struct {
/// Relative to build root.
dest_path: String,
bytes: Bytes,
};
pub const Copy = extern struct {
/// Relative to build root.
dest_path: String,
src_path: LazyPath.Index,
};
pub const Flags = packed struct(u32) {
tag: Tag = .update_source_files,
_: u27 = 0,
embeds: bool,
copies: bool,
_: u25 = 0,
};
};
+1 -1
View File
@@ -72,7 +72,7 @@ pub fn Type(comptime tag: Tag) type {
.run => Run,
.check_file => CheckFile,
.config_header => ConfigHeader,
.objcopy => ObjCopy,
.obj_copy => ObjCopy,
.options => Options,
};
}
+31 -148
View File
@@ -1,17 +1,26 @@
const std = @import("std");
const ObjCopy = @This();
const Allocator = std.mem.Allocator;
const ArenaAllocator = std.heap.ArenaAllocator;
const File = std.Io.File;
const InstallDir = std.Build.InstallDir;
const std = @import("std");
const Step = std.Build.Step;
const elf = std.elf;
const fs = std.fs;
const sort = std.sort;
const Configuration = std.Build.Configuration;
pub const base_tag: Step.Tag = .objcopy;
step: Step,
input_file: std.Build.LazyPath,
basename: []const u8,
output_file: Configuration.GeneratedFileIndex,
output_file_debug: Configuration.OptionalGeneratedFileIndex,
format: ?RawFormat,
only_section: ?[]const u8,
pad_to: ?u64,
strip: Strip,
compress_debug: bool,
add_section: ?AddSection,
set_section_alignment: ?SetSectionAlignment,
set_section_flags: ?SetSectionFlags,
pub const base_tag: Step.Tag = .obj_copy;
pub const RawFormat = enum {
bin,
@@ -28,28 +37,20 @@ pub const Strip = enum {
pub const SectionFlags = packed struct {
/// add SHF_ALLOC
alloc: bool = false,
/// if section is SHT_NOBITS, set SHT_PROGBITS, otherwise do nothing
contents: bool = false,
/// if section is SHT_NOBITS, set SHT_PROGBITS, otherwise do nothing (same as contents)
load: bool = false,
/// readonly: clear default SHF_WRITE flag
readonly: bool = false,
/// add SHF_EXECINSTR
code: bool = false,
/// add SHF_EXCLUDE
exclude: bool = false,
/// add SHF_X86_64_LARGE. Fatal error if target is not x86_64
large: bool = false,
/// add SHF_MERGE
merge: bool = false,
/// add SHF_STRINGS
strings: bool = false,
};
@@ -69,22 +70,6 @@ pub const SetSectionFlags = struct {
flags: SectionFlags,
};
step: Step,
input_file: std.Build.LazyPath,
basename: []const u8,
output_file: Configuration.GeneratedFileIndex,
output_file_debug: Configuration.OptionalGeneratedFileIndex,
format: ?RawFormat,
only_section: ?[]const u8,
pad_to: ?u64,
strip: Strip,
compress_debug: bool,
add_section: ?AddSection,
set_section_alignment: ?SetSectionAlignment,
set_section_flags: ?SetSectionFlags,
pub const Options = struct {
basename: ?[]const u8 = null,
format: ?RawFormat = null,
@@ -111,20 +96,19 @@ pub fn create(
) *ObjCopy {
const graph = owner.graph;
const arena = graph.arena;
const objcopy = arena.create(ObjCopy) catch @panic("OOM");
objcopy.* = ObjCopy{
.step = Step.init(.{
const obj_copy = graph.create(ObjCopy);
obj_copy.* = .{
.step = .init(.{
.tag = base_tag,
.name = owner.fmt("objcopy {f}", .{input_file.fmt(graph)}),
.owner = owner,
.makeFn = make,
}),
.input_file = input_file,
.basename = options.basename orelse std.fmt.allocPrint("{f}", .{input_file.fmt(graph)}) catch @panic("OOM"),
.output_file = graph.addGeneratedFile(&objcopy.step),
.basename = options.basename orelse
std.fmt.allocPrint(arena, "{f}", .{input_file.fmt(graph)}) catch @panic("OOM"),
.output_file = graph.addGeneratedFile(&obj_copy.step),
.output_file_debug = if (options.strip != .none and options.extract_to_separate_file)
.init(graph.addGeneratedFile(&objcopy.step))
.init(graph.addGeneratedFile(&obj_copy.step))
else
.none,
.format = options.format,
@@ -136,115 +120,14 @@ pub fn create(
.set_section_alignment = options.set_section_alignment,
.set_section_flags = options.set_section_flags,
};
input_file.addStepDependencies(&objcopy.step);
return objcopy;
input_file.addStepDependencies(&obj_copy.step);
return obj_copy;
}
pub fn getOutput(objcopy: *const ObjCopy) std.Build.LazyPath {
return .{ .generated = .{ .index = objcopy.output_file } };
}
pub fn getOutputSeparatedDebug(objcopy: *const ObjCopy) ?std.Build.LazyPath {
return if (objcopy.output_file_debug.unwrap()) |index| .{ .generated = .{ .index = index } } else null;
pub fn getOutput(obj_copy: *const ObjCopy) std.Build.LazyPath {
return .{ .generated = .{ .index = obj_copy.output_file } };
}
fn make(step: *Step, options: Step.MakeOptions) !void {
const prog_node = options.progress_node;
const b = step.owner;
const io = b.graph.io;
const objcopy: *ObjCopy = @fieldParentPtr("step", step);
try step.singleUnchangingWatchInput(objcopy.input_file);
var man = b.graph.cache.obtain();
defer man.deinit();
const full_src_path = objcopy.input_file.getPath2(b, step);
_ = try man.addFile(full_src_path, null);
man.hash.addOptionalBytes(objcopy.only_section);
man.hash.addOptional(objcopy.pad_to);
man.hash.addOptional(objcopy.format);
man.hash.add(objcopy.compress_debug);
man.hash.add(objcopy.strip);
man.hash.add(objcopy.output_file_debug != null);
if (try step.cacheHit(&man)) {
// Cache hit, skip subprocess execution.
const digest = man.final();
objcopy.output_file.path = try b.cache_root.join(b.allocator, &.{
"o", &digest, objcopy.basename,
});
if (objcopy.output_file_debug) |*file| {
file.path = try b.cache_root.join(b.allocator, &.{
"o", &digest, b.fmt("{s}.debug", .{objcopy.basename}),
});
}
return;
}
const digest = man.final();
const cache_path = "o" ++ fs.path.sep_str ++ digest;
const full_dest_path = try b.cache_root.join(b.allocator, &.{ cache_path, objcopy.basename });
const full_dest_path_debug = try b.cache_root.join(b.allocator, &.{ cache_path, b.fmt("{s}.debug", .{objcopy.basename}) });
b.cache_root.handle.createDirPath(io, cache_path) catch |err| {
return step.fail("unable to make path {s}: {s}", .{ cache_path, @errorName(err) });
};
var argv = std.array_list.Managed([]const u8).init(b.allocator);
try argv.appendSlice(&.{ b.graph.zig_exe, "objcopy" });
if (objcopy.only_section) |only_section| {
try argv.appendSlice(&.{ "-j", only_section });
}
switch (objcopy.strip) {
.none => {},
.debug => try argv.appendSlice(&.{"--strip-debug"}),
.debug_and_symbols => try argv.appendSlice(&.{"--strip-all"}),
}
if (objcopy.pad_to) |pad_to| {
try argv.appendSlice(&.{ "--pad-to", b.fmt("{d}", .{pad_to}) });
}
if (objcopy.format) |format| switch (format) {
.bin => try argv.appendSlice(&.{ "-O", "binary" }),
.hex => try argv.appendSlice(&.{ "-O", "hex" }),
.elf => try argv.appendSlice(&.{ "-O", "elf" }),
};
if (objcopy.compress_debug) {
try argv.appendSlice(&.{"--compress-debug-sections"});
}
if (objcopy.output_file_debug != null) {
try argv.appendSlice(&.{b.fmt("--extract-to={s}", .{full_dest_path_debug})});
}
if (objcopy.add_section) |section| {
try argv.append("--add-section");
try argv.appendSlice(&.{b.fmt("{s}={s}", .{ section.section_name, section.file_path.getPath2(b, step) })});
}
if (objcopy.set_section_alignment) |set_align| {
try argv.append("--set-section-alignment");
try argv.appendSlice(&.{b.fmt("{s}={d}", .{ set_align.section_name, set_align.alignment })});
}
if (objcopy.set_section_flags) |set_flags| {
const f = set_flags.flags;
// trailing comma is allowed
try argv.append("--set-section-flags");
try argv.appendSlice(&.{b.fmt("{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{
set_flags.section_name,
if (f.alloc) "alloc," else "",
if (f.contents) "contents," else "",
if (f.load) "load," else "",
if (f.readonly) "readonly," else "",
if (f.code) "code," else "",
if (f.exclude) "exclude," else "",
if (f.large) "large," else "",
if (f.merge) "merge," else "",
if (f.strings) "strings," else "",
})});
}
try argv.appendSlice(&.{ full_src_path, full_dest_path });
try argv.append("--listen=-");
_ = try step.evalZigProcess(argv.items, prog_node, false, options.web_server, options.gpa);
objcopy.output_file.path = full_dest_path;
if (objcopy.output_file_debug) |*file| file.path = full_dest_path_debug;
try man.writeManifest();
pub fn getOutputSeparatedDebug(obj_copy: *const ObjCopy) ?std.Build.LazyPath {
return if (obj_copy.output_file_debug.unwrap()) |index| .{ .generated = .{ .index = index } } else null;
}
+2 -2
View File
@@ -421,7 +421,7 @@ pub fn addPrefixedOutputDirectoryArg(
output.* = .{
.prefix = graph.dupeString(prefix),
.basename = graph.dupeString(basename),
.generated_file = .{ .step = &run.step },
.generated_file = graph.addGeneratedFile(&run.step),
};
run.argv.append(arena, .{ .output_directory = output }) catch @panic("OOM");
@@ -429,7 +429,7 @@ pub fn addPrefixedOutputDirectoryArg(
run.setName(std.fmt.allocPrint(arena, "{s} ({s})", .{ run.step.name, basename }) catch @panic("OOM"));
}
return .{ .generated = .{ .file = &output.generated_file } };
return .{ .generated = .{ .index = output.generated_file } };
}
pub fn addDirectoryArg(run: *Run, lazy_directory: std.Build.LazyPath) void {
+1 -1
View File
@@ -31,7 +31,7 @@ pub fn create(owner: *std.Build, options: Options) *TranslateC {
const graph = owner.graph;
const arena = graph.arena;
const translate_c = arena.create(TranslateC) catch @panic("OOM");
const source = options.root_source_file.dupe(owner);
const source = options.root_source_file.dupe(graph);
translate_c.* = .{
.step = Step.init(.{
.tag = base_tag,
+1 -48
View File
@@ -29,11 +29,10 @@ pub const Contents = union(enum) {
pub fn create(owner: *std.Build) *UpdateSourceFiles {
const usf = owner.allocator.create(UpdateSourceFiles) catch @panic("OOM");
usf.* = .{
.step = Step.init(.{
.step = .init(.{
.tag = base_tag,
.name = "UpdateSourceFiles",
.owner = owner,
.makeFn = make,
}),
.output_source_files = .empty,
};
@@ -68,49 +67,3 @@ pub fn addBytesToSource(usf: *UpdateSourceFiles, bytes: []const u8, sub_path: []
.sub_path = sub_path,
}) catch @panic("OOM");
}
fn make(step: *Step, options: Step.MakeOptions) !void {
_ = options;
const b = step.owner;
const io = b.graph.io;
const usf: *UpdateSourceFiles = @fieldParentPtr("step", step);
var any_miss = false;
for (usf.output_source_files.items) |output_source_file| {
if (fs.path.dirname(output_source_file.sub_path)) |dirname| {
b.build_root.handle.createDirPath(io, dirname) catch |err| {
return step.fail("unable to make path '{f}{s}': {t}", .{ b.build_root, dirname, err });
};
}
switch (output_source_file.contents) {
.bytes => |bytes| {
b.build_root.handle.writeFile(io, .{ .sub_path = output_source_file.sub_path, .data = bytes }) catch |err| {
return step.fail("unable to write file '{f}{s}': {t}", .{
b.build_root, output_source_file.sub_path, err,
});
};
any_miss = true;
},
.copy => |file_source| {
if (!step.inputs.populated()) try step.addWatchInput(file_source);
const source_path = file_source.getPath2(b, step);
const prev_status = Io.Dir.updateFile(
.cwd(),
io,
source_path,
b.build_root.handle,
output_source_file.sub_path,
.{},
) catch |err| {
return step.fail("unable to update file from '{s}' to '{f}{s}': {t}", .{
source_path, b.build_root, output_source_file.sub_path, err,
});
};
any_miss = any_miss or prev_status == .stale;
},
}
}
step.result_cached = !any_miss;
}