From 387d550b6c3e73989513ea720437fdc5aac195d3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 4 Feb 2026 13:15:53 -0800 Subject: [PATCH 1/8] compiler: remove btrfs workaround functionality has been fixed in the kernel code for a while now --- lib/std/zig.zig | 1 - src/Package/Fetch.zig | 13 +------------ src/main.zig | 10 +--------- 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 4f0f47b11f..16c15e5109 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -737,7 +737,6 @@ pub const EnvVar = enum { ZIG_BUILD_MULTILINE_ERRORS, ZIG_VERBOSE_LINK, ZIG_VERBOSE_CC, - ZIG_BTRFS_WORKAROUND, ZIG_DEBUG_CMD, ZIG_IS_DETECTING_LIBC_PATHS, ZIG_IS_TRYING_TO_NOT_CALL_ITSELF, diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index d42e441d77..5a600421e9 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -129,7 +129,6 @@ pub const JobQueue = struct { /// two hashes of the same package do not match. /// If this is true, `recursive` must be false. debug_hash: bool, - work_around_btrfs_bug: bool, mode: Mode, /// Set of hashes that will be additionally fetched even if they are marked /// as lazy. @@ -524,16 +523,7 @@ fn runResource( // Fetch and unpack a resource into a temporary directory. var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); - var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; - - // Apply btrfs workaround if needed. Reopen tmp_directory. - if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { - // https://github.com/ziglang/zig/issues/17095 - pkg_path.root_dir.handle.close(io); - pkg_path.root_dir.handle = cache_root.handle.createDirPathOpen(io, tmp_dir_sub_path, .{ - .open_options = .{ .iterate = true }, - }) catch @panic("btrfs workaround failed"); - } + const pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; // Load, parse, and validate the unpacked build.zig.zon file. It is allowed // for the file to be missing, in which case this fetched package is @@ -2276,7 +2266,6 @@ const TestFetchBuilder = struct { .recursive = false, .read_only = false, .debug_hash = false, - .work_around_btrfs_bug = false, .mode = .needed, }; diff --git a/src/main.zig b/src/main.zig index 0efab88f6a..1e9c0de656 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5098,8 +5098,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, } } - const work_around_btrfs_bug = native_os == .linux and - EnvVar.ZIG_BTRFS_WORKAROUND.isSet(environ_map); const root_prog_node = std.Progress.start(io, .{ .disable_printing = (color == .off), .root_name = "Compile Build Script", @@ -5244,7 +5242,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, .read_only = false, .recursive = true, .debug_hash = false, - .work_around_btrfs_bug = work_around_btrfs_bug, .unlazy_set = unlazy_set, .mode = fetch_mode, }; @@ -5254,9 +5251,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, job_queue.global_cache = .{ .path = p, .handle = Io.Dir.cwd().openDir(io, p, .{}) catch |err| { - fatal("unable to open system package directory '{s}': {s}", .{ - p, @errorName(err), - }); + fatal("unable to open system package directory '{s}': {t}", .{ p, err }); }, }; job_queue.read_only = true; @@ -6938,8 +6933,6 @@ fn cmdFetch( dev.check(.fetch_command); const color: Color = .auto; - const work_around_btrfs_bug = native_os == .linux and - EnvVar.ZIG_BTRFS_WORKAROUND.isSet(environ_map); var opt_path_or_url: ?[]const u8 = null; var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); var debug_hash: bool = false; @@ -7010,7 +7003,6 @@ fn cmdFetch( .recursive = false, .read_only = false, .debug_hash = debug_hash, - .work_around_btrfs_bug = work_around_btrfs_bug, .mode = .all, }; defer job_queue.deinit(); From 76d275b20f4ba420f741e019a7456ca392c040e4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 4 Feb 2026 15:31:42 -0800 Subject: [PATCH 2/8] std.Io.Threaded: flatten some switch cases --- lib/std/Io/Threaded.zig | 80 ++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 45 deletions(-) diff --git a/lib/std/Io/Threaded.zig b/lib/std/Io/Threaded.zig index ab2e9af757..f998466e6e 100644 --- a/lib/std/Io/Threaded.zig +++ b/lib/std/Io/Threaded.zig @@ -3191,29 +3191,24 @@ fn dirCreateDirPosix(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8, perm try syscall.checkCancel(); continue; }, - else => |e| { - syscall.finish(); - switch (e) { - .ACCES => return error.AccessDenied, - .BADF => |err| return errnoBug(err), // File descriptor used after closed. - .PERM => return error.PermissionDenied, - .DQUOT => return error.DiskQuota, - .EXIST => return error.PathAlreadyExists, - .FAULT => |err| return errnoBug(err), - .LOOP => return error.SymLinkLoop, - .MLINK => return error.LinkQuotaExceeded, - .NAMETOOLONG => return error.NameTooLong, - .NOENT => return error.FileNotFound, - .NOMEM => return error.SystemResources, - .NOSPC => return error.NoSpaceLeft, - .NOTDIR => return error.NotDir, - .ROFS => return error.ReadOnlyFileSystem, - // dragonfly: when dir_fd is unlinked from filesystem - .NOTCONN => return error.FileNotFound, - .ILSEQ => return error.BadPathName, - else => |err| return posix.unexpectedErrno(err), - } - }, + .ACCES => return syscall.fail(error.AccessDenied), + .PERM => return syscall.fail(error.PermissionDenied), + .DQUOT => return syscall.fail(error.DiskQuota), + .EXIST => return syscall.fail(error.PathAlreadyExists), + .LOOP => return syscall.fail(error.SymLinkLoop), + .MLINK => return syscall.fail(error.LinkQuotaExceeded), + .NAMETOOLONG => return syscall.fail(error.NameTooLong), + .NOENT => return syscall.fail(error.FileNotFound), + .NOMEM => return syscall.fail(error.SystemResources), + .NOSPC => return syscall.fail(error.NoSpaceLeft), + .NOTDIR => return syscall.fail(error.NotDir), + .ROFS => return syscall.fail(error.ReadOnlyFileSystem), + // dragonfly: when dir_fd is unlinked from filesystem + .NOTCONN => return syscall.fail(error.FileNotFound), + .ILSEQ => return syscall.fail(error.BadPathName), + .BADF => |err| return syscall.errnoBug(err), // File descriptor used after closed. + .FAULT => |err| return syscall.errnoBug(err), + else => |err| return syscall.unexpectedErrno(err), } } } @@ -5261,28 +5256,23 @@ fn dirOpenDirPosix( try syscall.checkCancel(); continue; }, - else => |e| { - syscall.finish(); - switch (e) { - .FAULT => |err| return errnoBug(err), - .INVAL => return error.BadPathName, - .BADF => |err| return errnoBug(err), // File descriptor used after closed. - .ACCES => return error.AccessDenied, - .LOOP => return error.SymLinkLoop, - .MFILE => return error.ProcessFdQuotaExceeded, - .NAMETOOLONG => return error.NameTooLong, - .NFILE => return error.SystemFdQuotaExceeded, - .NODEV => return error.NoDevice, - .NOENT => return error.FileNotFound, - .NOMEM => return error.SystemResources, - .NOTDIR => return error.NotDir, - .PERM => return error.PermissionDenied, - .BUSY => |err| return errnoBug(err), // O_EXCL not passed - .NXIO => return error.NoDevice, - .ILSEQ => return error.BadPathName, - else => |err| return posix.unexpectedErrno(err), - } - }, + .INVAL => return syscall.fail(error.BadPathName), + .ACCES => return syscall.fail(error.AccessDenied), + .LOOP => return syscall.fail(error.SymLinkLoop), + .MFILE => return syscall.fail(error.ProcessFdQuotaExceeded), + .NAMETOOLONG => return syscall.fail(error.NameTooLong), + .NFILE => return syscall.fail(error.SystemFdQuotaExceeded), + .NODEV => return syscall.fail(error.NoDevice), + .NOENT => return syscall.fail(error.FileNotFound), + .NOMEM => return syscall.fail(error.SystemResources), + .NOTDIR => return syscall.fail(error.NotDir), + .PERM => return syscall.fail(error.PermissionDenied), + .NXIO => return syscall.fail(error.NoDevice), + .ILSEQ => return syscall.fail(error.BadPathName), + .FAULT => |err| return syscall.errnoBug(err), + .BADF => |err| return syscall.errnoBug(err), // File descriptor used after closed. + .BUSY => |err| return syscall.errnoBug(err), // O_EXCL not passed + else => |err| return syscall.unexpectedErrno(err), } } } From 64dc1cdad8faf1fd8330d4b0f9149817b5205abf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 4 Feb 2026 15:55:56 -0800 Subject: [PATCH 3/8] fetch: download to local zig-pkg directory rather than global cache p/ directory. closes #14283 does not recompress packages into global cache yet --- src/Package/Fetch.zig | 148 ++++++++++++++++++++++-------------------- src/main.zig | 52 +++++++++++---- 2 files changed, 115 insertions(+), 85 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 5a600421e9..96018851ce 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -68,8 +68,7 @@ use_latest_commit: bool, // Above this are fields provided as inputs to `run`. // Below this are fields populated by `run`. -/// This will either be relative to `global_cache`, or to the build root of -/// the root package. +/// Relative to the build root of the root package. package_root: Cache.Path, error_bundle: ErrorBundle.Wip, manifest: ?Manifest, @@ -115,6 +114,9 @@ pub const JobQueue = struct { http_client: *std.http.Client, group: Io.Group = .init, global_cache: Cache.Directory, + local_cache: Cache.Path, + /// Path to "zig-pkg" inside the package in which the user ran `zig build`. + root_pkg_path: Cache.Path, /// If true then, no fetching occurs, and: /// * The `global_cache` directory is assumed to be the direct parent /// directory of on-disk packages rather than having the "p/" directory @@ -325,11 +327,12 @@ pub const RunError = error{ }; pub fn run(f: *Fetch) RunError!void { - const io = f.job_queue.io; + const job_queue = f.job_queue; + const io = job_queue.io; const eb = &f.error_bundle; const arena = f.arena.allocator(); const gpa = f.arena.child_allocator; - const cache_root = f.job_queue.global_cache; + const local_cache_root = job_queue.local_cache; try eb.init(gpa); @@ -350,13 +353,13 @@ pub fn run(f: *Fetch) RunError!void { ); // Packages fetched by URL may not use relative paths to escape outside the // fetched package directory from within the package cache. - if (pkg_root.root_dir.eql(cache_root)) { + if (pkg_root.root_dir.eql(local_cache_root.root_dir)) { // `parent_package_root.sub_path` contains a path like this: // "p/$hash", or // "p/$hash/foo", with possibly more directories after "foo". // We want to fail unless the resolved relative path has a // prefix of "p/$hash/". - const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len; + const prefix_len: usize = if (job_queue.read_only) 0 else "p/".len; const parent_sub_path = f.parent_package_root.sub_path; const end = find_end: { if (parent_sub_path.len > prefix_len) { @@ -379,7 +382,7 @@ pub fn run(f: *Fetch) RunError!void { f.package_root = pkg_root; try loadManifest(f, pkg_root); if (!f.has_build_zig) try checkBuildFileExistence(f); - if (!f.job_queue.recursive) return; + if (!job_queue.recursive) return; return queueJobsForDeps(f); }, .remote => |remote| remote, @@ -411,51 +414,39 @@ pub fn run(f: *Fetch) RunError!void { }; if (remote.hash) |expected_hash| { - var prefixed_pkg_sub_path_buffer: [Package.Hash.max_len + 2]u8 = undefined; - prefixed_pkg_sub_path_buffer[0] = 'p'; - prefixed_pkg_sub_path_buffer[1] = fs.path.sep; - const hash_slice = expected_hash.toSlice(); - @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice); - const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len]; - const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0; - const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..]; - if (cache_root.handle.access(io, pkg_sub_path, .{})) |_| { + const package_root = try job_queue.root_pkg_path.join(arena, expected_hash.toSlice()); + if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| { assert(f.lazy_status != .unavailable); - f.package_root = .{ - .root_dir = cache_root, - .sub_path = try arena.dupe(u8, pkg_sub_path), - }; + f.package_root = package_root; try loadManifest(f, f.package_root); try checkBuildFileExistence(f); - if (!f.job_queue.recursive) return; + if (!job_queue.recursive) return; return queueJobsForDeps(f); } else |err| switch (err) { error.FileNotFound => { switch (f.lazy_status) { .eager => {}, - .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) { + .available => if (!job_queue.unlazy_set.contains(expected_hash)) { f.lazy_status = .unavailable; return; }, .unavailable => unreachable, } - if (f.job_queue.read_only) return f.fail( + if (job_queue.read_only) return f.fail( f.name_tok, - try eb.printString("package not found at '{f}{s}'", .{ - cache_root, pkg_sub_path, - }), + try eb.printString("package not found at '{f}'", .{package_root}), ); }, else => |e| { try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to open global package cache directory '{f}{s}': {s}", .{ - cache_root, pkg_sub_path, @errorName(e), + .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{ + package_root, e, }), }); return error.FetchFailed; }, } - } else if (f.job_queue.read_only) { + } else if (job_queue.read_only) { try eb.addRootErrorMessage(.{ .msg = try eb.addString("dependency is missing hash field"), .src_loc = try f.srcLoc(f.location_tok), @@ -467,7 +458,7 @@ pub fn run(f: *Fetch) RunError!void { const uri = std.Uri.parse(remote.url) catch |err| return f.fail( f.location_tok, - try eb.printString("invalid URI: {s}", .{@errorName(err)}), + try eb.printString("invalid URI: {t}", .{err}), ); var buffer: [init_resource_buffer_size]u8 = undefined; var resource: Resource = undefined; @@ -487,29 +478,30 @@ fn runResource( resource: *Resource, remote_hash: ?Package.Hash, ) RunError!void { - const io = f.job_queue.io; + const job_queue = f.job_queue; + const io = job_queue.io; defer resource.deinit(io); const arena = f.arena.allocator(); const eb = &f.error_bundle; const s = fs.path.sep_str; - const cache_root = f.job_queue.global_cache; + const local_cache_root = job_queue.local_cache; const rand_int = r: { var x: u64 = undefined; io.random(@ptrCast(&x)); break :r x; }; const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(rand_int); + const tmp_directory_path = try local_cache_root.join(arena, tmp_dir_sub_path); const package_sub_path = blk: { - const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); var tmp_directory: Cache.Directory = .{ - .path = tmp_directory_path, + .path = tmp_directory_path.sub_path, .handle = handle: { - const dir = cache_root.handle.createDirPathOpen(io, tmp_dir_sub_path, .{ + const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{ .open_options = .{ .iterate = true }, }) catch |err| { try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to create temporary directory '{s}': {t}", .{ + .msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{ tmp_directory_path, err, }), }); @@ -545,36 +537,33 @@ fn runResource( // directory. f.computed_hash = try computeHash(f, pkg_path, filter); - break :blk if (unpack_result.root_dir.len > 0) - try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir }) - else - tmp_dir_sub_path; + if (unpack_result.root_dir.len > 0) + break :blk try tmp_directory_path.join(arena, unpack_result.root_dir); + + break :blk tmp_directory_path; }; const computed_package_hash = computedPackageHash(f); - // Rename the temporary directory into the global zig package cache - // directory. If the hash already exists, delete the temporary directory - // and leave the zig package cache directory untouched as it may be in use - // by the system. This is done even if the hash is invalid, in case the - // package with the different hash is used in the future. - - f.package_root = .{ - .root_dir = cache_root, - .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}), - }; - renameTmpIntoCache(io, cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| { - const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); - const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); + // Rename the temporary directory into the local zig package directory. If + // the hash already exists, delete the temporary directory and leave the + // zig package directory untouched as it may be in use. This is done even + // if the hash is invalid, in case the package with the different hash is + // used in the future. + f.package_root = try job_queue.root_pkg_path.join(arena, computed_package_hash.toSlice()); + renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| { try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}", - .{ src, dest, @errorName(err) }, + "unable to rename temporary directory {f} into package cache directory {f}: {t}", + .{ package_sub_path, f.package_root, err }, ) }); return error.FetchFailed; }; // Remove temporary directory root if not already renamed to global cache. - if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) { - cache_root.handle.deleteDir(io, tmp_dir_sub_path) catch {}; + if (!package_sub_path.eql(tmp_directory_path)) { + tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| std.log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_directory_path, e }), + }; } // Validate the computed hash against the expected hash. If invalid, this @@ -614,7 +603,7 @@ fn runResource( // Spawn a new fetch job for each dependency in the manifest file. Use // a mutex and a hash map so that redundant jobs do not get queued up. - if (!f.job_queue.recursive) return; + if (!job_queue.recursive) return; return queueJobsForDeps(f); } @@ -641,8 +630,8 @@ fn checkBuildFileExistence(f: *Fetch) RunError!void { error.FileNotFound => {}, else => |e| { try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to access '{f}{s}': {s}", .{ - f.package_root, Package.build_zig_basename, @errorName(e), + .msg = try eb.printString("unable to access '{f}{s}': {t}", .{ + f.package_root, Package.build_zig_basename, e, }), }); return error.FetchFailed; @@ -667,9 +656,7 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { else => |e| { const file_path = try pkg_root.join(arena, Manifest.basename); try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to load package manifest '{f}': {s}", .{ - file_path, @errorName(e), - }), + .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ file_path, e }), }); return error.FetchFailed; }, @@ -1453,14 +1440,20 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void } } -pub fn renameTmpIntoCache(io: Io, cache_dir: Io.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void { - assert(dest_dir_sub_path[1] == fs.path.sep); +pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) !void { var handled_missing_dir = false; while (true) { - cache_dir.rename(tmp_dir_sub_path, cache_dir, dest_dir_sub_path, io) catch |err| switch (err) { + Io.Dir.rename( + tmp_path.root_dir.handle, + tmp_path.sub_path, + dest_path.root_dir.handle, + dest_path.sub_path, + io, + ) catch |err| switch (err) { error.FileNotFound => { if (handled_missing_dir) return err; - cache_dir.createDir(io, dest_dir_sub_path[0..1], .default_dir) catch |mkd_err| switch (mkd_err) { + const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?; + dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch |er| switch (er) { error.PathAlreadyExists => handled_missing_dir = true, else => |e| return e, }; @@ -1468,9 +1461,11 @@ pub fn renameTmpIntoCache(io: Io, cache_dir: Io.Dir, tmp_dir_sub_path: []const u }, error.DirNotEmpty, error.AccessDenied => { // Package has been already downloaded and may already be in use on the system. - cache_dir.deleteTree(io, tmp_dir_sub_path) catch { + tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch |er| switch (er) { + error.Canceled => |e| return e, // Garbage files leftover in zig-cache/tmp/ is, as they say // on Star Trek, "operating within normal parameters". + else => |e| std.log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }), }; }, else => |e| return e, @@ -2244,6 +2239,7 @@ fn saveEmbedFile(io: Io, comptime tarball_name: []const u8, dir: Io.Dir) !void { const TestFetchBuilder = struct { http_client: std.http.Client, global_cache_directory: Cache.Directory, + local_cache_path: Cache.Path, job_queue: Fetch.JobQueue, fetch: Fetch, @@ -2254,15 +2250,25 @@ const TestFetchBuilder = struct { cache_parent_dir: std.Io.Dir, path_or_url: []const u8, ) !*Fetch { - const cache_dir = try cache_parent_dir.createDirPathOpen(io, "zig-global-cache", .{}); + const global_cache_dir = try cache_parent_dir.createDirPathOpen(io, "zig-global-cache", .{}); + const package_root_dir = try cache_parent_dir.createDirPathOpen(io, "local-project-root", .{}); self.http_client = .{ .allocator = allocator, .io = io }; - self.global_cache_directory = .{ .handle = cache_dir, .path = null }; + self.global_cache_directory = .{ .handle = global_cache_dir, .path = "zig-global-cache" }; + self.local_cache_path = .{ + .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" }, + .sub_path = ".zig-cache", + }; self.job_queue = .{ .io = io, .http_client = &self.http_client, .global_cache = self.global_cache_directory, + .local_cache = self.local_cache_path, + .root_pkg_path = .{ + .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" }, + .sub_path = "zig-pkg", + }, .recursive = false, .read_only = false, .debug_hash = false, @@ -2276,7 +2282,7 @@ const TestFetchBuilder = struct { .hash_tok = .none, .name_tok = 0, .lazy_status = .eager, - .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } }, + .parent_package_root = .{ .root_dir = .{ .handle = package_root_dir, .path = null } }, .parent_manifest_ast = null, .prog_node = std.Progress.Node.none, .job_queue = &self.job_queue, diff --git a/src/main.zig b/src/main.zig index 1e9c0de656..02a49ef3e0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5239,6 +5239,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, .io = io, .http_client = &http_client, .global_cache = dirs.global_cache, + .local_cache = .{ .root_dir = dirs.local_cache, .sub_path = "" }, + .root_pkg_path = .{ .root_dir = build_root.directory, .sub_path = "zig-pkg" }, .read_only = false, .recursive = true, .debug_hash = false, @@ -5248,12 +5250,17 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, defer job_queue.deinit(); if (system_pkg_dir_path) |p| { - job_queue.global_cache = .{ - .path = p, - .handle = Io.Dir.cwd().openDir(io, p, .{}) catch |err| { - fatal("unable to open system package directory '{s}': {t}", .{ p, err }); + const system_pkg_path: Path = .{ + .root_dir = .{ + .path = p, + .handle = Io.Dir.cwd().openDir(io, p, .{}) catch |err| { + fatal("unable to open system package directory '{s}': {t}", .{ p, err }); + }, }, + .sub_path = "", }; + job_queue.global_cache = system_pkg_path.root_dir; + job_queue.root_pkg_path = system_pkg_path; job_queue.read_only = true; cleanup_build_dir = job_queue.global_cache.handle; } else { @@ -6996,10 +7003,27 @@ fn cmdFetch( }; defer global_cache_directory.handle.close(io); + const cwd_path = try introspect.getResolvedCwd(io, arena); + + var build_root = try findBuildRoot(arena, io, .{ + .cwd_path = cwd_path, + }); + defer build_root.deinit(io); + + const local_cache_path: Path = .{ + .root_dir = build_root.directory, + .sub_path = ".zig-cache", + }; + var job_queue: Package.Fetch.JobQueue = .{ .io = io, .http_client = &http_client, .global_cache = global_cache_directory, + .local_cache = local_cache_path, + .root_pkg_path = .{ + .root_dir = build_root.directory, + .sub_path = "zig-pkg", + }, .recursive = false, .read_only = false, .debug_hash = debug_hash, @@ -7069,13 +7093,6 @@ fn cmdFetch( }, }; - const cwd_path = try introspect.getResolvedCwd(io, arena); - - var build_root = try findBuildRoot(arena, io, .{ - .cwd_path = cwd_path, - }); - defer build_root.deinit(io); - // The name to use in case the manifest file needs to be created now. const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path); var manifest, var ast = try loadManifest(gpa, arena, io, .{ @@ -7239,18 +7256,25 @@ fn createDependenciesModule( defer tmp_dir.close(io); try tmp_dir.writeFile(io, .{ .sub_path = basename, .data = source }); } + const tmp_dir_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = tmp_dir_sub_path, + }; var hh: Cache.HashHelper = .{}; hh.addBytes(build_options.version); hh.addBytes(source); const hex_digest = hh.final(); - const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest); - try Package.Fetch.renameTmpIntoCache(io, dirs.local_cache.handle, tmp_dir_sub_path, o_dir_sub_path); + const o_dir_path: Path = .{ + .root_dir = dirs.local_cache, + .sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest), + }; + try Package.Fetch.renameTmpIntoCache(io, tmp_dir_path, o_dir_path); const deps_mod = try Package.Module.create(arena, .{ .paths = .{ - .root = try .fromRoot(arena, dirs, .local_cache, o_dir_sub_path), + .root = try .fromRoot(arena, dirs, .local_cache, o_dir_path.sub_path), .root_src_path = basename, }, .fully_qualified_name = "root.@dependencies", From df64a3a36815fce6cc8671d047e52795655b3b9b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 4 Feb 2026 17:40:54 -0800 Subject: [PATCH 4/8] build: packages now require fingerprint also the name must be an enum literal. delete some .tar.gz test data. Test data should be in text form when it can be, and this could definitely be. --- src/Package/Fetch.zig | 132 ------------------ .../Fetch/testdata/duplicate_paths.tar.gz | Bin 3230 -> 0 bytes .../testdata/duplicate_paths_excluded.tar.gz | Bin 3237 -> 0 bytes src/Package/Fetch/testdata/no_root.tar.gz | Bin 3172 -> 0 bytes src/Package/Manifest.zig | 34 +---- src/main.zig | 4 - 6 files changed, 7 insertions(+), 163 deletions(-) delete mode 100644 src/Package/Fetch/testdata/duplicate_paths.tar.gz delete mode 100644 src/Package/Fetch/testdata/duplicate_paths_excluded.tar.gz delete mode 100644 src/Package/Fetch/testdata/no_root.tar.gz diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 96018851ce..a9cc86b398 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -60,8 +60,6 @@ omit_missing_hash_error: bool, /// which specifies inclusion rules. This is intended to be true for the first /// fetch task and false for the recursive dependencies. allow_missing_paths_field: bool, -allow_missing_fingerprint: bool, -allow_name_string: bool, /// If true and URL points to a Git repository, will use the latest commit. use_latest_commit: bool, @@ -675,8 +673,6 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { f.manifest = try Manifest.parse(arena, ast.*, rng.interface(), .{ .allow_missing_paths_field = f.allow_missing_paths_field, - .allow_missing_fingerprint = f.allow_missing_fingerprint, - .allow_name_string = f.allow_name_string, }); const manifest = &f.manifest.?; @@ -794,8 +790,6 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { .job_queue = f.job_queue, .omit_missing_hash_error = false, .allow_missing_paths_field = true, - .allow_missing_fingerprint = true, - .allow_name_string = true, .use_latest_commit = false, .package_root = undefined, @@ -2049,130 +2043,6 @@ const UnpackResult = struct { } }; -test "tarball with duplicate paths" { - // This tarball has duplicate path 'dir1/file1' to simulate case sensitve - // file system on any file sytstem. - // - // duplicate_paths/ - // duplicate_paths/dir1/ - // duplicate_paths/dir1/file1 - // duplicate_paths/dir1/file1 - // duplicate_paths/build.zig.zon - // duplicate_paths/src/ - // duplicate_paths/src/main.zig - // duplicate_paths/src/root.zig - // duplicate_paths/build.zig - // - - const gpa = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const tarball_name = "duplicate_paths.tar.gz"; - try saveEmbedFile(io, tarball_name, tmp.dir); - const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); - defer gpa.free(tarball_path); - - // Run tarball fetch, expect to fail - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, io, tmp.dir, tarball_path); - defer fb.deinit(); - try std.testing.expectError(error.FetchFailed, fetch.run()); - - try fb.expectFetchErrors(1, - \\error: unable to unpack tarball - \\ note: unable to create file 'dir1/file1': PathAlreadyExists - \\ - ); -} - -test "tarball with excluded duplicate paths" { - // Same as previous tarball but has build.zig.zon wich excludes 'dir1'. - // - // .paths = .{ - // "build.zig", - // "build.zig.zon", - // "src", - // } - // - - const gpa = std.testing.allocator; - const io = std.testing.io; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const tarball_name = "duplicate_paths_excluded.tar.gz"; - try saveEmbedFile(io, tarball_name, tmp.dir); - const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); - defer gpa.free(tarball_path); - - // Run tarball fetch, should succeed - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, io, tmp.dir, tarball_path); - defer fb.deinit(); - try fetch.run(); - - const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); - try std.testing.expectEqualStrings( - "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da", - &hex_digest, - ); - - const expected_files: []const []const u8 = &.{ - "build.zig", - "build.zig.zon", - "src/main.zig", - "src/root.zig", - }; - try fb.expectPackageFiles(expected_files); -} - -test "tarball without root folder" { - // Tarball with root folder. Manifest excludes dir1 and dir2. - // - // build.zig - // build.zig.zon - // dir1/ - // dir1/file2 - // dir1/file1 - // dir2/ - // dir2/file2 - // src/ - // src/main.zig - // - - const gpa = std.testing.allocator; - const io = std.testing.io; - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const tarball_name = "no_root.tar.gz"; - try saveEmbedFile(io, tarball_name, tmp.dir); - const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); - defer gpa.free(tarball_path); - - // Run tarball fetch, should succeed - var fb: TestFetchBuilder = undefined; - var fetch = try fb.build(gpa, io, tmp.dir, tarball_path); - defer fb.deinit(); - try fetch.run(); - - const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); - try std.testing.expectEqualStrings( - "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793", - &hex_digest, - ); - - const expected_files: []const []const u8 = &.{ - "build.zig", - "build.zig.zon", - "src/main.zig", - }; - try fb.expectPackageFiles(expected_files); -} - test "set executable bit based on file content" { if (!Io.File.Permissions.has_executable_bit) return error.SkipZigTest; const gpa = std.testing.allocator; @@ -2288,8 +2158,6 @@ const TestFetchBuilder = struct { .job_queue = &self.job_queue, .omit_missing_hash_error = true, .allow_missing_paths_field = false, - .allow_missing_fingerprint = true, // so we can keep using the old testdata .tar.gz - .allow_name_string = true, // so we can keep using the old testdata .tar.gz .use_latest_commit = true, .package_root = undefined, diff --git a/src/Package/Fetch/testdata/duplicate_paths.tar.gz b/src/Package/Fetch/testdata/duplicate_paths.tar.gz deleted file mode 100644 index 118a934c1b03d764e4854f9aeb60ed684467caad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3230 zcmV;P3}N#hiwFoH>keiD17vk@Y-wX*bY)*~VRUG7E_7jX0PR|9a~ro6^=JPIgdZBp zrB+W%GjZLClPFWqxOK-$nwh2@5!?ky#QO?>rA#~e?>+YdSniTi;!GWRl3CaziMznX zeVqHi1+%8kbt;{@s-;`ng9pFCIK((Ve@wrR&L1Ckf5-9Q==ALHF9jdjup@j%N`r$00Am)`$QN491EOq-*Fk7)ms!xPy5^zis}u>VK1|6J$l z=)>Co?CkVC?SDx2KR-EsAPxun5B4AIf3NnRH9F6dw|bGht;+kc|KroMv)2BHXUG#r zkIyLn5BC4%Rx)o?B%Tzy)J0QV!ELU3EFK;G8XhlXx`f!Z zDl=84Yf%83LSat@t(a=1GX+gf8@{s^O~H#`XBPYWbLG;bqw0w{oS#U!a-4P8mcJ zBnzN4+s}2W{6=#Lw)th2!Kgj%lMeMny^%$otDRl_&+nG16iuB0mmuOcCWT6+DAl$2 z_SK6CDcTa^@ibGpa`#EEmta`Lti;~T+wN+wWHLO_@rzzPRcvtE8m3mV{ zZ){BsM9>x$SRpb(y4?+ELSQl3dYeFdgxcuKK?@0XKA2Zkf-spZ-iCnSm+jJT-K;9= z#*s&xs^DFzrXV{zcprm|9-`U+`|N~so?*Rc+U}CBW=_JEnUHfB(_5Z&vGE64*ZJD3 zhfJR44SS+#i0%Ry2K?8zD4iqSG}(D`hj^_6(>Ath3}8j_n8q!iM9;AlKPZ$y6Uu8L zxb72LcMBW6iHw6J$cC+cpioeiyb)l%7IkH~$`nD+EKt z8R8`hMW(45;DANlK+C8cUbA9X#Qa^_K1`abatS3X-X)b*b*X=9y;BhbaN%67kOCQX zy+by4UK+_eg zP_m5LLe}$opjtqey#zVwDInxA4lX;Xgt1V_*C6x=G#`1QTh#6}i+D~UQY=)d3<0N0 zd8V9WnJG7*e9~0wkTh)+-nF4FA*>{Ks_1V{Tg!g>Xv3`!=+exvF&O5sZr4~}2o`n1 z)MFJy0pC

kWwy5ur;oK*Un**`@}<%Lf0W9^A#(fJ;nFN!Lq8?ZKtDVAMO$^j}$% z_8+`>{^i9#E-(5IUtK)=>hBjxk&RkcyLtG>J^w)|AG`<5{lCZW*Z(^@=l<{c+40c- z`&jlbP%Mzk+*AMO=p>0lNf|v~gl4>7t(=j2m6(&42p7%6Qh~zkz)AIspi89~ft**J^EV*S6 z&T~bLyhd`))()u@!Z#={==6tWozlZ|H{mk@74R(eg6I}lPbUVS&RTr=q96 zmyH|mCD%qH&5t2IEm{G>v+K>wSu;oBqq6V(tPZuXpt}o}&8|BHd4eJfZKz)w7b_wF7iL5F=b)yzWXM3>r;j0r7)MiW#U6%1UNAZ^BFAQ1rAc(&(+n4^PrFfi9x zq!6lLtYScrz!^^Osq+POp$aY!m3;k4D?MTv}p% zA~1X6m)BK2-ow&+E-r41|6E$%U;IBjK0n`y|M)e;|BoF1{kwqQ;CkT`yc>ANWXcaR z%f|9E!GNCe?>+hveAg(~7&?*vS;yvao-?LxPA1}f zPdt4pjt=kNSN@3O&ikJaT>t-!|DT_p?D#(>4F3Po>p$v=y zDZ{-Pu4`0i-Xc~0P|dhQS`m#*rMdL;^0hBLOp_wbhEif-WWDrj<86?7qf+h<&DO}q z_(&H8GAC-NH%euaSe!D6F~ZIx3Qu3qr#;cHR4*M8x-=QJEvV-37cZ8Y64kX&>yX28 ztGFz=r`)-|K!uVntIDeO7Kj)Pr0~+qfN~}gf1_^EwTi9ZYK5|xE^WBgt>&`Hot^%-M)uvt;!MGA8wjg*9!IZ zS5zEN#4Ef+m47Mcs{Qb_w9acLzAdoVw(i@in*(z<$A{Rk;R}nr_&dDW~5n;`n~*dLXr!JZWH-GgIGthqZdm}cO^vPzs(zn%qLHEF$bDO# zce>OGm5f)Og?MO5-fh~}-KugVZZ_g1E$I9ooi`mV$3H^{t^-2uqx$&5eN1X0w#UER zYnYu>ag!4U=XPr7BuCptYJ6%S6kq$>h5!m(uBz9ep*{*^Mp&$BG48r2KcS_O-Yu99 z*W$Az(0OK@p5vT42D5#@AOuFhTyQibfBltcc^|vPZqNPMMYZDGX`0QcIRu&i&omoy z!9u^vG~jnzv`48?P`G^rr=Wu|6Ata{PEwl$0o!!Od&`pVcY92MrlKzuuyb)L^w-mt z`X}1a?GfGFzM();`mV^e&z6YyP^GqQO0f-H>vz6L(2ZS?`zkH6h%E4ahnQci>3$*x zM-yfAREa5n$;z7ISCj6yIcR>o`rX|2^JH(v&h?z+R0f&jQ@U%Swf>65_gQ4xy)NGQ zS_h@uQl->MG3#!)^Y_f1mQPo&Y=#Z`+BE9#VmSCIz7Ao4ty}4+<)gy&UtrLmQPc*g zp2eWf5)+m@`0zlWRPUND<{ z)kVFA*NP?ktlP{XbM5aF^&|RmER;*9Oacl7yLe{z#833=C$Wh@Q91zW8D5Qfn(uzx z^yLZe(w2Oz54y&RDqU%RogHrtyI)^fs=S^eqm@njZ6p#Ye4WKt5TyCLM4R?l@lF$b z&2L6?JiNPL?8J{yv=P$UD6{QVyxvy?u;>VnMiaqZvtXxDP)(e?!NZ@rFzGh?$;u9UeM$->&U5!Z>aZEe-K%@>@=;Myhq`s9j;UvVEV#)oh4p_g+$N}}H= z`F1Y5QLH~>C#8NM89#i|+{k!4ms^qg78$7~2W6yOKC=*ZcEY!kCf5`geQ7{)M$F^pjh Q<99Uv2QI@^xByT905N}g`2YX_ diff --git a/src/Package/Fetch/testdata/duplicate_paths_excluded.tar.gz b/src/Package/Fetch/testdata/duplicate_paths_excluded.tar.gz deleted file mode 100644 index 760b37cd40fe43e0907f14943a33ae8341f2f649..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3237 zcmV;W3|jLaiwFqP>keiD17vk@Y-wX*bY)*~VRUG7UuAe>Y;|O1WG-}JascgGYjYd7 z71d|`3YZ@n(xq3To-=XXiIb>P&$xBRN}8Fb9TD6GNyPgKfF(^k`tLpW0=wKLrNo)K z@?^5GM-q2|i~BhDfeR*8nd(?NbyG^Wv_}X3hjEN?d3jF1Czt2P?cen{n4Dc4pPpZw zUz|-2jwh#+%hLmKe((UJvQ8Qy4z$d5{4?`@>HTkXBs=y`w3!@zI{TlU9-qSgXU8XJ zz5PF#{pUJWlaFivi;J@lwEr>L|MK+YKpgk>@9p2)|3U3Pt8|(~@AM*iSL6?4|MRoS zNt6H2&o5yA^NW*7Z~uoF(fdPz5#=&d;+Yui;9)T4*GEUYn#FLErJSFSQjto#a5%*DU-LTD$}I!(n)iOQ-( z%nKt*8NZPWB@$Khf(4#fR4)gk#}WR4y$*kKt(`c9dxH3lNF5L*3th6o6 zMihCvdCKf0g|Z?qoY*Lb4?^Y}x77KfCbHIUDI8H+$+S{6@hH=|&Z_JNZgbONaWeTe zJYLCo39(C+Cn}FOA_F#s!k!9RF~v$J3Ywf&CY^${L`;{`E~jGwrQtEzYueD7f)~Ng zES@}>D;F=PB9?ihX;%n-DkRk3?5QC?Oe1R}jD=CqnAIzlZW`LJ@>uP!lQ1unRLQut z{S*onX^}4=gi$|M+9+ES(z#P6hpyuy&sFU7O1rR4)S>-M1e}J2*Gl;OtPmL?-TnqNA+Q+jyp5neLT&BKK?@OfKAIOrgfJN`-i3hR zm+jMU+^opT%8^H#BI8}DrXV{zcprm|9-`U+`|N~so?*Rc+U}DsW=_JEiI8&`(_5Z& zvGoU8mg&Z;hfE%)6?>woi0%Ry2K+a7D4iqSG}?P}k9ds((>AuK3}8j_n941m)}CW2 zeo!cZCY0AgaNVc0?hZCO6B!3lkPTb?K%t<>c_YAjBg(>BJ%dU!ZA@V-1AAGl0fqZi zw{>ff2+CvlhTT3u{%@`FVt=1isnfJ>8~phZ*Z^@G>EfuFax@bD@gC$Dr@&kkMZW+n z_Bt2P6@nq+4DnJ6MW!h#;DANlLd&QWUbA9X#Qc5QK1`aTa1kXd-X)e+wW)t*y;Bhb zaN$&~kOCQX14G6bA9<(YDhWv1MK@<~&zL(()+c;ALPhp>{|si40(Z7sX$qY1Y@pi48u#$cGk zy4_%XCRo%lQ?IKa3izf-S#L;uhzM<>0V0-a&n7hxUbgrjGzXFv?gyPyO8bR1amPOf zFJF9h_4n(m?!(tt&%gfLRg@)z#(i!N4}OeSa2~yM-Ix7p~h68*YjGi7I zmeowmb3u3*j>Mx?p%d}GNzll(2-ip%n68WwL+w7J{(xPANL)EFlXTKV7MNlND#B=? z+_g)tj2VumBc?+y5;dz9QAy;6gXc&TIM^8R-rjOLohdpz(l(Wi{*7RdOge}QECThx zEwLFG0;+SY%9vmB3RJ8&n4|WjZE=w!)aF*5^IV6KZmt%jtQBv_zS2@lKaq zapv4I2tEtfv!$PkN0(F#CqH z>od_&->b?EN7347r1>Glr$sA3c(&fooKE-&}uKYsP`{}ac5|1RJ+xL)`K?*^VR zneu~7lA-)uFra7rdqh8i?;7PQLnrb-irLexmm_@Zn3jrpmB+pls@t-!|6iV+?)g6^^#1?J>p$v=qz)C$B3ygat}4=^ zcw?zNQigjoT-T`3yhW<~p_p-pv>+OZic{(5r5j&*m?}n^4W-1w$a3k|*0(|Gt%|un zG}|B><0D-Z$dss|-pG}S>f)413=wu7Q+WE4K8-}TQoVLa=+Y$AwxF8BU%XgqN)&6K z)**-GR&kzlPq}q{feIyF7KK&KEf6spNa3ZI0p(02{z~1VwW?dc#R_FHUD|M~Tg_#a zIy=UC9WQHQzPCgLVW6l3U)0?U?ZtX6y~|mRk$At~tXTRD+QX8vBMHz4E&ZTV-8atHea6-*G|R)$IqU)~Xb-{czj7 zS}WAoUsG{B7O(LVRsOY{tLDRBq;+01_1g@4?dra%x=CPBavu=&F!d5?jmU6-7<(q` z3xmq~wi(b$HuJsoTDXp)>SrBQy$TP#vf8@xi&8z~`N(3>oYV13Ju}j5hONfnFWkaz z?^C-iYC50Q7ZH47pxz?XL6@A|lpB*Y3TULBB4jt6CODm8{BB7AvyF`|395dd`a~l` z$C3NCI&F2SBPtnhJPY-qC3?SYTeqvqfw7uIFgUkULnpbmT}_Qo4TR#gzikMh(D|x(6B_D+KxTl&suaVvd-4-n zTGP7&^Wj>276m%bjni|SQ-@%-3mAmJ8ZcKJ4ar}BC0ahjE_Ju(;q0OsaqczE_S77L z%>QPZExBN!Uu7EbyDQq`*eEF6K891!!I%k$c6KYNje~$qI^(@%&iA_=ra)8CmkQXq zI2HQqX-oYR?dbN1Zf@UFAjy4KWbLyh;yqNUZIe=LMc4YRFA}t47v#Q)v!q5Ac)v%? zFE?~QQ3pp;%jl^RQ~r{bHN~&S?Qe6?{QBy5d)v>Gy%}5AbCOdTWUimmT@$VKS1i8I zBIEXT@!r=uDBb2NrcR34w8NdhXKuB8+InR(Y|+=GQGXZ1!B6pZ2m@@>NJlLn6|VmR zgZ_-7wm|hP26acIpzxK3SNU*8t(+Hr+iSexgMUY)R%ue_rBYhD4_j233VpX-$=dik z+y?iI+2pG(>NUJkEZOJnW)7L_$rGYJp&utgxp>MXpg^#TXLclhqF+CWZ3L>N1CXBM z)sUz8?#FFcp5QJ`$;bMjYphzOEA6kd>s!P2*H@M*EvLw6dDVOyi9`xtXYmyTY5qRZ zrX5zi(*$4ho6#H(?=~1)@go#%gtR8g?0OZi_Z0ywI>4jBSa8=Y*l7?{Q%~OF;SX(? zw4m@+qctW@@w3Wxb0iN}%G``(VQtfhYsJ{6wyNCr3r=Kk?V@gdazn&#xQ|!Ihi~wq zlXE^vqTed{ZZ6wVtUF^brG6wCKYr5O%6K=IJCXVh8L1`*Wu#m_vk-Rn!ncwp)f5L(T{%gqaXd~M?d<}kAC!{AN}Y@Kl;&+ Xe)OXs{pd$OenaEGmP{QK08jt`nu}hw diff --git a/src/Package/Fetch/testdata/no_root.tar.gz b/src/Package/Fetch/testdata/no_root.tar.gz deleted file mode 100644 index a3a4baf40fd1d6b2bd101ee59f27f88c6a61f97f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3172 zcmV-q44d;GiwFq9_zq?O18#3$a&K>RE_7jX0PPz6Z`(FDKj*LD{KL&I93@U-2Uu^1 zWm|?T(7O)Jb^~sAh-rzoxk{o&Qg+=g|NFi7NXnF*^aES69i}&|ttImDeSe9wTIN~u zMlKE>4o(0jlXLugbTK}O|3>olj2A#*_1t$@t)8G@6`G4%o>9($qk0EZ2-3 zNM6eH9qZor{{IwGRT|5T&DiJc3t3c3+kOvz_6AQ6508)8Wp0bY8huUv_7`byjr@w zYnkV4CYX3F(%OoQ%{Gje@Y4!i@*J`0x-5lG4y$^`=A}c?pMA!j0M*Ht_-nvcN@ncs zAp?LN*Or$V*BJv%FN9^PLLLm`d9Kz#fVZTh50a+n9Qc`XLM(ODi*$^IQ z@m_pak{A>A0-o!@Nk9ffFfBm360`(tn7TrRiiUFo|8t;`uei+lEEfrFbw=RVe9mTx z@e=!v9=&j<^pm%@XcfCsMIs9MhLd5mWcvZ4H6j0v(h!sje`%Mv^s`7t^@#wD0&dhB+~ zPSO~i`$zO9l5+T`dm9_!cSxX%F9PCGX+>$t4Nxw*lQ20c2nDnN3J2O@nW6FasqF)?ZG(Xj z7!2l$q9OY0M_S8==mo9karGh^cJ5V8U0TYx&5L{H*xF&zHibZ3zh}6-^1pY8;C2fLU7c9M~4-O!pB}x z^a^6jHh<&#xmHE2OTXBuF_RkNz5?%rc!H@6cn(h?ZqeSgg!tnQN_mRf44aacGB7Pd zx-CU2i~)y_!`+;=^U8S$oxGJRsbOnPYfC~W+6|YcwGT$ zVr!_5L@(|H1(djxUzGh>0xtE;9eWLL`h#8jib|7^mnv@Hxn9%-9F3*b9%r%r1#A%9 z3nJ4(rkxMLdR__Q?6b)9NnJ-rh<${AoHAk4DWL!X0=e+a4A?LD*Dq`vfdY46NSE-c zPs^UVV9@5ryGv8@kv!P5t--9Mkr)s>4-sugM@x}cQ^;s#Jq$>Zr4$SXcDf1zC4V1l z(-tYtX}qtw!*Go=@7Nfl_~DB-w6rG5?0OaLcNGC>v$}7;q)KoYV%8Z9zKvpRQ(M&#Q~WgNxYEA~?AZ^YwopES2T-c99Bq`rektjPg0q+IG*(02C1 zx8ORL7#Lk}dtE_agZ4E=Ox_2eY4hmb<%tjk)kK_soCB z=cCgw{~3?RlY^7d={daa=0A@BKJLAzmxwAY&VIZWP*Ok#%l-weJro}Q?JupZ%xA~P zMifwXB}y-j2ThFz{VHek+Cm-0R&EDVbM|kDD^bM=VS}b9Q0Jtt_ySXJprxs%lCT{J zJxSm{Ag@K=TndO&m=mcB=lNctnATcD{e-P?1~ms3pqM|O5;;wJ+z8B-IF`W*bkR(H zpdfXT6Uzq?kSB#KWlfOqfwEmc+g&PxaJhHeJJ~25S zj*bjp3HDcz%-^PDL_8F}N7OFSL6ACydzn$%icd%)?1^~Ii^`|{_*D1j57@%hRTi>q zqf!tlhbCmrzJK|ANR=sCJWZ5x;Y-a_f%%zAm02*W7Vbu)sPnZnf^Aeyl0|kPL8akT zXb|z0Ydc7r(hmz2kb-C!xBCZJ>LQ86Wan)H(nG5azU-xtz{$sRrLee37H?XksD1p6 zo2jCrhBq{UZZHLUw$!~S*OzU(KzKlHmmeXUebj1Zq38mXd=AQVnx_uF@dxaMHlfCl zWt!LIiMmF-3$S4@|IHmjr${&5mB+p8H4aQ8BsX?Lespb@PlM-}Fa!ok2qy*ax=*Q> z+{fKoNOq8T-r%U8Fcheg4k~b3427F(m{BuryKBQve}i`6IAFuz!6)5^4-VnHpg5;K#+6k^@Q{9#GWD5ny(d84s`6SGoTG;MY3l2`;r<=xu;l!9{!dlmWL2iKZNn)*o+%!>m zUxzr-5Im!4r7T_ATDH?i6K-8V=XwSQ1Hly5%?9=t^r^wPZ!@6^!ho+;PI8097e=wc z5S)n^tUa65fcCPr|6bFeM0Ve6buIA==*?UH(R=>vtE+!rU$q~;yt@4QA6H3{^&0p2 z6SR7F0+_+*_<_%$kNg+o`}#kG=KKHI$r<#2CzJ8n0UJLsT?Bvf{*UyJPJH@sXS}EX zzeE3{@nqcf|BnHU{zngY#>eUZ#OZ(3>HjeR^nd!tiT@Ypr)L)w|0iAie-t47|6$^P z(En(B(Z&Bq0sNTt&}WeUk3Z1=>Hg=*WHjph|1lu`nE6m=yvP4Xlhd=E`@g5B<1YU{ z3cU0FufJl8_R$}(PujO?ztz&;LxJT&Yu1`LqYU^*Z?_#(s0o=Ju1`n}6M$HBH{Z_O=@Q}Ct}4k{I^P4XYSQ<1#wr3y9y_FjDF z3~;I1x-p>p6wOl{tTA|6)iehq%+ZyZAuSE_en|_OD<(M0!RL721}7hjH?pFMW9hD3 zxOskomw^o3RK}|sz`VaLNjHSHqbEll-IJ#4zQpwc@fLV%X(3wQ!rkg{`=BMh7q#sV zlC_prfE?Xt?G>2tY`tAMtLJl}MfQVR6=Cxm;#2T*`#M7CK9uW?e;qV~pWvgfO76k{ z&e;477x(4dYv?w{1K8(zZI*x>C6TG}Q@RZ3FHqCFPY!Pn59#Jg&yds!U(cHx@<%f) z$Kj>ceDfTB_vP8?Ad#iC$YT=j#o+BJP_WXT+lvl3<2KR*pu4V(lUqTHvNtDnIl`(JQ_qz@Ua z9=h$7a@WBAtb?cyI_RK-4m#+dgAO|Apo0!N=%9lRI_RK-4m#+dgAO|Apo0#675ooA KksQ4MPyhh%Hx@Dg diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig index a8bc8b5013..e66a78ae14 100644 --- a/src/Package/Manifest.zig +++ b/src/Package/Manifest.zig @@ -49,10 +49,6 @@ arena_state: std.heap.ArenaAllocator.State, pub const ParseOptions = struct { allow_missing_paths_field: bool = false, - /// Deprecated, to be removed after 0.14.0 is tagged. - allow_name_string: bool = true, - /// Deprecated, to be removed after 0.14.0 is tagged. - allow_missing_fingerprint: bool = true, }; pub const Error = Allocator.Error; @@ -77,8 +73,6 @@ pub fn parse(gpa: Allocator, ast: Ast, rng: std.Random, options: ParseOptions) E .dependencies_node = .none, .paths = .{}, .allow_missing_paths_field = options.allow_missing_paths_field, - .allow_name_string = options.allow_name_string, - .allow_missing_fingerprint = options.allow_missing_fingerprint, .minimum_zig_version = null, .buf = .{}, }; @@ -151,8 +145,6 @@ const Parse = struct { dependencies_node: Ast.Node.OptionalIndex, paths: std.StringArrayHashMapUnmanaged(void), allow_missing_paths_field: bool, - allow_name_string: bool, - allow_missing_fingerprint: bool, minimum_zig_version: ?std.SemanticVersion, const InnerError = error{ ParseFailure, OutOfMemory }; @@ -221,12 +213,10 @@ const Parse = struct { }); } p.id = n.id; - } else if (!p.allow_missing_fingerprint) { + } else { try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{ Package.Fingerprint.generate(rng, p.name).int(), }); - } else { - p.id = 0; } } @@ -395,19 +385,6 @@ const Parse = struct { const ast = p.ast; const main_token = ast.nodeMainToken(node); - if (p.allow_name_string and ast.nodeTag(node) == .string_literal) { - const name = try parseString(p, node); - if (!std.zig.isValidId(name)) - return fail(p, main_token, "name must be a valid bare zig identifier (hint: switch from string to enum literal)", .{}); - - if (name.len > max_name_len) - return fail(p, main_token, "name '{f}' exceeds max length of {d}", .{ - std.zig.fmtId(name), max_name_len, - }); - - return name; - } - if (ast.nodeTag(node) != .enum_literal) return fail(p, main_token, "expected enum literal", .{}); @@ -606,7 +583,8 @@ test "basic" { const example = \\.{ - \\ .name = "foo", + \\ .name = .foo, + \\ .fingerprint = 0x8c736521490b23df, \\ .version = "3.2.1", \\ .paths = .{""}, \\ .dependencies = .{ @@ -656,7 +634,8 @@ test "minimum_zig_version" { const example = \\.{ - \\ .name = "foo", + \\ .name = .foo, + \\ .fingerprint = 0x8c736521490b23df, \\ .version = "3.2.1", \\ .paths = .{""}, \\ .minimum_zig_version = "0.11.1", @@ -690,7 +669,8 @@ test "minimum_zig_version - invalid version" { const example = \\.{ - \\ .name = "foo", + \\ .name = .foo, + \\ .fingerprint = 0x8c736521490b23df, \\ .version = "3.2.1", \\ .minimum_zig_version = "X.11.1", \\ .paths = .{""}, diff --git a/src/main.zig b/src/main.zig index 02a49ef3e0..4ef99de0ae 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5285,8 +5285,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, .job_queue = &job_queue, .omit_missing_hash_error = true, .allow_missing_paths_field = false, - .allow_missing_fingerprint = false, - .allow_name_string = false, .use_latest_commit = false, .package_root = undefined, @@ -7044,8 +7042,6 @@ fn cmdFetch( .job_queue = &job_queue, .omit_missing_hash_error = true, .allow_missing_paths_field = false, - .allow_missing_fingerprint = true, - .allow_name_string = true, .use_latest_commit = true, .package_root = undefined, From ee21a1f988f05a5d45bcb1724095c27cc2c7259b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 4 Feb 2026 21:40:06 -0800 Subject: [PATCH 5/8] fetch: implement recompression After fetching a package and applying the filter by deleting files that are not part of the hash, creates a recompressed $GLOBAL_CACHE/p/$PKG_HASH.tar.gz Checking this cache before fetching network URLs is not yet implemented. --- lib/std/Io.zig | 6 ++ lib/std/compress/flate/Compress.zig | 2 +- src/Package/Fetch.zig | 142 +++++++++++++++++++++++++--- 3 files changed, 136 insertions(+), 14 deletions(-) diff --git a/lib/std/Io.zig b/lib/std/Io.zig index 47ba7c2072..c56ec68866 100644 --- a/lib/std/Io.zig +++ b/lib/std/Io.zig @@ -1031,6 +1031,9 @@ pub const Group = struct { /// Once this function is called, there are resources associated with the /// group. To release those resources, `Group.await` or `Group.cancel` must /// eventually be called. + /// + /// If `error.Canceled` is returned from any operation this task performs, + /// it is asserted that `function` returns `error.Canceled`. pub fn async(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void { const Args = @TypeOf(args); const TypeErased = struct { @@ -1050,6 +1053,9 @@ pub const Group = struct { /// Once this function is called, there are resources associated with the /// group. To release those resources, `Group.await` or `Group.cancel` must /// eventually be called. + /// + /// If `error.Canceled` is returned from any operation this task performs, + /// it is asserted that `function` returns `error.Canceled`. pub fn concurrent(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) ConcurrentError!void { const Args = @TypeOf(args); const TypeErased = struct { diff --git a/lib/std/compress/flate/Compress.zig b/lib/std/compress/flate/Compress.zig index 41b7d8bf04..0a85dd9d0a 100644 --- a/lib/std/compress/flate/Compress.zig +++ b/lib/std/compress/flate/Compress.zig @@ -267,7 +267,7 @@ pub const Options = struct { pub const best = level_9; }; -/// It is asserted `buffer` is least `flate.max_history_len` bytes. +/// It is asserted `buffer` is least `flate.max_window_len` bytes. /// It is asserted `output` has a capacity of at least 8 bytes. pub fn init( output: *Writer, diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index a9cc86b398..e9e340c099 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1,28 +1,34 @@ //! Represents one independent job whose responsibility is to: //! -//! 1. Check the global zig package cache to see if the hash already exists. +//! 1. Check the local zig package directory to see if the hash already exists. //! If so, load, parse, and validate the build.zig.zon file therein, and -//! goto step 8. Likewise if the location is a relative path, treat this +//! goto step 9. Likewise if the location is a relative path, treat this //! the same as a cache hit. Otherwise, proceed. -//! 2. Fetch and unpack a URL into a temporary directory. -//! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed +//! 2. Check the global package cache for a compressed tarball matching the +//! hash. If it is found, unpack the contents into a temporary directory inside +//! project local zig cache. Rename this directory into the local zig package +//! directory and goto step 9, skipping step 10. +//! 3. Fetch and unpack a URL into a temporary directory. +//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed //! for the file to be missing, in which case this fetched package is considered //! to be a "naked" package. -//! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by +//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by //! deleting excluded files. If any files had errors for files that were //! ultimately excluded, those errors should be ignored, such as failure to //! create symlinks that weren't supposed to be included anyway. -//! 5. Compute the package hash based on the remaining files in the temporary +//! 6. Compute the package hash based on the remaining files in the temporary //! directory. -//! 6. Rename the temporary directory into the global zig package cache -//! directory. If the hash already exists, delete the temporary directory and -//! leave the zig package cache directory untouched as it may be in use by the -//! system. This is done even if the hash is invalid, in case the package with -//! the different hash is used in the future. -//! 7. Validate the computed hash against the expected hash. If invalid, +//! 7. Rename the temporary directory into the local zig package directory. If +//! the hash already exists, delete the temporary directory and leave the zig +//! package directory untouched as it may be in use. This is done even if +//! the hash is invalid, in case the package with the different hash is used +//! in the future. +//! 8. Validate the computed hash against the expected hash. If invalid, //! this job is done. -//! 8. Spawn a new fetch job for each dependency in the manifest file. Use +//! 9. Spawn a new fetch job for each dependency in the manifest file. Use //! a mutex and a hash map so that redundant jobs do not get queued up. +//! 10.Compress the package directory and store it into the global package +//! cache. //! //! All of this must be done with only referring to the state inside this struct //! because this work will be done in a dedicated thread. @@ -110,6 +116,7 @@ pub const JobQueue = struct { all_fetches: std.ArrayList(*Fetch) = .empty, http_client: *std.http.Client, + /// This tracks `Fetch` tasks as well as recompression tasks. group: Io.Group = .init, global_cache: Cache.Directory, local_cache: Cache.Path, @@ -293,8 +300,109 @@ pub const JobQueue = struct { \\ ); } + + fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void { + var dest_sub_path_buffer: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined; + const dest_path: Cache.Path = .{ + .root_dir = jq.global_cache, + .sub_path = std.fmt.bufPrint(&dest_sub_path_buffer, "p/{s}.tar.gz", .{ + package_hash.toSlice(), + }) catch unreachable, + }; + + const gpa = jq.http_client.allocator; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + recompressFallible(jq, arena, dest_path, package_hash.toSlice()) catch |err| switch (err) { + error.Canceled => |e| return e, + error.ReadFailed => comptime unreachable, + error.WriteFailed => comptime unreachable, + else => |e| std.log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }), + }; + } + + fn recompressFallible(jq: *JobQueue, arena: Allocator, dest_path: Cache.Path, package_hash: []const u8) !void { + const gpa = jq.http_client.allocator; + const io = jq.io; + + // We have to walk the file system up front in order to sort the file + // list for determinism purposes. The hash of the recompressed file is + // not critical because the true hash is based on the content alone. + // However, if we want Zig users to be able to share cached package + // data with each other via peer-to-peer protocols, we benefit greatly + // from the data being identical on everyone's computers. + var scanned_files: std.ArrayList([]const u8) = .empty; + defer scanned_files.deinit(gpa); + + var pkg_dir = try jq.root_pkg_path.openDir(io, package_hash, .{ .iterate = true }); + defer pkg_dir.close(io); + + { + var walker = try pkg_dir.walk(gpa); + defer walker.deinit(); + + while (try walker.next(io)) |entry| { + switch (entry.kind) { + .directory => continue, + .file, .sym_link => {}, + else => { + return error.IllegalFileType; + }, + } + const entry_path = try arena.dupe(u8, entry.path); + try scanned_files.append(gpa, entry_path); + } + + std.mem.sortUnstable([]const u8, scanned_files.items, {}, stringCmp); + } + + var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{ + .make_path = true, + .replace = true, + }); + defer atomic_file.deinit(io); + + var file_write_buffer: [4096]u8 = undefined; + var file_writer = atomic_file.file.writer(io, &file_write_buffer); + + var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined; + var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch |err| switch (err) { + error.WriteFailed => return file_writer.err.?, + }; + + var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer }; + archiver.prefix = package_hash; + + var file_read_buffer: [4096]u8 = undefined; + + for (scanned_files.items) |entry_path| { + var file = try pkg_dir.openFile(io, entry_path, .{}); + defer file.close(io); + var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer); + archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) { + error.ReadFailed => return file_reader.err.?, + error.WriteFailed => return file_writer.err.?, + else => |e| return e, + }; + } + + // intentionally omitting the pointless trailer + //try archiver.finish(); + compress.writer.flush() catch |err| switch (err) { + error.WriteFailed => return file_writer.err.?, + }; + try file_writer.flush(); + try atomic_file.replace(io); + } }; +fn stringCmp(_: void, lhs: []const u8, rhs: []const u8) bool { + return std.mem.lessThan(u8, lhs, rhs); +} + pub const Location = union(enum) { remote: Remote, /// A directory found inside the parent package. @@ -477,8 +585,11 @@ fn runResource( remote_hash: ?Package.Hash, ) RunError!void { const job_queue = f.job_queue; + assert(!job_queue.read_only); + const io = job_queue.io; defer resource.deinit(io); + const arena = f.arena.allocator(); const eb = &f.error_bundle; const s = fs.path.sep_str; @@ -556,6 +667,11 @@ fn runResource( ) }); return error.FetchFailed; }; + + // Spin off a task to recompress the tarball, with filtered files deleted, into + // the global cache. + job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash }); + // Remove temporary directory root if not already renamed to global cache. if (!package_sub_path.eql(tmp_directory_path)) { tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) { From 7246eee1e706b142abf8183e960fd692fde52bb0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 5 Feb 2026 16:44:37 -0800 Subject: [PATCH 6/8] std.Progress: add Node.startFmt convenience method for starting a child node with a formatted string as a name. --- lib/std/Progress.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/std/Progress.zig b/lib/std/Progress.zig index 2240f95fdd..aee7602bf5 100644 --- a/lib/std/Progress.zig +++ b/lib/std/Progress.zig @@ -325,6 +325,12 @@ pub const Node = struct { return init(@enumFromInt(free_index), parent, name, estimated_total_items); } + pub fn startFmt(node: Node, estimated_total_items: usize, comptime format: []const u8, args: anytype) Node { + var buffer: [max_name_len]u8 = undefined; + const name = std.fmt.bufPrint(&buffer, format, args) catch &buffer; + return Node.start(node, name, estimated_total_items); + } + /// This is the same as calling `start` and then `end` on the returned `Node`. Thread-safe. pub fn completeOne(n: Node) void { const index = n.index.unwrap() orelse return; From 1f65e7cccc3124ce2fa9a5e8107c32367a39ec29 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 5 Feb 2026 16:46:20 -0800 Subject: [PATCH 7/8] fetch: recompress task integrates with std.Progress --- src/Package/Fetch.zig | 30 ++++++++++++++++++++++-------- src/main.zig | 2 ++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index e9e340c099..7d69265f5c 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -114,6 +114,7 @@ pub const JobQueue = struct { /// field contains references to all of them. /// Protected by `mutex`. all_fetches: std.ArrayList(*Fetch) = .empty, + prog_node: std.Progress.Node, http_client: *std.http.Client, /// This tracks `Fetch` tasks as well as recompression tasks. @@ -302,12 +303,15 @@ pub const JobQueue = struct { } fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void { - var dest_sub_path_buffer: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined; + const pkg_hash_slice = package_hash.toSlice(); + + const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice}); + defer prog_node.end(); + + var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined; const dest_path: Cache.Path = .{ .root_dir = jq.global_cache, - .sub_path = std.fmt.bufPrint(&dest_sub_path_buffer, "p/{s}.tar.gz", .{ - package_hash.toSlice(), - }) catch unreachable, + .sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable, }; const gpa = jq.http_client.allocator; @@ -316,7 +320,7 @@ pub const JobQueue = struct { defer arena_instance.deinit(); const arena = arena_instance.allocator(); - recompressFallible(jq, arena, dest_path, package_hash.toSlice()) catch |err| switch (err) { + recompressFallible(jq, arena, dest_path, pkg_hash_slice, prog_node) catch |err| switch (err) { error.Canceled => |e| return e, error.ReadFailed => comptime unreachable, error.WriteFailed => comptime unreachable, @@ -324,7 +328,13 @@ pub const JobQueue = struct { }; } - fn recompressFallible(jq: *JobQueue, arena: Allocator, dest_path: Cache.Path, package_hash: []const u8) !void { + fn recompressFallible( + jq: *JobQueue, + arena: Allocator, + dest_path: Cache.Path, + pkg_hash_slice: []const u8, + prog_node: std.Progress.Node, + ) !void { const gpa = jq.http_client.allocator; const io = jq.io; @@ -337,7 +347,7 @@ pub const JobQueue = struct { var scanned_files: std.ArrayList([]const u8) = .empty; defer scanned_files.deinit(gpa); - var pkg_dir = try jq.root_pkg_path.openDir(io, package_hash, .{ .iterate = true }); + var pkg_dir = try jq.root_pkg_path.openDir(io, pkg_hash_slice, .{ .iterate = true }); defer pkg_dir.close(io); { @@ -359,6 +369,8 @@ pub const JobQueue = struct { std.mem.sortUnstable([]const u8, scanned_files.items, {}, stringCmp); } + prog_node.setEstimatedTotalItems(scanned_files.items.len); + var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{ .make_path = true, .replace = true, @@ -374,7 +386,7 @@ pub const JobQueue = struct { }; var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer }; - archiver.prefix = package_hash; + archiver.prefix = pkg_hash_slice; var file_read_buffer: [4096]u8 = undefined; @@ -387,6 +399,7 @@ pub const JobQueue = struct { error.WriteFailed => return file_writer.err.?, else => |e| return e, }; + prog_node.completeOne(); } // intentionally omitting the pointless trailer @@ -2259,6 +2272,7 @@ const TestFetchBuilder = struct { .read_only = false, .debug_hash = false, .mode = .needed, + .prog_node = std.Progress.Node.none, }; self.fetch = .{ diff --git a/src/main.zig b/src/main.zig index 4ef99de0ae..dba52807f5 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5246,6 +5246,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8, .debug_hash = false, .unlazy_set = unlazy_set, .mode = fetch_mode, + .prog_node = fetch_prog_node, }; defer job_queue.deinit(); @@ -7026,6 +7027,7 @@ fn cmdFetch( .read_only = false, .debug_hash = debug_hash, .mode = .all, + .prog_node = root_prog_node, }; defer job_queue.deinit(); From d8171e8a2ee56e76bcd91f187d5ca5664b87bc83 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 5 Feb 2026 17:36:14 -0800 Subject: [PATCH 8/8] fetch: check global cache for compressed tarball before remote URL --- src/Package/Fetch.zig | 72 ++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 7d69265f5c..d873fc9bd9 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -40,6 +40,7 @@ const native_os = builtin.os.tag; const std = @import("std"); const Io = std.Io; const fs = std.fs; +const log = std.log.scoped(.fetch); const assert = std.debug.assert; const ascii = std.ascii; const Allocator = std.mem.Allocator; @@ -324,7 +325,7 @@ pub const JobQueue = struct { error.Canceled => |e| return e, error.ReadFailed => comptime unreachable, error.WriteFailed => comptime unreachable, - else => |e| std.log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }), + else => |e| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }), }; } @@ -508,14 +509,14 @@ pub fn run(f: *Fetch) RunError!void { .path_or_url => |path_or_url| { if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) |dir| { var resource: Resource = .{ .dir = dir }; - return f.runResource(path_or_url, &resource, null); + return f.runResource(path_or_url, &resource, null, false); } else |dir_err| { var server_header_buffer: [init_resource_buffer_size]u8 = undefined; const file_err = if (dir_err == error.NotDir) e: { if (Io.Dir.cwd().openFile(io, path_or_url, .{})) |file| { var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) }; - return f.runResource(path_or_url, &resource, null); + return f.runResource(path_or_url, &resource, null, false); } else |err| break :e err; } else dir_err; @@ -527,11 +528,13 @@ pub fn run(f: *Fetch) RunError!void { }; var resource: Resource = undefined; try f.initResource(uri, &resource, &server_header_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false); } }, }; + var resource_buffer: [init_resource_buffer_size]u8 = undefined; + if (remote.hash) |expected_hash| { const package_root = try job_queue.root_pkg_path.join(arena, expected_hash.toSlice()); if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| { @@ -543,19 +546,13 @@ pub fn run(f: *Fetch) RunError!void { return queueJobsForDeps(f); } else |err| switch (err) { error.FileNotFound => { - switch (f.lazy_status) { - .eager => {}, - .available => if (!job_queue.unlazy_set.contains(expected_hash)) { - f.lazy_status = .unavailable; - return; - }, - .unavailable => unreachable, - } + log.debug("FileNotFound: {f}", .{package_root}); if (job_queue.read_only) return f.fail( f.name_tok, try eb.printString("package not found at '{f}'", .{package_root}), ); }, + error.Canceled => |e| return e, else => |e| { try eb.addRootErrorMessage(.{ .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{ @@ -565,6 +562,38 @@ pub fn run(f: *Fetch) RunError!void { return error.FetchFailed; }, } + + // Check global cache before remote fetch. + const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()}); + const cached_tarball_path: Cache.Path = .{ + .root_dir = job_queue.global_cache, + .sub_path = cached_tarball_sub_path, + }; + if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) |file| { + log.debug("found global cached tarball {f}", .{cached_tarball_path}); + var resource: Resource = .{ .file = file.reader(io, &resource_buffer) }; + return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true); + } else |err| switch (err) { + error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}), + error.Canceled => |e| return e, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open globally cached package {f}: {t}", .{ + cached_tarball_path, e, + }), + }); + return error.FetchFailed; + }, + } + + switch (f.lazy_status) { + .eager => {}, + .available => if (!job_queue.unlazy_set.contains(expected_hash)) { + f.lazy_status = .unavailable; + return; + }, + .unavailable => unreachable, + } } else if (job_queue.read_only) { try eb.addRootErrorMessage(.{ .msg = try eb.addString("dependency is missing hash field"), @@ -574,15 +603,13 @@ pub fn run(f: *Fetch) RunError!void { } // Fetch and unpack the remote into a temporary directory. - const uri = std.Uri.parse(remote.url) catch |err| return f.fail( f.location_tok, try eb.printString("invalid URI: {t}", .{err}), ); - var buffer: [init_resource_buffer_size]u8 = undefined; var resource: Resource = undefined; - try f.initResource(uri, &resource, &buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash); + try f.initResource(uri, &resource, &resource_buffer); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false); } pub fn deinit(f: *Fetch) void { @@ -596,6 +623,7 @@ fn runResource( uri_path: []const u8, resource: *Resource, remote_hash: ?Package.Hash, + disable_recompress: bool, ) RunError!void { const job_queue = f.job_queue; assert(!job_queue.read_only); @@ -681,15 +709,17 @@ fn runResource( return error.FetchFailed; }; - // Spin off a task to recompress the tarball, with filtered files deleted, into - // the global cache. - job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash }); + if (!disable_recompress) { + // Spin off a task to recompress the tarball, with filtered files deleted, into + // the global cache. + job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash }); + } // Remove temporary directory root if not already renamed to global cache. if (!package_sub_path.eql(tmp_directory_path)) { tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) { error.Canceled => |e| return e, - else => |e| std.log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_directory_path, e }), + else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_directory_path, e }), }; } @@ -1588,7 +1618,7 @@ pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) ! error.Canceled => |e| return e, // Garbage files leftover in zig-cache/tmp/ is, as they say // on Star Trek, "operating within normal parameters". - else => |e| std.log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }), + else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }), }; }, else => |e| return e,