Mirrors/zig
1
0
Fork 0
mirror of https://codeberg.org/ziglang/zig.git synced 2026-04-26 13:01:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request 'package fetching fixes and enhancements' (#31992) from fetch-enhancements into master

Browse Source 
Reviewed-on: https://codeberg.org/ziglang/zig/pulls/31992
This commit is contained in:
Andrew Kelley
2026-04-21 18:06:13 +02:00
parent 3252a05531 74b56501b3
commit b5d8966e05
3 changed files with 157 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/std/zig.zig
+1
View File
@@ -744,6 +744,7 @@ pub fn parseTargetQueryOrReportFatalError(
pub const EnvVar = enum {
ZIG_GLOBAL_CACHE_DIR,
ZIG_LOCAL_CACHE_DIR,
ZIG_LOCAL_PKG_DIR,
ZIG_LIB_DIR,
ZIG_LIBC,
ZIG_BUILD_RUNNER,
src/Package/Fetch.zig
+81 -69
View File
@@ -56,6 +56,9 @@ location_tok: std.zig.Ast.TokenIndex,
hash_tok: std.zig.Ast.OptionalTokenIndex,
name_tok: std.zig.Ast.TokenIndex,
lazy_status: LazyStatus,
/// Same as `parent_packge_root` except it is unchanged when recursing into
/// relative file paths (as opposed to URL).
remote_package_root: Cache.Path,
parent_package_root: Cache.Path,
parent_manifest_ast: ?*const std.zig.Ast,
prog_node: std.Progress.Node,
@@ -104,6 +107,12 @@ pub const LazyStatus = enum {
unavailable,
};
pub const LocalStorage = struct {
cache_root: Cache.Path,
/// Path to "zig-pkg" inside the package in which the user ran `zig build`.
pkg_root: Cache.Path,
};
/// Contains shared state among all `Fetch` tasks.
pub const JobQueue = struct {
io: Io,
@@ -122,9 +131,8 @@ pub const JobQueue = struct {
/// This tracks `Fetch` tasks as well as recompression tasks.
group: Io.Group = .init,
global_cache: Cache.Directory,
local_cache: Cache.Path,
/// Path to "zig-pkg" inside the package in which the user ran `zig build`.
root_pkg_path: Cache.Path,
/// If `null`, indicates fetch globally only.
local_storage: ?*const LocalStorage,
/// If true then, no fetching occurs, and:
/// * The `global_cache` directory is assumed to be the direct parent
/// directory of on-disk packages rather than having the "p/" directory
@@ -341,7 +349,7 @@ pub const JobQueue = struct {
);
}
fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void {
fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Cache.Path) Io.Cancelable!void {
const pkg_hash_slice = package_hash.toSlice();
const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});
@@ -359,7 +367,7 @@ pub const JobQueue = struct {
defer arena_instance.deinit();
const arena = arena_instance.allocator();
recompressFallible(jq, arena, dest_path, pkg_hash_slice, prog_node) catch |err| switch (err) {
recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch |err| switch (err) {
error.Canceled => |e| return e,
error.ReadFailed => comptime unreachable,
error.WriteFailed => comptime unreachable,
@@ -372,6 +380,7 @@ pub const JobQueue = struct {
arena: Allocator,
dest_path: Cache.Path,
pkg_hash_slice: []const u8,
package_root: Cache.Path,
prog_node: std.Progress.Node,
) !void {
const gpa = jq.http_client.allocator;
@@ -386,7 +395,7 @@ pub const JobQueue = struct {
var scanned_files: std.ArrayList(ScannedFile) = .empty;
defer scanned_files.deinit(gpa);
var pkg_dir = try jq.root_pkg_path.openDir(io, pkg_hash_slice, .{ .iterate = true });
var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true });
defer pkg_dir.close(io);
{
@@ -513,7 +522,6 @@ pub fn run(f: *Fetch) RunError!void {
const eb = &f.error_bundle;
const arena = f.arena.allocator();
const gpa = f.arena.child_allocator;
const local_cache_root = job_queue.local_cache;
try eb.init(gpa);
@@ -534,32 +542,19 @@ pub fn run(f: *Fetch) RunError!void {
);
// Packages fetched by URL may not use relative paths to escape outside the
// fetched package directory from within the package cache.
if (pkg_root.root_dir.eql(local_cache_root.root_dir)) {
// `parent_package_root.sub_path` contains a path like this:
// "p/$hash", or
// "p/$hash/foo", with possibly more directories after "foo".
// We want to fail unless the resolved relative path has a
// prefix of "p/$hash/".
const prefix_len: usize = if (job_queue.read_only) 0 else "p/".len;
const parent_sub_path = f.parent_package_root.sub_path;
const end = find_end: {
if (parent_sub_path.len > prefix_len) {
// Use `isSep` instead of `indexOfScalarPos` to account for
// Windows accepting both `\` and `/` as path separators.
for (parent_sub_path[prefix_len..], prefix_len..) |c, i| {
if (std.fs.path.isSep(c)) break :find_end i;
}
}
break :find_end parent_sub_path.len;
};
const expected_prefix = parent_sub_path[0..end];
if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {
return f.fail(
f.location_tok,
try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
);
}
}
// This code path is only reachable recursively and the sub_path
// will already have been resolved to no longer have extra ".." or
// "." components.
assert(job_queue.local_storage != null);
log.debug("checking pkg root \"{s}\" against parent package root \"{s}\"", .{
pkg_root.sub_path, f.remote_package_root.sub_path,
});
assert(pkg_root.root_dir.eql(f.remote_package_root.root_dir));
if (!std.mem.startsWith(u8, pkg_root.sub_path, f.remote_package_root.sub_path)) return f.fail(
f.location_tok,
try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
);
f.package_root = pkg_root;
try loadManifest(f, pkg_root);
if (!f.has_build_zig) try checkBuildFileExistence(f);
@@ -602,6 +597,7 @@ pub fn run(f: *Fetch) RunError!void {
log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name });
fork.uses += 1;
f.package_root = fork.path;
f.remote_package_root = f.package_root;
f.manifest_ast = fork.manifest_ast;
f.manifest = fork.manifest;
f.have_manifest = true;
@@ -610,31 +606,34 @@ pub fn run(f: *Fetch) RunError!void {
return queueJobsForDeps(f);
}
const package_root = try job_queue.root_pkg_path.join(arena, expected_hash.toSlice());
if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| {
assert(f.lazy_status != .unavailable);
f.package_root = package_root;
try loadManifest(f, f.package_root);
try checkBuildFileExistence(f);
if (!job_queue.recursive) return;
return queueJobsForDeps(f);
} else |err| switch (err) {
error.FileNotFound => {
log.debug("FileNotFound: {f}", .{package_root});
if (job_queue.read_only and f.lazy_status == .eager) return f.fail(
f.name_tok,
try eb.printString("package not found at '{f}'", .{package_root}),
);
},
error.Canceled => |e| return e,
else => |e| {
try eb.addRootErrorMessage(.{
.msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
package_root, e,
}),
});
return error.FetchFailed;
},
if (job_queue.local_storage) |ls| {
const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice());
if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| {
assert(f.lazy_status != .unavailable);
f.package_root = package_root;
f.remote_package_root = f.package_root;
try loadManifest(f, f.package_root);
try checkBuildFileExistence(f);
if (!job_queue.recursive) return;
return queueJobsForDeps(f);
} else |err| switch (err) {
error.FileNotFound => {
log.debug("FileNotFound: {f}", .{package_root});
if (job_queue.read_only and f.lazy_status == .eager) return f.fail(
f.name_tok,
try eb.printString("package not found at '{f}'", .{package_root}),
);
},
error.Canceled => |e| return e,
else => |e| {
try eb.addRootErrorMessage(.{
.msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
package_root, e,
}),
});
return error.FetchFailed;
},
}
}
// Check global cache before remote fetch.
@@ -713,7 +712,14 @@ fn runResource(
break :r x;
};
const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int);
const tmp_directory_path = try job_queue.root_pkg_path.join(arena, tmp_dir_sub_path);
const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path;
const tmp_directory_path: Cache.Path = if (job_queue.local_storage) |ls|
try ls.pkg_root.join(arena, tmp_dir_sub_path)
else
.{
.root_dir = job_queue.global_cache,
.sub_path = tmp_tmp_dir_sub_path,
};
const package_sub_path = blk: {
var tmp_directory: Cache.Directory = .{
@@ -772,19 +778,24 @@ fn runResource(
// zig package directory untouched as it may be in use. This is done even
// if the hash is invalid, in case the package with the different hash is
// used in the future.
f.package_root = try job_queue.root_pkg_path.join(arena, computed_package_hash.toSlice());
renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| {
try eb.addRootErrorMessage(.{ .msg = try eb.printString(
"unable to rename temporary directory {f} into package cache directory {f}: {t}",
.{ package_sub_path, f.package_root, err },
) });
return error.FetchFailed;
};
if (job_queue.local_storage) |ls| {
f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice());
renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| {
try eb.addRootErrorMessage(.{ .msg = try eb.printString(
"unable to rename temporary directory {f} into package cache directory {f}: {t}",
.{ package_sub_path, f.package_root, err },
) });
return error.FetchFailed;
};
} else {
f.package_root = tmp_directory_path;
}
f.remote_package_root = f.package_root;
if (!disable_recompress) {
// Spin off a task to recompress the tarball, with filtered files deleted, into
// the global cache.
job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash });
job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root });
}
// Remove temporary directory root if not already renamed to global cache.
@@ -991,6 +1002,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
.all => .eager,
},
.parent_package_root = f.package_root,
.remote_package_root = f.remote_package_root,
.parent_manifest_ast = &f.manifest_ast,
.prog_node = f.prog_node,
.job_queue = f.job_queue,
@@ -1185,7 +1197,7 @@ fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u
if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
const path = try uri.path.toRawMaybeAlloc(arena);
const file = f.parent_package_root.openFile(io, path, .{}) catch |err| {
return f.fail(f.location_tok, try eb.printString("unable to open '{f}{s}': {t}", .{
return f.fail(f.location_tok, try eb.printString("unable to open {f}/{s}: {t}", .{
f.parent_package_root, path, err,
}));
};
src/main.zig
+75 -45
View File
@@ -1359,12 +1359,7 @@ fn buildOutputType(
} else if (mem.eql(u8, arg, "--zig-lib-dir")) {
override_lib_dir = args_iter.nextOrFatal();
} else if (mem.eql(u8, arg, "--debug-log")) {
if (!build_options.enable_logging) {
warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{});
_ = args_iter.nextOrFatal();
} else {
try log_scopes.append(arena, args_iter.nextOrFatal());
}
try addDebugLog(arena, args_iter.nextOrFatal());
} else if (mem.eql(u8, arg, "--listen")) {
const next_arg = args_iter.nextOrFatal();
if (mem.eql(u8, next_arg, "-")) {
@@ -4958,6 +4953,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
var override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map);
var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);
var override_build_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map);
var child_argv: std.ArrayList([]const u8) = .empty;
var forks: std.ArrayList(Fork) = .empty;
@@ -5048,6 +5044,11 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
i += 1;
override_local_cache_dir = args[i];
continue;
} else if (mem.eql(u8, arg, "--pkg-dir")) {
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
i += 1;
override_pkg_dir = args[i];
continue;
} else if (mem.eql(u8, arg, "--global-cache-dir")) {
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
i += 1;
@@ -5092,11 +5093,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
try child_argv.appendSlice(arena, args[i .. i + 2]);
i += 1;
if (!build_options.enable_logging) {
warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{});
} else {
try log_scopes.append(arena, args[i]);
}
try addDebugLog(arena, args[i]);
continue;
} else if (mem.eql(u8, arg, "--debug-compile-errors")) {
if (build_options.enable_debug_extensions) {
@@ -5325,9 +5322,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
.parent = root_mod,
});
var cleanup_build_dir: ?Io.Dir = null;
defer if (cleanup_build_dir) |*dir| dir.close(io);
if (dev.env.supports(.fetch_command)) {
const fetch_prog_node = root_prog_node.start("Fetch Packages", 0);
defer fetch_prog_node.end();
@@ -5339,33 +5333,29 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
.io = io,
.http_client = &http_client,
.global_cache = dirs.global_cache,
.local_cache = .{ .root_dir = dirs.local_cache, .sub_path = "" },
.root_pkg_path = .{ .root_dir = build_root.directory, .sub_path = "zig-pkg" },
.read_only = false,
.local_storage = &.{
.cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" },
.pkg_root = if (override_pkg_dir) |p|
.initCwd(p)
else if (system_pkg_dir_path) |p|
.initCwd(p)
else
.{
.root_dir = build_root.directory,
.sub_path = "zig-pkg",
},
},
.recursive = true,
.debug_hash = false,
.unlazy_set = unlazy_set,
.fork_set = fork_set,
.mode = fetch_mode,
.prog_node = fetch_prog_node,
.read_only = system_pkg_dir_path != null,
};
defer job_queue.deinit();
if (system_pkg_dir_path) |p| {
const system_pkg_path: Path = .{
.root_dir = .{
.path = p,
.handle = Io.Dir.cwd().openDir(io, p, .{}) catch |err| {
fatal("unable to open system package directory '{s}': {t}", .{ p, err });
},
},
.sub_path = "",
};
job_queue.global_cache = system_pkg_path.root_dir;
job_queue.root_pkg_path = system_pkg_path;
job_queue.read_only = true;
cleanup_build_dir = job_queue.global_cache.handle;
} else {
if (system_pkg_dir_path == null) {
try http_client.initDefaultProxies(arena, environ_map);
}
@@ -5381,6 +5371,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
.hash_tok = .none,
.name_tok = 0,
.lazy_status = .eager,
.remote_package_root = phantom_package_root,
.parent_package_root = phantom_package_root,
.parent_manifest_ast = null,
.prog_node = fetch_prog_node,
@@ -5401,6 +5392,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
.module = build_mod,
};
job_queue.all_fetches.appendAssumeCapacity(&fetch);
job_queue.table.putAssumeCapacityNoClobber(
@@ -7032,7 +7024,10 @@ const usage_fetch =
\\Options:
\\ -h, --help Print this help and exit
\\ --global-cache-dir [path] Override path to global Zig cache directory
\\ --cache-dir [path] Override path to local cache directory
\\ --pkg-dir [path] Override path to local package directory
\\ --debug-hash Print verbose hash information to stdout
\\ --debug-log [scope] Enable printing debug/info log messages for scope
\\ --save Add the fetched package to build.zig.zon
\\ --save=[name] Add the fetched package to build.zig.zon as name
\\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim
@@ -7052,6 +7047,8 @@ fn cmdFetch(
const color: Color = .auto;
var opt_path_or_url: ?[]const u8 = null;
var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);
var debug_hash: bool = false;
var save: union(enum) {
no,
@@ -7068,11 +7065,23 @@ fn cmdFetch(
try Io.File.stdout().writeStreamingAll(io, usage_fetch);
return cleanExit(io);
} else if (mem.eql(u8, arg, "--global-cache-dir")) {
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
i += 1;
override_global_cache_dir = args[i];
} else if (mem.eql(u8, arg, "--cache-dir")) {
if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
i += 1;
override_local_cache_dir = args[i];
} else if (mem.eql(u8, arg, "--pkg-dir")) {
if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
i += 1;
override_pkg_dir = args[i];
} else if (mem.eql(u8, arg, "--debug-hash")) {
debug_hash = true;
} else if (mem.eql(u8, arg, "--debug-log")) {
if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
i += 1;
try addDebugLog(arena, args[i]);
} else if (mem.eql(u8, arg, "--save")) {
save = .{ .yes = null };
} else if (mem.cutPrefix(u8, arg, "--save=")) |rest| {
@@ -7113,27 +7122,39 @@ fn cmdFetch(
};
defer global_cache_directory.handle.close(io);
var local_storage: Package.Fetch.LocalStorage = undefined;
var build_root: BuildRoot = undefined;
var build_root_initialized = false;
defer if (build_root_initialized) build_root.deinit(io);
const cwd_path = try introspect.getResolvedCwd(io, arena);
var build_root = try findBuildRoot(arena, io, .{
.cwd_path = cwd_path,
});
defer build_root.deinit(io);
const local_storage_ptr = switch (save) {
.no => null,
.yes, .exact => ls: {
build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path });
build_root_initialized = true;
const local_cache_path: Path = .{
.root_dir = build_root.directory,
.sub_path = ".zig-cache",
local_storage = .{
.cache_root = if (override_local_cache_dir) |p| .initCwd(p) else .{
.root_dir = build_root.directory,
.sub_path = ".zig-cache",
},
.pkg_root = if (override_pkg_dir) |p| .initCwd(p) else .{
.root_dir = build_root.directory,
.sub_path = "zig-pkg",
},
};
break :ls &local_storage;
},
};
var job_queue: Package.Fetch.JobQueue = .{
.io = io,
.http_client = &http_client,
.global_cache = global_cache_directory,
.local_cache = local_cache_path,
.root_pkg_path = .{
.root_dir = build_root.directory,
.sub_path = "zig-pkg",
},
.local_storage = local_storage_ptr,
.recursive = false,
.read_only = false,
.debug_hash = debug_hash,
@@ -7149,6 +7170,7 @@ fn cmdFetch(
.hash_tok = .none,
.name_tok = 0,
.lazy_status = .eager,
.remote_package_root = undefined,
.parent_package_root = undefined,
.parent_manifest_ast = null,
.prog_node = root_prog_node,
@@ -7793,3 +7815,11 @@ fn randInt(io: Io, comptime T: type) T {
io.random(@ptrCast(&x));
return x;
}
fn addDebugLog(arena: Allocator, scope_name: []const u8) error{OutOfMemory}!void {
if (!build_options.enable_logging) {
warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{});
} else {
try log_scopes.append(arena, scope_name);
}
}