From 908c2c902ada674eae30d81a8fb5b6c43cdfe141 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 5 Jul 2024 12:23:54 -0700 Subject: [PATCH 01/38] std.Build.Cache.Path: add eql method --- lib/std/Build/Cache/Path.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/std/Build/Cache/Path.zig b/lib/std/Build/Cache/Path.zig index 48bb8c32beab..cb6e4c708758 100644 --- a/lib/std/Build/Cache/Path.zig +++ b/lib/std/Build/Cache/Path.zig @@ -147,6 +147,10 @@ pub fn format( } } +pub fn eql(self: Path, other: Path) bool { + return self.root_dir.eql(other.root_dir) and std.mem.eql(u8, self.sub_path, other.sub_path); +} + const Path = @This(); const std = @import("../../std.zig"); const fs = std.fs; From d2bec8f92f15ac16a0714ddc8282ab31dd5bb889 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 5 Jul 2024 12:24:14 -0700 Subject: [PATCH 02/38] delete dead CLI usage code from main.zig --- src/main.zig | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/main.zig b/src/main.zig index c7bbb9883cf4..e00442f399e5 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4649,31 +4649,6 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { return cleanExit(); } -const usage_build = - \\Usage: zig build [steps] [options] - \\ - \\ Build a project from build.zig. - \\ - \\Options: - \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error - \\ -fno-reference-trace Disable reference trace - \\ --summary [mode] Control the printing of the build summary - \\ all Print the build summary in its entirety - \\ failures (Default) Only print failed steps - \\ none Do not print the build summary - \\ -j Limit concurrent jobs (default is to use all CPU cores) - \\ --build-file [file] Override path to build.zig - \\ --cache-dir [path] Override path to local Zig cache directory - \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --zig-lib-dir [arg] Override path to Zig lib directory - \\ --build-runner [file] Override path to build runner - \\ --prominent-compile-errors Buffer compile errors and display at end - \\ --seed [integer] For shuffling dependency traversal order (default: random) - \\ --fetch Exit after fetching dependency tree - \\ -h, --help Print this help and exit - \\ -; - fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { var build_file: ?[]const u8 = null; var override_lib_dir: ?[]const u8 = try EnvVar.ZIG_LIB_DIR.get(arena); From 6e025fc2e298c633ab36e9058a2cc610f57e4522 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 5 Jul 2024 12:24:32 -0700 Subject: [PATCH 03/38] build system: add --watch flag and report source file in InstallFile This direction is not quite right because it mutates shared state in a threaded context, so the next commit will need to fix this. --- lib/compiler/build_runner.zig | 66 ++++++++++++++++++++---------- lib/std/Build.zig | 55 +++++++++++++++++++++++++ lib/std/Build/Step.zig | 46 +++++++++++++++++++++ lib/std/Build/Step/InstallFile.zig | 1 + 4 files changed, 147 insertions(+), 21 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 79585086b317..cf78ad320f4f 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -74,6 +74,7 @@ pub fn main() !void { .query = .{}, .result = try std.zig.system.resolveTargetQuery(.{}), }, + .watch = null, }; graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); @@ -97,12 +98,12 @@ pub fn main() !void { var dir_list = std.Build.DirList{}; var summary: ?Summary = null; var max_rss: u64 = 0; - var skip_oom_steps: bool = false; + var skip_oom_steps = false; var color: Color = .auto; var seed: u32 = 0; - var prominent_compile_errors: bool = false; - var help_menu: bool = false; - var steps_menu: bool = false; + var prominent_compile_errors = false; + var help_menu = false; + var steps_menu = false; var output_tmp_nonce: ?[16]u8 = null; while (nextArg(args, &arg_idx)) |arg| { @@ -227,6 +228,10 @@ pub fn main() !void { builder.verbose_llvm_cpu_features = true; } else if (mem.eql(u8, arg, "--prominent-compile-errors")) { prominent_compile_errors = true; + } else if (mem.eql(u8, arg, "--watch")) { + const watch = try arena.create(std.Build.Watch); + watch.* = std.Build.Watch.init; + graph.watch = watch; } else if (mem.eql(u8, arg, "-fwine")) { builder.enable_wine = true; } else if (mem.eql(u8, arg, "-fno-wine")) { @@ -344,7 +349,7 @@ pub fn main() !void { .prominent_compile_errors = prominent_compile_errors, .claimed_rss = 0, - .summary = summary, + .summary = summary orelse if (graph.watch != null) .new else .failures, .ttyconf = ttyconf, .stderr = stderr, }; @@ -363,7 +368,10 @@ pub fn main() !void { &run, seed, ) catch |err| switch (err) { - error.UncleanExit => process.exit(1), + error.UncleanExit => { + if (graph.watch == null) + process.exit(1); + }, else => return err, }; } @@ -377,7 +385,7 @@ const Run = struct { prominent_compile_errors: bool, claimed_rss: usize, - summary: ?Summary, + summary: Summary, ttyconf: std.io.tty.Config, stderr: File, }; @@ -417,7 +425,7 @@ fn runStepNames( for (starting_steps) |s| { constructGraphAndCheckForDependencyLoop(b, s, &step_stack, rand) catch |err| switch (err) { - error.DependencyLoopDetected => return error.UncleanExit, + error.DependencyLoopDetected => return uncleanExit(), else => |e| return e, }; } @@ -442,7 +450,7 @@ fn runStepNames( if (run.max_rss_is_default) { std.debug.print("note: use --maxrss to override the default", .{}); } - return error.UncleanExit; + return uncleanExit(); } } @@ -524,13 +532,19 @@ fn runStepNames( // A proper command line application defaults to silently succeeding. // The user may request verbose mode if they have a different preference. - const failures_only = run.summary != .all and run.summary != .new; - if (failure_count == 0 and failures_only) return cleanExit(); + const failures_only = switch (run.summary) { + .failures, .none => true, + else => false, + }; + if (failure_count == 0 and failures_only) { + if (b.graph.watch != null) return; + return cleanExit(); + } const ttyconf = run.ttyconf; const stderr = run.stderr; - if (run.summary != Summary.none) { + if (run.summary != .none) { const total_count = success_count + failure_count + pending_count + skipped_count; ttyconf.setColor(stderr, .cyan) catch {}; stderr.writeAll("Build Summary:") catch {}; @@ -544,11 +558,6 @@ fn runStepNames( if (test_fail_count > 0) stderr.writer().print("; {d} failed", .{test_fail_count}) catch {}; if (test_leak_count > 0) stderr.writer().print("; {d} leaked", .{test_leak_count}) catch {}; - if (run.summary == null) { - ttyconf.setColor(stderr, .dim) catch {}; - stderr.writeAll(" (disable with --summary none)") catch {}; - ttyconf.setColor(stderr, .reset) catch {}; - } stderr.writeAll("\n") catch {}; // Print a fancy tree with build results. @@ -562,7 +571,7 @@ fn runStepNames( while (i > 0) { i -= 1; const step = b.top_level_steps.get(step_names[i]).?.step; - const found = switch (run.summary orelse .failures) { + const found = switch (run.summary) { .all, .none => unreachable, .failures => step.state != .success, .new => !step.result_cached, @@ -579,7 +588,10 @@ fn runStepNames( } } - if (failure_count == 0) return cleanExit(); + if (failure_count == 0) { + if (b.graph.watch != null) return; + return cleanExit(); + } // Finally, render compile errors at the bottom of the terminal. // We use a separate compile_error_steps array list because step_stack is destructively @@ -591,13 +603,24 @@ fn runStepNames( } } + if (b.graph.watch != null) return uncleanExit(); + // Signal to parent process that we have printed compile errors. The // parent process may choose to omit the "following command failed" // line in this case. process.exit(2); } - process.exit(1); + return uncleanExit(); +} + +fn uncleanExit() error{UncleanExit}!void { + if (builtin.mode == .Debug) { + return error.UncleanExit; + } else { + std.debug.lockStdErr(); + process.exit(1); + } } const PrintNode = struct { @@ -768,7 +791,7 @@ fn printTreeStep( step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), ) !void { const first = step_stack.swapRemove(s); - const summary = run.summary orelse .failures; + const summary = run.summary; const skip = switch (summary) { .none => unreachable, .all => false, @@ -1124,6 +1147,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void { \\ --maxrss Limit memory usage (default is to use available memory) \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss \\ --fetch Exit after fetching dependency tree + \\ --watch Continuously rebuild when source files are modified \\ \\Project-Specific Options: \\ diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 0255245a3c63..bdfd71891bd0 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -120,6 +120,61 @@ pub const Graph = struct { needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{}, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, + /// When `--watch` is provided, collects the set of files that should be + /// watched and the state to required to poll the system for changes. + watch: ?*Watch, +}; + +pub const Watch = struct { + table: Table, + + pub const init: Watch = .{ + .table = .{}, + }; + + /// Key is the directory to watch which contains one or more files we are + /// interested in noticing changes to. + pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, ReactionSet, TableContext, false); + + const Hash = std.hash.Wyhash; + + pub const TableContext = struct { + pub fn hash(self: TableContext, a: Cache.Path) u32 { + _ = self; + const seed: u32 = @bitCast(a.root_dir.handle.fd); + return @truncate(Hash.hash(seed, a.sub_path)); + } + pub fn eql(self: TableContext, a: Cache.Path, b: Cache.Path, b_index: usize) bool { + _ = self; + _ = b_index; + return a.eql(b); + } + }; + + pub const ReactionSet = std.ArrayHashMapUnmanaged(Match, void, Match.Context, false); + + pub const Match = struct { + /// Relative to the watched directory, the file path that triggers this + /// match. + basename: []const u8, + /// The step to re-run when file corresponding to `basename` is changed. + step: *Step, + + pub const Context = struct { + pub fn hash(self: Context, a: Match) u32 { + _ = self; + var hasher = Hash.init(0); + std.hash.autoHash(&hasher, a.step); + hasher.update(a.basename); + return @truncate(hasher.final()); + } + pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool { + _ = self; + _ = b_index; + return a.step == b.step and mem.eql(u8, a.basename, b.basename); + } + }; + }; }; const AvailableDeps = []const struct { []const u8, []const u8 }; diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 0813aba6897f..91fe29866696 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -562,6 +562,52 @@ pub fn writeManifest(s: *Step, man: *std.Build.Cache.Manifest) !void { } } +fn oom(err: anytype) noreturn { + switch (err) { + error.OutOfMemory => @panic("out of memory"), + } +} + +pub fn addWatchInput(step: *Step, lazy_path: std.Build.LazyPath) void { + errdefer |err| oom(err); + const w = step.owner.graph.watch orelse return; + switch (lazy_path) { + .src_path => |src_path| try addWatchInputFromBuilder(step, w, src_path.owner, src_path.sub_path), + .dependency => |d| try addWatchInputFromBuilder(step, w, d.dependency.builder, d.sub_path), + .cwd_relative => |path_string| { + try addWatchInputFromPath(w, .{ + .root_dir = .{ + .path = null, + .handle = std.fs.cwd(), + }, + .sub_path = std.fs.path.dirname(path_string) orelse "", + }, .{ + .step = step, + .basename = std.fs.path.basename(path_string), + }); + }, + // Nothing to watch because this dependency edge is modeled instead via `dependants`. + .generated => {}, + } +} + +fn addWatchInputFromBuilder(step: *Step, w: *std.Build.Watch, builder: *std.Build, sub_path: []const u8) !void { + return addWatchInputFromPath(w, .{ + .root_dir = builder.build_root, + .sub_path = std.fs.path.dirname(sub_path) orelse "", + }, .{ + .step = step, + .basename = std.fs.path.basename(sub_path), + }); +} + +fn addWatchInputFromPath(w: *std.Build.Watch, path: std.Build.Cache.Path, match: std.Build.Watch.Match) !void { + const gpa = match.step.owner.allocator; + const gop = try w.table.getOrPut(gpa, path); + if (!gop.found_existing) gop.value_ptr.* = .{}; + try gop.value_ptr.put(gpa, match, {}); +} + test { _ = CheckFile; _ = CheckObject; diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index 8202a9d79648..c7f1088c48df 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -40,6 +40,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { _ = prog_node; const b = step.owner; const install_file: *InstallFile = @fieldParentPtr("step", step); + step.addWatchInput(install_file.source); const full_src_path = install_file.source.getPath2(b, step); const full_dest_path = b.getInstallPath(install_file.dir, install_file.dest_rel_path); const cwd = std.fs.cwd(); From 26d506c0f8b5249fe29186506b82e5f515fcc56f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 5 Jul 2024 13:05:56 -0700 Subject: [PATCH 04/38] std.Build: remove the "push installed file" mechanism Tracked by #14943 --- lib/std/Build.zig | 36 ++---------------------------- lib/std/Build/Step/InstallDir.zig | 1 - lib/std/Build/Step/InstallFile.zig | 1 - 3 files changed, 2 insertions(+), 36 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index bdfd71891bd0..bfab90971c78 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -50,7 +50,6 @@ install_path: []const u8, sysroot: ?[]const u8 = null, search_prefixes: std.ArrayListUnmanaged([]const u8), libc_file: ?[]const u8 = null, -installed_files: ArrayList(InstalledFile), /// Path to the directory containing build.zig. build_root: Cache.Directory, cache_root: Cache.Directory, @@ -331,7 +330,6 @@ pub fn create( .exe_dir = undefined, .h_dir = undefined, .dest_dir = graph.env_map.get("DESTDIR"), - .installed_files = ArrayList(InstalledFile).init(arena), .install_tls = .{ .step = Step.init(.{ .id = TopLevelStep.base_id, @@ -433,7 +431,6 @@ fn createChildOnly( .sysroot = parent.sysroot, .search_prefixes = parent.search_prefixes, .libc_file = parent.libc_file, - .installed_files = ArrayList(InstalledFile).init(allocator), .build_root = build_root, .cache_root = parent.cache_root, .zig_lib_dir = parent.zig_lib_dir, @@ -1138,15 +1135,8 @@ fn makeUninstall(uninstall_step: *Step, prog_node: std.Progress.Node) anyerror!v const uninstall_tls: *TopLevelStep = @fieldParentPtr("step", uninstall_step); const b: *Build = @fieldParentPtr("uninstall_tls", uninstall_tls); - for (b.installed_files.items) |installed_file| { - const full_path = b.getInstallPath(installed_file.dir, installed_file.path); - if (b.verbose) { - log.info("rm {s}", .{full_path}); - } - fs.cwd().deleteTree(full_path) catch {}; - } - - // TODO remove empty directories + _ = b; + @panic("TODO implement https://github.com/ziglang/zig/issues/14943"); } /// Creates a configuration option to be passed to the build.zig script. @@ -1719,15 +1709,6 @@ pub fn addCheckFile( return Step.CheckFile.create(b, file_source, options); } -/// deprecated: https://github.com/ziglang/zig/issues/14943 -pub fn pushInstalledFile(b: *Build, dir: InstallDir, dest_rel_path: []const u8) void { - const file = InstalledFile{ - .dir = dir, - .path = dest_rel_path, - }; - b.installed_files.append(file.dupe(b)) catch @panic("OOM"); -} - pub fn truncateFile(b: *Build, dest_path: []const u8) !void { if (b.verbose) { log.info("truncate {s}", .{dest_path}); @@ -2567,19 +2548,6 @@ pub const InstallDir = union(enum) { } }; -pub const InstalledFile = struct { - dir: InstallDir, - path: []const u8, - - /// Duplicates the installed file path and directory. - pub fn dupe(file: InstalledFile, builder: *Build) InstalledFile { - return .{ - .dir = file.dir.dupe(builder), - .path = builder.dupe(file.path), - }; - } -}; - /// This function is intended to be called in the `configure` phase only. /// It returns an absolute directory path, which is potentially going to be a /// source of API breakage in the future, so keep that in mind when using this diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index 0a6edafb338e..ca38e09ec199 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -41,7 +41,6 @@ pub const Options = struct { }; pub fn create(owner: *std.Build, options: Options) *InstallDir { - owner.pushInstalledFile(options.install_dir, options.install_subdir); const install_dir = owner.allocator.create(InstallDir) catch @panic("OOM"); install_dir.* = .{ .step = Step.init(.{ diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index c7f1088c48df..dd6aa2d1c4a6 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -19,7 +19,6 @@ pub fn create( dest_rel_path: []const u8, ) *InstallFile { assert(dest_rel_path.len != 0); - owner.pushInstalledFile(dir, dest_rel_path); const install_file = owner.allocator.create(InstallFile) catch @panic("OOM"); install_file.* = .{ .step = Step.init(.{ From 6c64090e7af56ccbf374e1a927fe232b340ed681 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 8 Jul 2024 23:35:00 -0700 Subject: [PATCH 05/38] std.os.linux: fanotify_init, fanotify_mark, name_to_handle_at * Delete existing `FAN` struct in favor of a `fanotify` struct which has type-safe bindings (breaking). * Add name_to_handle_at syscall wrapper. * Add file_handle * Add kernel_fsid_t * Add fsid_t * Add and update std.posix wrappers. --- lib/std/os/linux.zig | 232 +++++++++++++++++++++++++++++++++---------- lib/std/posix.zig | 60 ++++++++++- 2 files changed, 235 insertions(+), 57 deletions(-) diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 8ef74dec7b1f..91a050ee22b5 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -698,12 +698,42 @@ pub fn inotify_rm_watch(fd: i32, wd: i32) usize { return syscall2(.inotify_rm_watch, @as(usize, @bitCast(@as(isize, fd))), @as(usize, @bitCast(@as(isize, wd)))); } -pub fn fanotify_init(flags: u32, event_f_flags: u32) usize { - return syscall2(.fanotify_init, flags, event_f_flags); +pub fn fanotify_init(flags: fanotify.InitFlags, event_f_flags: u32) usize { + return syscall2(.fanotify_init, @as(u32, @bitCast(flags)), event_f_flags); } -pub fn fanotify_mark(fd: i32, flags: u32, mask: u64, dirfd: i32, pathname: ?[*:0]const u8) usize { - return syscall5(.fanotify_mark, @as(usize, @bitCast(@as(isize, fd))), flags, mask, @as(usize, @bitCast(@as(isize, dirfd))), @intFromPtr(pathname)); +pub fn fanotify_mark( + fd: fd_t, + flags: fanotify.MarkFlags, + mask: fanotify.MarkMask, + dirfd: fd_t, + pathname: ?[*:0]const u8, +) usize { + return syscall5( + .fanotify_mark, + @bitCast(@as(isize, fd)), + @as(u32, @bitCast(flags)), + @bitCast(mask), + @bitCast(@as(isize, dirfd)), + @intFromPtr(pathname), + ); +} + +pub fn name_to_handle_at( + dirfd: fd_t, + pathname: [*:0]const u8, + handle: *std.os.linux.file_handle, + mount_id: *i32, + flags: u32, +) usize { + return syscall5( + .name_to_handle_at, + @as(u32, @bitCast(dirfd)), + @intFromPtr(pathname), + @intFromPtr(handle), + @intFromPtr(mount_id), + flags, + ); } pub fn readlink(noalias path: [*:0]const u8, noalias buf_ptr: [*]u8, buf_len: usize) usize { @@ -4135,58 +4165,156 @@ pub const IN = struct { pub const ONESHOT = 0x80000000; }; -pub const FAN = struct { - pub const ACCESS = 0x00000001; - pub const MODIFY = 0x00000002; - pub const CLOSE_WRITE = 0x00000008; - pub const CLOSE_NOWRITE = 0x00000010; - pub const OPEN = 0x00000020; - pub const Q_OVERFLOW = 0x00004000; - pub const OPEN_PERM = 0x00010000; - pub const ACCESS_PERM = 0x00020000; - pub const ONDIR = 0x40000000; - pub const EVENT_ON_CHILD = 0x08000000; - pub const CLOSE = CLOSE_WRITE | CLOSE_NOWRITE; - pub const CLOEXEC = 0x00000001; - pub const NONBLOCK = 0x00000002; - pub const CLASS_NOTIF = 0x00000000; - pub const CLASS_CONTENT = 0x00000004; - pub const CLASS_PRE_CONTENT = 0x00000008; - pub const ALL_CLASS_BITS = CLASS_NOTIF | CLASS_CONTENT | CLASS_PRE_CONTENT; - pub const UNLIMITED_QUEUE = 0x00000010; - pub const UNLIMITED_MARKS = 0x00000020; - pub const ALL_INIT_FLAGS = CLOEXEC | NONBLOCK | ALL_CLASS_BITS | UNLIMITED_QUEUE | UNLIMITED_MARKS; - pub const MARK_ADD = 0x00000001; - pub const MARK_REMOVE = 0x00000002; - pub const MARK_DONT_FOLLOW = 0x00000004; - pub const MARK_ONLYDIR = 0x00000008; - pub const MARK_MOUNT = 0x00000010; - pub const MARK_IGNORED_MASK = 0x00000020; - pub const MARK_IGNORED_SURV_MODIFY = 0x00000040; - pub const MARK_FLUSH = 0x00000080; - pub const ALL_MARK_FLAGS = MARK_ADD | MARK_REMOVE | MARK_DONT_FOLLOW | MARK_ONLYDIR | MARK_MOUNT | MARK_IGNORED_MASK | MARK_IGNORED_SURV_MODIFY | MARK_FLUSH; - pub const ALL_EVENTS = ACCESS | MODIFY | CLOSE | OPEN; - pub const ALL_PERM_EVENTS = OPEN_PERM | ACCESS_PERM; - pub const ALL_OUTGOING_EVENTS = ALL_EVENTS | ALL_PERM_EVENTS | Q_OVERFLOW; - pub const ALLOW = 0x01; - pub const DENY = 0x02; -}; - -pub const fanotify_event_metadata = extern struct { - event_len: u32, - vers: u8, - reserved: u8, - metadata_len: u16, - mask: u64 align(8), - fd: i32, - pid: i32, +pub const fanotify = struct { + pub const InitFlags = packed struct(u32) { + CLOEXEC: bool = false, + NONBLOCK: bool = false, + CLASS: enum(u2) { + NOTIF = 0, + CONTENT = 1, + PRE_CONTENT = 2, + } = .NOTIF, + UNLIMITED_QUEUE: bool = false, + UNLIMITED_MARKS: bool = false, + ENABLE_AUDIT: bool = false, + REPORT_PIDFD: bool = false, + REPORT_TID: bool = false, + REPORT_FID: bool = false, + REPORT_DIR_FID: bool = false, + REPORT_NAME: bool = false, + REPORT_TARGET_FID: bool = false, + _: u19 = 0, + }; + + pub const MarkFlags = packed struct(u32) { + ADD: bool = false, + REMOVE: bool = false, + DONT_FOLLOW: bool = false, + ONLYDIR: bool = false, + MOUNT: bool = false, + /// Mutually exclusive with `IGNORE` + IGNORED_MASK: bool = false, + IGNORED_SURV_MODIFY: bool = false, + FLUSH: bool = false, + FILESYSTEM: bool = false, + EVICTABLE: bool = false, + /// Mutually exclusive with `IGNORED_MASK` + IGNORE: bool = false, + _: u21 = 0, + }; + + pub const MarkMask = packed struct(u64) { + /// File was accessed + ACCESS: bool = false, + /// File was modified + MODIFY: bool = false, + /// Metadata changed + ATTRIB: bool = false, + /// Writtable file closed + CLOSE_WRITE: bool = false, + /// Unwrittable file closed + CLOSE_NOWRITE: bool = false, + /// File was opened + OPEN: bool = false, + /// File was moved from X + MOVED_FROM: bool = false, + /// File was moved to Y + MOVED_TO: bool = false, + + /// Subfile was created + CREATE: bool = false, + /// Subfile was deleted + DELETE: bool = false, + /// Self was deleted + DELETE_SELF: bool = false, + /// Self was moved + MOVE_SELF: bool = false, + /// File was opened for exec + OPEN_EXEC: bool = false, + reserved13: u1 = 0, + /// Event queued overflowed + Q_OVERFLOW: bool = false, + /// Filesystem error + FS_ERROR: bool = false, + + /// File open in perm check + OPEN_PERM: bool = false, + /// File accessed in perm check + ACCESS_PERM: bool = false, + /// File open/exec in perm check + OPEN_EXEC_PERM: bool = false, + reserved19: u8 = 0, + /// Interested in child events + EVENT_ON_CHILD: bool = false, + /// File was renamed + RENAME: bool = false, + reserved30: u1 = 0, + /// Event occurred against dir + ONDIR: bool = false, + reserved31: u33 = 0, + }; + + pub const event_metadata = extern struct { + event_len: u32, + vers: u8, + reserved: u8, + metadata_len: u16, + mask: u64 align(8), + fd: i32, + pid: i32, + + pub const VERSION = 3; + }; + + pub const response = extern struct { + fd: i32, + response: u32, + }; + + /// Unique file identifier info record. + /// + /// This structure is used for records of types `EVENT_INFO_TYPE.FID`. + /// `EVENT_INFO_TYPE.DFID` and `EVENT_INFO_TYPE.DFID_NAME`. + /// + /// For `EVENT_INFO_TYPE.DFID_NAME` there is additionally a null terminated + /// name immediately after the file handle. + pub const event_info_fid = extern struct { + hdr: event_info_header, + fsid: kernel_fsid_t, + /// Following is an opaque struct file_handle that can be passed as + /// an argument to open_by_handle_at(2). + handle: [0]u8, + }; + + /// Variable length info record following event metadata. + pub const event_info_header = extern struct { + info_type: EVENT_INFO_TYPE, + pad: u8, + len: u16, + }; + + pub const EVENT_INFO_TYPE = enum(u8) { + FID = 1, + DFID_NAME = 2, + DFID = 3, + PIDFD = 4, + ERROR = 5, + OLD_DFID_NAME = 10, + OLD_DFID = 11, + NEW_DFID_NAME = 12, + NEW_DFID = 13, + }; }; -pub const fanotify_response = extern struct { - fd: i32, - response: u32, +pub const file_handle = extern struct { + handle_bytes: u32, + handle_type: i32, + f_handle: [0]u8, }; +pub const kernel_fsid_t = fsid_t; +pub const fsid_t = [2]i32; + pub const S = struct { pub const IFMT = 0o170000; diff --git a/lib/std/posix.zig b/lib/std/posix.zig index d244dad51d10..199b4d117d46 100644 --- a/lib/std/posix.zig +++ b/lib/std/posix.zig @@ -4501,7 +4501,7 @@ pub const FanotifyInitError = error{ PermissionDenied, } || UnexpectedError; -pub fn fanotify_init(flags: u32, event_f_flags: u32) FanotifyInitError!i32 { +pub fn fanotify_init(flags: std.os.linux.fanotify.InitFlags, event_f_flags: u32) FanotifyInitError!i32 { const rc = system.fanotify_init(flags, event_f_flags); switch (errno(rc)) { .SUCCESS => return @intCast(rc), @@ -4530,16 +4530,28 @@ pub const FanotifyMarkError = error{ NameTooLong, } || UnexpectedError; -pub fn fanotify_mark(fanotify_fd: i32, flags: u32, mask: u64, dirfd: i32, pathname: ?[]const u8) FanotifyMarkError!void { +pub fn fanotify_mark( + fanotify_fd: fd_t, + flags: std.os.linux.fanotify.MarkFlags, + mask: std.os.linux.fanotify.MarkMask, + dirfd: fd_t, + pathname: ?[]const u8, +) FanotifyMarkError!void { if (pathname) |path| { const path_c = try toPosixPath(path); return fanotify_markZ(fanotify_fd, flags, mask, dirfd, &path_c); + } else { + return fanotify_markZ(fanotify_fd, flags, mask, dirfd, null); } - - return fanotify_markZ(fanotify_fd, flags, mask, dirfd, null); } -pub fn fanotify_markZ(fanotify_fd: i32, flags: u32, mask: u64, dirfd: i32, pathname: ?[*:0]const u8) FanotifyMarkError!void { +pub fn fanotify_markZ( + fanotify_fd: fd_t, + flags: std.os.linux.fanotify.MarkFlags, + mask: std.os.linux.fanotify.MarkMask, + dirfd: fd_t, + pathname: ?[*:0]const u8, +) FanotifyMarkError!void { const rc = system.fanotify_mark(fanotify_fd, flags, mask, dirfd, pathname); switch (errno(rc)) { .SUCCESS => return, @@ -7274,6 +7286,44 @@ pub fn ptrace(request: u32, pid: pid_t, addr: usize, signal: usize) PtraceError! }; } +pub const NameToFileHandleAtError = error{ + FileNotFound, + NotDir, + OperationNotSupported, + NameTooLong, + Unexpected, +}; + +pub fn name_to_handle_at( + dirfd: fd_t, + pathname: []const u8, + handle: *std.os.linux.file_handle, + mount_id: *i32, + flags: u32, +) NameToFileHandleAtError!void { + const pathname_c = try toPosixPath(pathname); + return name_to_handle_atZ(dirfd, &pathname_c, handle, mount_id, flags); +} + +pub fn name_to_handle_atZ( + dirfd: fd_t, + pathname_z: [*:0]const u8, + handle: *std.os.linux.file_handle, + mount_id: *i32, + flags: u32, +) NameToFileHandleAtError!void { + switch (errno(system.name_to_handle_at(dirfd, pathname_z, handle, mount_id, flags))) { + .SUCCESS => {}, + .FAULT => unreachable, // pathname, mount_id, or handle outside accessible address space + .INVAL => unreachable, // bad flags, or handle_bytes too big + .NOENT => return error.FileNotFound, + .NOTDIR => return error.NotDir, + .OPNOTSUPP => return error.OperationNotSupported, + .OVERFLOW => return error.NameTooLong, + else => |err| return unexpectedErrno(err), + } +} + pub const IoCtl_SIOCGIFINDEX_Error = error{ FileSystem, InterfaceNotFound, From deea36250ffe458d92b32b1ad090b8a958ba8082 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 8 Jul 2024 23:41:08 -0700 Subject: [PATCH 06/38] std.Build.Cache.Path: add `subPathOpt` and `TableAdapter` Helpful methods when using one of these structs as a hash table key. --- lib/std/Build/Cache/Path.zig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/std/Build/Cache/Path.zig b/lib/std/Build/Cache/Path.zig index cb6e4c708758..89dba6b577dc 100644 --- a/lib/std/Build/Cache/Path.zig +++ b/lib/std/Build/Cache/Path.zig @@ -151,6 +151,26 @@ pub fn eql(self: Path, other: Path) bool { return self.root_dir.eql(other.root_dir) and std.mem.eql(u8, self.sub_path, other.sub_path); } +pub fn subPathOpt(self: Path) ?[]const u8 { + return if (self.sub_path.len == 0) null else self.sub_path; +} + +/// Useful to make `Path` a key in `std.ArrayHashMap`. +pub const TableAdapter = struct { + pub const Hash = std.hash.Wyhash; + + pub fn hash(self: TableAdapter, a: Cache.Path) u32 { + _ = self; + const seed: u32 = @bitCast(a.root_dir.handle.fd); + return @truncate(Hash.hash(seed, a.sub_path)); + } + pub fn eql(self: TableAdapter, a: Cache.Path, b: Cache.Path, b_index: usize) bool { + _ = self; + _ = b_index; + return a.eql(b); + } +}; + const Path = @This(); const std = @import("../../std.zig"); const fs = std.fs; From bbd90a562efd6e802ed41df2649a05fad763a4de Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 8 Jul 2024 23:42:20 -0700 Subject: [PATCH 07/38] build runner: implement --watch (work-in-progress) I'm still learning how the fanotify API works but I think after playing with it in this commit, I finally know how to implement it, at least on Linux. This commit does not accomplish the goal but I want to take the code in a different direction and still be able to reference this point in time by viewing a source control diff. I think the move is going to be saving the file_handle for the parent directory, which combined with the dirent names is how we can correlate the events back to the Step instances that have registered file system inputs. I predict this to be similar to implementations on other operating systems. --- lib/compiler/build_runner.zig | 418 ++++++++++++++++++++++++----- lib/std/Build.zig | 55 ---- lib/std/Build/Step.zig | 73 +++-- lib/std/Build/Step/InstallFile.zig | 5 +- 4 files changed, 407 insertions(+), 144 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index cf78ad320f4f..d1c6af6189a2 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -8,6 +8,7 @@ const process = std.process; const ArrayList = std.ArrayList; const File = std.fs.File; const Step = std.Build.Step; +const Allocator = std.mem.Allocator; pub const root = @import("@build"); pub const dependencies = @import("@dependencies"); @@ -74,7 +75,6 @@ pub fn main() !void { .query = .{}, .result = try std.zig.system.resolveTargetQuery(.{}), }, - .watch = null, }; graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); @@ -105,6 +105,7 @@ pub fn main() !void { var help_menu = false; var steps_menu = false; var output_tmp_nonce: ?[16]u8 = null; + var watch = false; while (nextArg(args, &arg_idx)) |arg| { if (mem.startsWith(u8, arg, "-Z")) { @@ -229,9 +230,7 @@ pub fn main() !void { } else if (mem.eql(u8, arg, "--prominent-compile-errors")) { prominent_compile_errors = true; } else if (mem.eql(u8, arg, "--watch")) { - const watch = try arena.create(std.Build.Watch); - watch.* = std.Build.Watch.init; - graph.watch = watch; + watch = true; } else if (mem.eql(u8, arg, "-fwine")) { builder.enable_wine = true; } else if (mem.eql(u8, arg, "-fno-wine")) { @@ -297,6 +296,7 @@ pub fn main() !void { const main_progress_node = std.Progress.start(.{ .disable_printing = (color == .off), }); + defer main_progress_node.end(); builder.debug_log_scopes = debug_log_scopes.items; builder.resolveInstallPrefix(install_prefix, dir_list); @@ -345,13 +345,16 @@ pub fn main() !void { .max_rss_is_default = false, .max_rss_mutex = .{}, .skip_oom_steps = skip_oom_steps, + .watch = watch, .memory_blocked_steps = std.ArrayList(*Step).init(arena), + .step_stack = .{}, .prominent_compile_errors = prominent_compile_errors, .claimed_rss = 0, - .summary = summary orelse if (graph.watch != null) .new else .failures, + .summary = summary orelse if (watch) .new else .failures, .ttyconf = ttyconf, .stderr = stderr, + .thread_pool = undefined, }; if (run.max_rss == 0) { @@ -359,30 +362,311 @@ pub fn main() !void { run.max_rss_is_default = true; } - runStepNames( - arena, - builder, - targets.items, - main_progress_node, - thread_pool_options, - &run, - seed, - ) catch |err| switch (err) { - error.UncleanExit => { - if (graph.watch == null) - process.exit(1); - }, + const gpa = arena; + prepare(gpa, arena, builder, targets.items, &run, seed) catch |err| switch (err) { + error.UncleanExit => process.exit(1), else => return err, }; + + var w = Watch.init; + if (watch) { + w.fan_fd = try std.posix.fanotify_init(.{ + .CLASS = .NOTIF, + .CLOEXEC = true, + .NONBLOCK = true, + .REPORT_NAME = true, + .REPORT_DIR_FID = true, + .REPORT_FID = true, + .REPORT_TARGET_FID = true, + }, 0); + } + + try run.thread_pool.init(thread_pool_options); + defer run.thread_pool.deinit(); + + rebuild: while (true) { + runStepNames( + gpa, + builder, + targets.items, + main_progress_node, + &run, + ) catch |err| switch (err) { + error.UncleanExit => { + assert(!run.watch); + process.exit(1); + }, + else => return err, + }; + if (!watch) return cleanExit(); + + // Clear all file handles. + for (w.handle_table.keys(), w.handle_table.values()) |lfh, *step_set| { + lfh.destroy(gpa); + step_set.clearAndFree(gpa); + } + w.handle_table.clearRetainingCapacity(); + + // Add missing marks and note persisted ones. + for (run.step_stack.keys()) |step| { + for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { + { + const gop = try w.dir_table.getOrPut(gpa, path); + gop.value_ptr.* = w.generation; + if (!gop.found_existing) { + try std.posix.fanotify_mark(w.fan_fd, .{ + .ADD = true, + .ONLYDIR = true, + }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt()); + } + } + for (files.items) |basename| { + const file_handle = try Watch.getFileHandle(gpa, path, basename); + std.debug.print("watching file_handle '{}{s}' = {}\n", .{ + path, basename, std.fmt.fmtSliceHexLower(file_handle.slice()), + }); + const gop = try w.handle_table.getOrPut(gpa, file_handle); + if (!gop.found_existing) gop.value_ptr.* = .{}; + try gop.value_ptr.put(gpa, step, {}); + } + } + } + + { + // Remove marks for files that are no longer inputs. + var i: usize = 0; + while (i < w.dir_table.entries.len) { + const generations = w.dir_table.values(); + if (generations[i] == w.generation) { + i += 1; + continue; + } + + const path = w.dir_table.keys()[i]; + + try std.posix.fanotify_mark(w.fan_fd, .{ + .REMOVE = true, + .ONLYDIR = true, + }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt()); + + w.dir_table.swapRemoveAt(i); + } + w.generation +%= 1; + } + + // Wait until a file system notification arrives. Read all such events + // until the buffer is empty. Then wait for a debounce interval, resetting + // if any more events come in. After the debounce interval has passed, + // trigger a rebuild on all steps with modified inputs, as well as their + // recursive dependants. + const debounce_interval_ms = 10; + var poll_fds: [1]std.posix.pollfd = .{ + .{ + .fd = w.fan_fd, + .events = std.posix.POLL.IN, + .revents = undefined, + }, + }; + var caption_buf: [40]u8 = undefined; + const caption = std.fmt.bufPrint(&caption_buf, "Watching {d} Directories", .{ + w.dir_table.entries.len, + }) catch &caption_buf; + var debouncing_node = main_progress_node.start(caption, 0); + var debouncing = false; + while (true) { + const timeout: i32 = if (debouncing) debounce_interval_ms else -1; + const events_len = try std.posix.poll(&poll_fds, timeout); + if (events_len == 0) { + debouncing_node.end(); + continue :rebuild; + } + if (try markDirtySteps(&w)) { + if (!debouncing) { + debouncing = true; + debouncing_node.end(); + debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); + } + } + } + } } +fn markDirtySteps(w: *Watch) !bool { + const fanotify = std.os.linux.fanotify; + const M = fanotify.event_metadata; + var events_buf: [256 + 4096]u8 = undefined; + var any_dirty = false; + while (true) { + var len = std.posix.read(w.fan_fd, &events_buf) catch |err| switch (err) { + error.WouldBlock => return any_dirty, + else => |e| return e, + }; + //std.debug.dump_hex(events_buf[0..len]); + var meta: [*]align(1) M = @ptrCast(&events_buf); + while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({ + len -= meta[0].event_len; + meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); + }) { + assert(meta[0].vers == M.VERSION); + std.debug.print("meta = {any}\n", .{meta[0]}); + const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); + switch (fid.hdr.info_type) { + .DFID_NAME => { + const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); + const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes); + const file_name = mem.span(file_name_z); + std.debug.print("DFID_NAME file_handle = {any}, found: '{s}'\n", .{ file_handle.*, file_name }); + const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; + if (w.handle_table.get(lfh)) |step_set| { + for (step_set.keys()) |step| { + std.debug.print("DFID_NAME marking step '{s}' dirty\n", .{step.name}); + step.state = .precheck_done; + any_dirty = true; + } + } else { + std.debug.print("DFID_NAME changed file did not match any steps: '{}'\n", .{ + std.fmt.fmtSliceHexLower(lfh.slice()), + }); + } + }, + .FID => { + const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); + const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; + if (w.handle_table.get(lfh)) |step_set| { + for (step_set.keys()) |step| { + std.debug.print("FID marking step '{s}' dirty\n", .{step.name}); + step.state = .precheck_done; + any_dirty = true; + } + } else { + std.debug.print("FID changed file did not match any steps: '{}'\n", .{ + std.fmt.fmtSliceHexLower(lfh.slice()), + }); + } + }, + .DFID => { + const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); + const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; + if (w.handle_table.get(lfh)) |step_set| { + for (step_set.keys()) |step| { + std.debug.print("DFID marking step '{s}' dirty\n", .{step.name}); + step.state = .precheck_done; + any_dirty = true; + } + } else { + std.debug.print("DFID changed file did not match any steps\n", .{}); + } + }, + else => |t| { + std.debug.panic("TODO: received event type '{s}'", .{@tagName(t)}); + }, + } + } + } +} + +const Watch = struct { + dir_table: DirTable, + handle_table: HandleTable, + fan_fd: std.posix.fd_t, + generation: u8, + + const fan_mask: std.os.linux.fanotify.MarkMask = .{ + .CLOSE_WRITE = true, + .DELETE = true, + .MOVED_FROM = true, + .MOVED_TO = true, + .EVENT_ON_CHILD = true, + }; + + const init: Watch = .{ + .dir_table = .{}, + .handle_table = .{}, + .fan_fd = -1, + .generation = 0, + }; + + /// Key is the directory to watch which contains one or more files we are + /// interested in noticing changes to. + /// + /// Value is generation. + const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, u8, Cache.Path.TableAdapter, false); + + const HandleTable = std.ArrayHashMapUnmanaged(LinuxFileHandle, StepSet, LinuxFileHandle.Adapter, false); + const StepSet = std.AutoArrayHashMapUnmanaged(*Step, void); + + const Hash = std.hash.Wyhash; + const Cache = std.Build.Cache; + + const LinuxFileHandle = struct { + handle: *align(1) std.os.linux.file_handle, + + fn clone(lfh: LinuxFileHandle, gpa: Allocator) Allocator.Error!LinuxFileHandle { + const bytes = lfh.slice(); + const new_ptr = try gpa.alignedAlloc( + u8, + @alignOf(std.os.linux.file_handle), + @sizeOf(std.os.linux.file_handle) + bytes.len, + ); + const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr); + new_header.* = lfh.handle.*; + const new: LinuxFileHandle = .{ .handle = new_header }; + @memcpy(new.slice(), lfh.slice()); + return new; + } + + fn destroy(lfh: LinuxFileHandle, gpa: Allocator) void { + const ptr: [*]u8 = @ptrCast(lfh.handle); + const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes]; + return gpa.free(allocated_slice); + } + + fn slice(lfh: LinuxFileHandle) []u8 { + const ptr: [*]u8 = &lfh.handle.f_handle; + return ptr[0..lfh.handle.handle_bytes]; + } + + const Adapter = struct { + pub fn hash(self: Adapter, a: LinuxFileHandle) u32 { + _ = self; + const unsigned_type: u32 = @bitCast(a.handle.handle_type); + return @truncate(Hash.hash(unsigned_type, a.slice())); + } + pub fn eql(self: Adapter, a: LinuxFileHandle, b: LinuxFileHandle, b_index: usize) bool { + _ = self; + _ = b_index; + return a.handle.handle_type == b.handle.handle_type and mem.eql(u8, a.slice(), b.slice()); + } + }; + }; + + fn getFileHandle(gpa: Allocator, path: std.Build.Cache.Path, basename: []const u8) !LinuxFileHandle { + var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; + var mount_id: i32 = undefined; + var buf: [std.fs.max_path_bytes]u8 = undefined; + const joined_path = if (path.sub_path.len == 0) basename else path: { + break :path std.fmt.bufPrint(&buf, "{s}" ++ std.fs.path.sep_str ++ "{s}", .{ + path.sub_path, basename, + }) catch return error.NameTooLong; + }; + const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); + stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); + try std.posix.name_to_handle_at(path.root_dir.handle.fd, joined_path, stack_ptr, &mount_id, 0); + const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr }; + return stack_lfh.clone(gpa); + } +}; + const Run = struct { max_rss: u64, max_rss_is_default: bool, max_rss_mutex: std.Thread.Mutex, skip_oom_steps: bool, + watch: bool, memory_blocked_steps: std.ArrayList(*Step), + step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), prominent_compile_errors: bool, + thread_pool: std.Thread.Pool, claimed_rss: usize, summary: Summary, @@ -390,18 +674,15 @@ const Run = struct { stderr: File, }; -fn runStepNames( - arena: std.mem.Allocator, +fn prepare( + gpa: Allocator, + arena: Allocator, b: *std.Build, step_names: []const []const u8, - parent_prog_node: std.Progress.Node, - thread_pool_options: std.Thread.Pool.Options, run: *Run, seed: u32, ) !void { - const gpa = b.allocator; - var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{}; - defer step_stack.deinit(gpa); + const step_stack = &run.step_stack; if (step_names.len == 0) { try step_stack.put(gpa, b.default_step, {}); @@ -424,7 +705,7 @@ fn runStepNames( rand.shuffle(*Step, starting_steps); for (starting_steps) |s| { - constructGraphAndCheckForDependencyLoop(b, s, &step_stack, rand) catch |err| switch (err) { + constructGraphAndCheckForDependencyLoop(b, s, &run.step_stack, rand) catch |err| switch (err) { error.DependencyLoopDetected => return uncleanExit(), else => |e| return e, }; @@ -453,14 +734,19 @@ fn runStepNames( return uncleanExit(); } } +} - var thread_pool: std.Thread.Pool = undefined; - try thread_pool.init(thread_pool_options); - defer thread_pool.deinit(); +fn runStepNames( + gpa: Allocator, + b: *std.Build, + step_names: []const []const u8, + parent_prog_node: std.Progress.Node, + run: *Run, +) !void { + const step_stack = &run.step_stack; + const thread_pool = &run.thread_pool; { - defer parent_prog_node.end(); - const step_prog = parent_prog_node.start("steps", step_stack.count()); defer step_prog.end(); @@ -476,7 +762,7 @@ fn runStepNames( if (step.state == .skipped_oom) continue; thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{ - &wait_group, &thread_pool, b, step, step_prog, run, + &wait_group, b, step, step_prog, run, }); } } @@ -493,8 +779,6 @@ fn runStepNames( var failure_count: usize = 0; var pending_count: usize = 0; var total_compile_errors: usize = 0; - var compile_error_steps: std.ArrayListUnmanaged(*Step) = .{}; - defer compile_error_steps.deinit(gpa); for (step_stack.keys()) |s| { test_fail_count += s.test_results.fail_count; @@ -524,7 +808,6 @@ fn runStepNames( const compile_errors_len = s.result_error_bundle.errorMessageCount(); if (compile_errors_len > 0) { total_compile_errors += compile_errors_len; - try compile_error_steps.append(gpa, s); } }, } @@ -537,8 +820,8 @@ fn runStepNames( else => false, }; if (failure_count == 0 and failures_only) { - if (b.graph.watch != null) return; - return cleanExit(); + if (!run.watch) cleanExit(); + return; } const ttyconf = run.ttyconf; @@ -561,10 +844,13 @@ fn runStepNames( stderr.writeAll("\n") catch {}; // Print a fancy tree with build results. + var step_stack_copy = try step_stack.clone(gpa); + defer step_stack_copy.deinit(gpa); + var print_node: PrintNode = .{ .parent = null }; if (step_names.len == 0) { print_node.last = true; - printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack) catch {}; + printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {}; } else { const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: { var i: usize = step_names.len; @@ -583,44 +869,34 @@ fn runStepNames( for (step_names, 0..) |step_name, i| { const tls = b.top_level_steps.get(step_name).?; print_node.last = i + 1 == last_index; - printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack) catch {}; + printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {}; } } } if (failure_count == 0) { - if (b.graph.watch != null) return; - return cleanExit(); + if (!run.watch) cleanExit(); + return; } // Finally, render compile errors at the bottom of the terminal. - // We use a separate compile_error_steps array list because step_stack is destructively - // mutated in printTreeStep above. if (run.prominent_compile_errors and total_compile_errors > 0) { - for (compile_error_steps.items) |s| { + for (step_stack.keys()) |s| { if (s.result_error_bundle.errorMessageCount() > 0) { s.result_error_bundle.renderToStdErr(renderOptions(ttyconf)); } } - if (b.graph.watch != null) return uncleanExit(); - - // Signal to parent process that we have printed compile errors. The - // parent process may choose to omit the "following command failed" - // line in this case. - process.exit(2); + if (!run.watch) { + // Signal to parent process that we have printed compile errors. The + // parent process may choose to omit the "following command failed" + // line in this case. + std.debug.lockStdErr(); + process.exit(2); + } } - return uncleanExit(); -} - -fn uncleanExit() error{UncleanExit}!void { - if (builtin.mode == .Debug) { - return error.UncleanExit; - } else { - std.debug.lockStdErr(); - process.exit(1); - } + if (!run.watch) return uncleanExit(); } const PrintNode = struct { @@ -912,12 +1188,13 @@ fn constructGraphAndCheckForDependencyLoop( fn workerMakeOneStep( wg: *std.Thread.WaitGroup, - thread_pool: *std.Thread.Pool, b: *std.Build, s: *Step, prog_node: std.Progress.Node, run: *Run, ) void { + const thread_pool = &run.thread_pool; + // First, check the conditions for running this step. If they are not met, // then we return without doing the step, relying on another worker to // queue this step up again when dependencies are met. @@ -997,7 +1274,7 @@ fn workerMakeOneStep( // Successful completion of a step, so we queue up its dependants as well. for (s.dependants.items) |dep| { thread_pool.spawnWg(wg, workerMakeOneStep, .{ - wg, thread_pool, b, dep, prog_node, run, + wg, b, dep, prog_node, run, }); } } @@ -1022,7 +1299,7 @@ fn workerMakeOneStep( remaining -= dep.max_rss; thread_pool.spawnWg(wg, workerMakeOneStep, .{ - wg, thread_pool, b, dep, prog_node, run, + wg, b, dep, prog_node, run, }); } else { run.memory_blocked_steps.items[i] = dep; @@ -1242,13 +1519,22 @@ fn argsRest(args: [][:0]const u8, idx: usize) ?[][:0]const u8 { return args[idx..]; } +/// Perhaps in the future there could be an Advanced Options flag such as +/// --debug-build-runner-leaks which would make this function return instead of +/// calling exit. fn cleanExit() void { - // Perhaps in the future there could be an Advanced Options flag such as - // --debug-build-runner-leaks which would make this function return instead - // of calling exit. + std.debug.lockStdErr(); process.exit(0); } +/// Perhaps in the future there could be an Advanced Options flag such as +/// --debug-build-runner-leaks which would make this function return instead of +/// calling exit. +fn uncleanExit() error{UncleanExit} { + std.debug.lockStdErr(); + process.exit(1); +} + const Color = std.zig.Color; const Summary = enum { all, new, failures, none }; diff --git a/lib/std/Build.zig b/lib/std/Build.zig index bfab90971c78..e46f3ea0bac4 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -119,61 +119,6 @@ pub const Graph = struct { needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{}, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, - /// When `--watch` is provided, collects the set of files that should be - /// watched and the state to required to poll the system for changes. - watch: ?*Watch, -}; - -pub const Watch = struct { - table: Table, - - pub const init: Watch = .{ - .table = .{}, - }; - - /// Key is the directory to watch which contains one or more files we are - /// interested in noticing changes to. - pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, ReactionSet, TableContext, false); - - const Hash = std.hash.Wyhash; - - pub const TableContext = struct { - pub fn hash(self: TableContext, a: Cache.Path) u32 { - _ = self; - const seed: u32 = @bitCast(a.root_dir.handle.fd); - return @truncate(Hash.hash(seed, a.sub_path)); - } - pub fn eql(self: TableContext, a: Cache.Path, b: Cache.Path, b_index: usize) bool { - _ = self; - _ = b_index; - return a.eql(b); - } - }; - - pub const ReactionSet = std.ArrayHashMapUnmanaged(Match, void, Match.Context, false); - - pub const Match = struct { - /// Relative to the watched directory, the file path that triggers this - /// match. - basename: []const u8, - /// The step to re-run when file corresponding to `basename` is changed. - step: *Step, - - pub const Context = struct { - pub fn hash(self: Context, a: Match) u32 { - _ = self; - var hasher = Hash.init(0); - std.hash.autoHash(&hasher, a.step); - hasher.update(a.basename); - return @truncate(hasher.final()); - } - pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool { - _ = self; - _ = b_index; - return a.step == b.step and mem.eql(u8, a.basename, b.basename); - } - }; - }; }; const AvailableDeps = []const struct { []const u8, []const u8 }; diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 91fe29866696..fcab831f676d 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -7,6 +7,16 @@ dependencies: std.ArrayList(*Step), /// This field is empty during execution of the user's build script, and /// then populated during dependency loop checking in the build runner. dependants: std.ArrayListUnmanaged(*Step), +/// Collects the set of files that retrigger this step to run. +/// +/// This is used by the build system's implementation of `--watch` but it can +/// also be potentially useful for IDEs to know what effects editing a +/// particular file has. +/// +/// Populated within `make`. Implementation may choose to clear and repopulate, +/// retain previous value, or update. +inputs: Inputs, + state: State, /// Set this field to declare an upper bound on the amount of bytes of memory it will /// take to run the step. Zero means no limit. @@ -63,6 +73,11 @@ pub const MakeFn = *const fn (step: *Step, prog_node: std.Progress.Node) anyerro pub const State = enum { precheck_unstarted, precheck_started, + /// This is also used to indicate "dirty" steps that have been modified + /// after a previous build completed, in which case, the step may or may + /// not have been completed before. Either way, one or more of its direct + /// file system inputs have been modified, meaning that the step needs to + /// be re-evaluated. precheck_done, running, dependency_failure, @@ -134,6 +149,26 @@ pub const Run = @import("Step/Run.zig"); pub const TranslateC = @import("Step/TranslateC.zig"); pub const WriteFile = @import("Step/WriteFile.zig"); +pub const Inputs = struct { + table: Table, + + pub const init: Inputs = .{ + .table = .{}, + }; + + pub const Table = std.ArrayHashMapUnmanaged(Build.Cache.Path, Files, Build.Cache.Path.TableAdapter, false); + pub const Files = std.ArrayListUnmanaged([]const u8); + + pub fn populated(inputs: *Inputs) bool { + return inputs.table.count() != 0; + } + + pub fn clear(inputs: *Inputs, gpa: Allocator) void { + for (inputs.table.values()) |*files| files.deinit(gpa); + inputs.table.clearRetainingCapacity(); + } +}; + pub const StepOptions = struct { id: Id, name: []const u8, @@ -153,6 +188,7 @@ pub fn init(options: StepOptions) Step { .makeFn = options.makeFn, .dependencies = std.ArrayList(*Step).init(arena), .dependants = .{}, + .inputs = Inputs.init, .state = .precheck_unstarted, .max_rss = options.max_rss, .debug_stack_trace = blk: { @@ -542,19 +578,19 @@ pub fn allocPrintCmd2( return buf.toOwnedSlice(arena); } -pub fn cacheHit(s: *Step, man: *std.Build.Cache.Manifest) !bool { +pub fn cacheHit(s: *Step, man: *Build.Cache.Manifest) !bool { s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); return s.result_cached; } -fn failWithCacheError(s: *Step, man: *const std.Build.Cache.Manifest, err: anyerror) anyerror { +fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: anyerror) anyerror { const i = man.failed_file_index orelse return err; const pp = man.files.keys()[i].prefixed_path; const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path }); } -pub fn writeManifest(s: *Step, man: *std.Build.Cache.Manifest) !void { +pub fn writeManifest(s: *Step, man: *Build.Cache.Manifest) !void { if (s.test_results.isSuccess()) { man.writeManifest() catch |err| { try s.addError("unable to write cache manifest: {s}", .{@errorName(err)}); @@ -568,44 +604,37 @@ fn oom(err: anytype) noreturn { } } -pub fn addWatchInput(step: *Step, lazy_path: std.Build.LazyPath) void { +pub fn addWatchInput(step: *Step, lazy_path: Build.LazyPath) void { errdefer |err| oom(err); - const w = step.owner.graph.watch orelse return; switch (lazy_path) { - .src_path => |src_path| try addWatchInputFromBuilder(step, w, src_path.owner, src_path.sub_path), - .dependency => |d| try addWatchInputFromBuilder(step, w, d.dependency.builder, d.sub_path), + .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), + .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), .cwd_relative => |path_string| { - try addWatchInputFromPath(w, .{ + try addWatchInputFromPath(step, .{ .root_dir = .{ .path = null, .handle = std.fs.cwd(), }, .sub_path = std.fs.path.dirname(path_string) orelse "", - }, .{ - .step = step, - .basename = std.fs.path.basename(path_string), - }); + }, std.fs.path.basename(path_string)); }, // Nothing to watch because this dependency edge is modeled instead via `dependants`. .generated => {}, } } -fn addWatchInputFromBuilder(step: *Step, w: *std.Build.Watch, builder: *std.Build, sub_path: []const u8) !void { - return addWatchInputFromPath(w, .{ +fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { + return addWatchInputFromPath(step, .{ .root_dir = builder.build_root, .sub_path = std.fs.path.dirname(sub_path) orelse "", - }, .{ - .step = step, - .basename = std.fs.path.basename(sub_path), - }); + }, std.fs.path.basename(sub_path)); } -fn addWatchInputFromPath(w: *std.Build.Watch, path: std.Build.Cache.Path, match: std.Build.Watch.Match) !void { - const gpa = match.step.owner.allocator; - const gop = try w.table.getOrPut(gpa, path); +fn addWatchInputFromPath(step: *Step, path: Build.Cache.Path, basename: []const u8) !void { + const gpa = step.owner.allocator; + const gop = try step.inputs.table.getOrPut(gpa, path); if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.put(gpa, match, {}); + try gop.value_ptr.append(gpa, basename); } test { diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index dd6aa2d1c4a6..f68e6e8aa023 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -39,7 +39,10 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { _ = prog_node; const b = step.owner; const install_file: *InstallFile = @fieldParentPtr("step", step); - step.addWatchInput(install_file.source); + + // Inputs never change when re-running `make`. + if (!step.inputs.populated()) step.addWatchInput(install_file.source); + const full_src_path = install_file.source.getPath2(b, step); const full_dest_path = b.getInstallPath(install_file.dir, install_file.dest_rel_path); const cwd = std.fs.cwd(); From c5a4177140f417b80c3d2e86f247ee5af769a39a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 18:17:01 -0700 Subject: [PATCH 08/38] std.os.linux: add AT.HANDLE_FID --- lib/std/os/linux.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 91a050ee22b5..56930c6b290f 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -2946,6 +2946,8 @@ pub const AT = struct { /// Apply to the entire subtree pub const RECURSIVE = 0x8000; + + pub const HANDLE_FID = REMOVEDIR; }; pub const FALLOC = struct { From 5ee3971b1828468c89104cb01e19edc87edf35a6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 18:17:27 -0700 Subject: [PATCH 09/38] proof-of-concept --watch implementation based on fanotify So far, only implemented for InstallFile steps. Default debounce interval bumped to 50ms. I think it should be configurable. Next I have an idea to simplify the fanotify implementation, but other OS implementations might want to refer back to this commit before I make those changes. --- lib/compiler/build_runner.zig | 220 +++++++++++----------------------- lib/std/Build.zig | 1 + lib/std/Build/Watch.zig | 135 +++++++++++++++++++++ 3 files changed, 208 insertions(+), 148 deletions(-) create mode 100644 lib/std/Build/Watch.zig diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index d1c6af6189a2..c9fd00e0e098 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -8,6 +8,7 @@ const process = std.process; const ArrayList = std.ArrayList; const File = std.fs.File; const Step = std.Build.Step; +const Watch = std.Build.Watch; const Allocator = std.mem.Allocator; pub const root = @import("@build"); @@ -400,34 +401,26 @@ pub fn main() !void { }; if (!watch) return cleanExit(); - // Clear all file handles. - for (w.handle_table.keys(), w.handle_table.values()) |lfh, *step_set| { - lfh.destroy(gpa); - step_set.clearAndFree(gpa); - } - w.handle_table.clearRetainingCapacity(); - // Add missing marks and note persisted ones. for (run.step_stack.keys()) |step| { for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { - { + const reaction_set = rs: { const gop = try w.dir_table.getOrPut(gpa, path); - gop.value_ptr.* = w.generation; if (!gop.found_existing) { try std.posix.fanotify_mark(w.fan_fd, .{ .ADD = true, .ONLYDIR = true, }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt()); + + const dir_handle = try Watch.getDirHandle(gpa, path); + try w.handle_table.putNoClobber(gpa, dir_handle, .{}); } - } + break :rs &w.handle_table.values()[gop.index]; + }; for (files.items) |basename| { - const file_handle = try Watch.getFileHandle(gpa, path, basename); - std.debug.print("watching file_handle '{}{s}' = {}\n", .{ - path, basename, std.fmt.fmtSliceHexLower(file_handle.slice()), - }); - const gop = try w.handle_table.getOrPut(gpa, file_handle); + const gop = try reaction_set.getOrPut(gpa, basename); if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.put(gpa, step, {}); + try gop.value_ptr.put(gpa, step, w.generation); } } } @@ -435,11 +428,31 @@ pub fn main() !void { { // Remove marks for files that are no longer inputs. var i: usize = 0; - while (i < w.dir_table.entries.len) { - const generations = w.dir_table.values(); - if (generations[i] == w.generation) { - i += 1; - continue; + while (i < w.handle_table.entries.len) { + { + const reaction_set = &w.handle_table.values()[i]; + var step_set_i: usize = 0; + while (step_set_i < reaction_set.entries.len) { + const step_set = &reaction_set.values()[step_set_i]; + var dirent_i: usize = 0; + while (dirent_i < step_set.entries.len) { + const generations = step_set.values(); + if (generations[dirent_i] == w.generation) { + dirent_i += 1; + continue; + } + step_set.swapRemoveAt(dirent_i); + } + if (step_set.entries.len > 0) { + step_set_i += 1; + continue; + } + reaction_set.swapRemoveAt(step_set_i); + } + if (reaction_set.entries.len > 0) { + i += 1; + continue; + } } const path = w.dir_table.keys()[i]; @@ -450,6 +463,7 @@ pub fn main() !void { }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt()); w.dir_table.swapRemoveAt(i); + w.handle_table.swapRemoveAt(i); } w.generation +%= 1; } @@ -459,7 +473,7 @@ pub fn main() !void { // if any more events come in. After the debounce interval has passed, // trigger a rebuild on all steps with modified inputs, as well as their // recursive dependants. - const debounce_interval_ms = 10; + const debounce_interval_ms = 50; var poll_fds: [1]std.posix.pollfd = .{ .{ .fd = w.fan_fd, @@ -517,46 +531,48 @@ fn markDirtySteps(w: *Watch) !bool { const file_name = mem.span(file_name_z); std.debug.print("DFID_NAME file_handle = {any}, found: '{s}'\n", .{ file_handle.*, file_name }); const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; - if (w.handle_table.get(lfh)) |step_set| { - for (step_set.keys()) |step| { - std.debug.print("DFID_NAME marking step '{s}' dirty\n", .{step.name}); - step.state = .precheck_done; - any_dirty = true; + if (w.handle_table.getPtr(lfh)) |reaction_set| { + if (reaction_set.getPtr(file_name)) |step_set| { + for (step_set.keys()) |step| { + std.debug.print("DFID_NAME marking step '{s}' dirty\n", .{step.name}); + step.state = .precheck_done; + any_dirty = true; + } } } else { - std.debug.print("DFID_NAME changed file did not match any steps: '{}'\n", .{ + std.debug.print("DFID_NAME changed file did not match any directories: '{}'\n", .{ std.fmt.fmtSliceHexLower(lfh.slice()), }); } }, - .FID => { - const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); - const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; - if (w.handle_table.get(lfh)) |step_set| { - for (step_set.keys()) |step| { - std.debug.print("FID marking step '{s}' dirty\n", .{step.name}); - step.state = .precheck_done; - any_dirty = true; - } - } else { - std.debug.print("FID changed file did not match any steps: '{}'\n", .{ - std.fmt.fmtSliceHexLower(lfh.slice()), - }); - } - }, - .DFID => { - const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); - const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; - if (w.handle_table.get(lfh)) |step_set| { - for (step_set.keys()) |step| { - std.debug.print("DFID marking step '{s}' dirty\n", .{step.name}); - step.state = .precheck_done; - any_dirty = true; - } - } else { - std.debug.print("DFID changed file did not match any steps\n", .{}); - } - }, + //.FID => { + // const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); + // const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; + // if (w.handle_table.get(lfh)) |step_set| { + // for (step_set.keys()) |step| { + // std.debug.print("FID marking step '{s}' dirty\n", .{step.name}); + // step.state = .precheck_done; + // any_dirty = true; + // } + // } else { + // std.debug.print("FID changed file did not match any steps: '{}'\n", .{ + // std.fmt.fmtSliceHexLower(lfh.slice()), + // }); + // } + //}, + //.DFID => { + // const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); + // const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; + // if (w.handle_table.get(lfh)) |step_set| { + // for (step_set.keys()) |step| { + // std.debug.print("DFID marking step '{s}' dirty\n", .{step.name}); + // step.state = .precheck_done; + // any_dirty = true; + // } + // } else { + // std.debug.print("DFID changed file did not match any steps\n", .{}); + // } + //}, else => |t| { std.debug.panic("TODO: received event type '{s}'", .{@tagName(t)}); }, @@ -565,98 +581,6 @@ fn markDirtySteps(w: *Watch) !bool { } } -const Watch = struct { - dir_table: DirTable, - handle_table: HandleTable, - fan_fd: std.posix.fd_t, - generation: u8, - - const fan_mask: std.os.linux.fanotify.MarkMask = .{ - .CLOSE_WRITE = true, - .DELETE = true, - .MOVED_FROM = true, - .MOVED_TO = true, - .EVENT_ON_CHILD = true, - }; - - const init: Watch = .{ - .dir_table = .{}, - .handle_table = .{}, - .fan_fd = -1, - .generation = 0, - }; - - /// Key is the directory to watch which contains one or more files we are - /// interested in noticing changes to. - /// - /// Value is generation. - const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, u8, Cache.Path.TableAdapter, false); - - const HandleTable = std.ArrayHashMapUnmanaged(LinuxFileHandle, StepSet, LinuxFileHandle.Adapter, false); - const StepSet = std.AutoArrayHashMapUnmanaged(*Step, void); - - const Hash = std.hash.Wyhash; - const Cache = std.Build.Cache; - - const LinuxFileHandle = struct { - handle: *align(1) std.os.linux.file_handle, - - fn clone(lfh: LinuxFileHandle, gpa: Allocator) Allocator.Error!LinuxFileHandle { - const bytes = lfh.slice(); - const new_ptr = try gpa.alignedAlloc( - u8, - @alignOf(std.os.linux.file_handle), - @sizeOf(std.os.linux.file_handle) + bytes.len, - ); - const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr); - new_header.* = lfh.handle.*; - const new: LinuxFileHandle = .{ .handle = new_header }; - @memcpy(new.slice(), lfh.slice()); - return new; - } - - fn destroy(lfh: LinuxFileHandle, gpa: Allocator) void { - const ptr: [*]u8 = @ptrCast(lfh.handle); - const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes]; - return gpa.free(allocated_slice); - } - - fn slice(lfh: LinuxFileHandle) []u8 { - const ptr: [*]u8 = &lfh.handle.f_handle; - return ptr[0..lfh.handle.handle_bytes]; - } - - const Adapter = struct { - pub fn hash(self: Adapter, a: LinuxFileHandle) u32 { - _ = self; - const unsigned_type: u32 = @bitCast(a.handle.handle_type); - return @truncate(Hash.hash(unsigned_type, a.slice())); - } - pub fn eql(self: Adapter, a: LinuxFileHandle, b: LinuxFileHandle, b_index: usize) bool { - _ = self; - _ = b_index; - return a.handle.handle_type == b.handle.handle_type and mem.eql(u8, a.slice(), b.slice()); - } - }; - }; - - fn getFileHandle(gpa: Allocator, path: std.Build.Cache.Path, basename: []const u8) !LinuxFileHandle { - var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; - var mount_id: i32 = undefined; - var buf: [std.fs.max_path_bytes]u8 = undefined; - const joined_path = if (path.sub_path.len == 0) basename else path: { - break :path std.fmt.bufPrint(&buf, "{s}" ++ std.fs.path.sep_str ++ "{s}", .{ - path.sub_path, basename, - }) catch return error.NameTooLong; - }; - const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); - stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); - try std.posix.name_to_handle_at(path.root_dir.handle.fd, joined_path, stack_ptr, &mount_id, 0); - const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr }; - return stack_lfh.clone(gpa); - } -}; - const Run = struct { max_rss: u64, max_rss_is_default: bool, diff --git a/lib/std/Build.zig b/lib/std/Build.zig index e46f3ea0bac4..30aa5b72db1e 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -20,6 +20,7 @@ const Build = @This(); pub const Cache = @import("Build/Cache.zig"); pub const Step = @import("Build/Step.zig"); pub const Module = @import("Build/Module.zig"); +pub const Watch = @import("Build/Watch.zig"); /// Shared state among all Build instances. graph: *Graph, diff --git a/lib/std/Build/Watch.zig b/lib/std/Build/Watch.zig new file mode 100644 index 000000000000..d85b0f0a9223 --- /dev/null +++ b/lib/std/Build/Watch.zig @@ -0,0 +1,135 @@ +const std = @import("../std.zig"); +const Watch = @This(); +const Step = std.Build.Step; +const Allocator = std.mem.Allocator; + +dir_table: DirTable, +/// Keyed differently but indexes correspond 1:1 with `dir_table`. +handle_table: HandleTable, +fan_fd: std.posix.fd_t, +generation: Generation, + +pub const fan_mask: std.os.linux.fanotify.MarkMask = .{ + .CLOSE_WRITE = true, + .DELETE = true, + .MOVED_FROM = true, + .MOVED_TO = true, + .EVENT_ON_CHILD = true, +}; + +pub const init: Watch = .{ + .dir_table = .{}, + .handle_table = .{}, + .fan_fd = -1, + .generation = 0, +}; + +/// Key is the directory to watch which contains one or more files we are +/// interested in noticing changes to. +/// +/// Value is generation. +const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAdapter, false); + +const HandleTable = std.ArrayHashMapUnmanaged(LinuxFileHandle, ReactionSet, LinuxFileHandle.Adapter, false); +const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet); +const StepSet = std.AutoArrayHashMapUnmanaged(*Step, Generation); + +const Generation = u8; + +const Hash = std.hash.Wyhash; +const Cache = std.Build.Cache; + +pub const Match = struct { + /// Relative to the watched directory, the file path that triggers this + /// match. + basename: []const u8, + /// The step to re-run when file corresponding to `basename` is changed. + step: *Step, + + pub const Context = struct { + pub fn hash(self: Context, a: Match) u32 { + _ = self; + var hasher = Hash.init(0); + std.hash.autoHash(&hasher, a.step); + hasher.update(a.basename); + return @truncate(hasher.final()); + } + pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool { + _ = self; + _ = b_index; + return a.step == b.step and std.mem.eql(u8, a.basename, b.basename); + } + }; +}; + +pub const LinuxFileHandle = struct { + handle: *align(1) std.os.linux.file_handle, + + pub fn clone(lfh: LinuxFileHandle, gpa: Allocator) Allocator.Error!LinuxFileHandle { + const bytes = lfh.slice(); + const new_ptr = try gpa.alignedAlloc( + u8, + @alignOf(std.os.linux.file_handle), + @sizeOf(std.os.linux.file_handle) + bytes.len, + ); + const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr); + new_header.* = lfh.handle.*; + const new: LinuxFileHandle = .{ .handle = new_header }; + @memcpy(new.slice(), lfh.slice()); + return new; + } + + pub fn destroy(lfh: LinuxFileHandle, gpa: Allocator) void { + const ptr: [*]u8 = @ptrCast(lfh.handle); + const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes]; + return gpa.free(allocated_slice); + } + + pub fn slice(lfh: LinuxFileHandle) []u8 { + const ptr: [*]u8 = &lfh.handle.f_handle; + return ptr[0..lfh.handle.handle_bytes]; + } + + pub const Adapter = struct { + pub fn hash(self: Adapter, a: LinuxFileHandle) u32 { + _ = self; + const unsigned_type: u32 = @bitCast(a.handle.handle_type); + return @truncate(Hash.hash(unsigned_type, a.slice())); + } + pub fn eql(self: Adapter, a: LinuxFileHandle, b: LinuxFileHandle, b_index: usize) bool { + _ = self; + _ = b_index; + return a.handle.handle_type == b.handle.handle_type and std.mem.eql(u8, a.slice(), b.slice()); + } + }; +}; + +pub fn getFileHandle(gpa: Allocator, path: std.Build.Cache.Path, basename: []const u8) !LinuxFileHandle { + var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; + var mount_id: i32 = undefined; + var buf: [std.fs.max_path_bytes]u8 = undefined; + const joined_path = if (path.sub_path.len == 0) basename else path: { + break :path std.fmt.bufPrint(&buf, "{s}/{s}", .{ + path.sub_path, basename, + }) catch return error.NameTooLong; + }; + const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); + stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); + try std.posix.name_to_handle_at(path.root_dir.handle.fd, joined_path, stack_ptr, &mount_id, 0); + const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr }; + return stack_lfh.clone(gpa); +} + +pub fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path) !LinuxFileHandle { + var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; + var mount_id: i32 = undefined; + var buf: [std.fs.max_path_bytes]u8 = undefined; + const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{ + path.sub_path, + }) catch return error.NameTooLong; + const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); + stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); + try std.posix.name_to_handle_at(path.root_dir.handle.fd, adjusted_path, stack_ptr, &mount_id, std.os.linux.AT.HANDLE_FID); + const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr }; + return stack_lfh.clone(gpa); +} From 6f89824c22b57e14951870c8f589df33e180f552 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 18:37:20 -0700 Subject: [PATCH 10/38] build system: make debounce interval CLI-configurable --- lib/compiler/build_runner.zig | 53 ++++++++--------------------------- lib/std/Build/Watch.zig | 16 ----------- 2 files changed, 12 insertions(+), 57 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index c9fd00e0e098..cf883a581157 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -107,6 +107,7 @@ pub fn main() !void { var steps_menu = false; var output_tmp_nonce: ?[16]u8 = null; var watch = false; + var debounce_interval_ms: u16 = 50; while (nextArg(args, &arg_idx)) |arg| { if (mem.startsWith(u8, arg, "-Z")) { @@ -194,7 +195,15 @@ pub fn main() !void { const next_arg = nextArg(args, &arg_idx) orelse fatalWithHint("expected u32 after '{s}'", .{arg}); seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { - fatal("unable to parse seed '{s}' as 32-bit integer: {s}\n", .{ + fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {s}\n", .{ + next_arg, @errorName(err), + }); + }; + } else if (mem.eql(u8, arg, "--debounce")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u16 after '{s}'", .{arg}); + debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { + fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {s}\n", .{ next_arg, @errorName(err), }); }; @@ -473,7 +482,6 @@ pub fn main() !void { // if any more events come in. After the debounce interval has passed, // trigger a rebuild on all steps with modified inputs, as well as their // recursive dependants. - const debounce_interval_ms = 50; var poll_fds: [1]std.posix.pollfd = .{ .{ .fd = w.fan_fd, @@ -515,67 +523,29 @@ fn markDirtySteps(w: *Watch) !bool { error.WouldBlock => return any_dirty, else => |e| return e, }; - //std.debug.dump_hex(events_buf[0..len]); var meta: [*]align(1) M = @ptrCast(&events_buf); while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({ len -= meta[0].event_len; meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); }) { assert(meta[0].vers == M.VERSION); - std.debug.print("meta = {any}\n", .{meta[0]}); const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); switch (fid.hdr.info_type) { .DFID_NAME => { const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes); const file_name = mem.span(file_name_z); - std.debug.print("DFID_NAME file_handle = {any}, found: '{s}'\n", .{ file_handle.*, file_name }); const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; if (w.handle_table.getPtr(lfh)) |reaction_set| { if (reaction_set.getPtr(file_name)) |step_set| { for (step_set.keys()) |step| { - std.debug.print("DFID_NAME marking step '{s}' dirty\n", .{step.name}); step.state = .precheck_done; any_dirty = true; } } - } else { - std.debug.print("DFID_NAME changed file did not match any directories: '{}'\n", .{ - std.fmt.fmtSliceHexLower(lfh.slice()), - }); } }, - //.FID => { - // const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); - // const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; - // if (w.handle_table.get(lfh)) |step_set| { - // for (step_set.keys()) |step| { - // std.debug.print("FID marking step '{s}' dirty\n", .{step.name}); - // step.state = .precheck_done; - // any_dirty = true; - // } - // } else { - // std.debug.print("FID changed file did not match any steps: '{}'\n", .{ - // std.fmt.fmtSliceHexLower(lfh.slice()), - // }); - // } - //}, - //.DFID => { - // const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); - // const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; - // if (w.handle_table.get(lfh)) |step_set| { - // for (step_set.keys()) |step| { - // std.debug.print("DFID marking step '{s}' dirty\n", .{step.name}); - // step.state = .precheck_done; - // any_dirty = true; - // } - // } else { - // std.debug.print("DFID changed file did not match any steps\n", .{}); - // } - //}, - else => |t| { - std.debug.panic("TODO: received event type '{s}'", .{@tagName(t)}); - }, + else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}), } } } @@ -1349,6 +1319,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void { \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss \\ --fetch Exit after fetching dependency tree \\ --watch Continuously rebuild when source files are modified + \\ --debounce Delay before rebuilding after watched file detection \\ \\Project-Specific Options: \\ diff --git a/lib/std/Build/Watch.zig b/lib/std/Build/Watch.zig index d85b0f0a9223..b04bfcb475d1 100644 --- a/lib/std/Build/Watch.zig +++ b/lib/std/Build/Watch.zig @@ -104,22 +104,6 @@ pub const LinuxFileHandle = struct { }; }; -pub fn getFileHandle(gpa: Allocator, path: std.Build.Cache.Path, basename: []const u8) !LinuxFileHandle { - var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; - var mount_id: i32 = undefined; - var buf: [std.fs.max_path_bytes]u8 = undefined; - const joined_path = if (path.sub_path.len == 0) basename else path: { - break :path std.fmt.bufPrint(&buf, "{s}/{s}", .{ - path.sub_path, basename, - }) catch return error.NameTooLong; - }; - const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); - stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); - try std.posix.name_to_handle_at(path.root_dir.handle.fd, joined_path, stack_ptr, &mount_id, 0); - const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr }; - return stack_lfh.clone(gpa); -} - pub fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path) !LinuxFileHandle { var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; var mount_id: i32 = undefined; From 001ff7b3b2176467a9169069938396dcca75e93c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 20:05:37 -0700 Subject: [PATCH 11/38] std.Build.Watch: make dirty steps invalidate each other and make failed steps always be invalidated and make steps that don't need to be reevaluated marked as cached --- lib/compiler/build_runner.zig | 43 ++-------------------------- lib/std/Build/Step.zig | 25 ++++++++++++++++ lib/std/Build/Watch.zig | 54 +++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 40 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index cf883a581157..ea1cfda2668a 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -500,9 +500,10 @@ pub fn main() !void { const events_len = try std.posix.poll(&poll_fds, timeout); if (events_len == 0) { debouncing_node.end(); + Watch.markFailedStepsDirty(gpa, run.step_stack.keys()); continue :rebuild; } - if (try markDirtySteps(&w)) { + if (try w.markDirtySteps(gpa)) { if (!debouncing) { debouncing = true; debouncing_node.end(); @@ -513,44 +514,6 @@ pub fn main() !void { } } -fn markDirtySteps(w: *Watch) !bool { - const fanotify = std.os.linux.fanotify; - const M = fanotify.event_metadata; - var events_buf: [256 + 4096]u8 = undefined; - var any_dirty = false; - while (true) { - var len = std.posix.read(w.fan_fd, &events_buf) catch |err| switch (err) { - error.WouldBlock => return any_dirty, - else => |e| return e, - }; - var meta: [*]align(1) M = @ptrCast(&events_buf); - while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({ - len -= meta[0].event_len; - meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); - }) { - assert(meta[0].vers == M.VERSION); - const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); - switch (fid.hdr.info_type) { - .DFID_NAME => { - const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); - const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes); - const file_name = mem.span(file_name_z); - const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; - if (w.handle_table.getPtr(lfh)) |reaction_set| { - if (reaction_set.getPtr(file_name)) |step_set| { - for (step_set.keys()) |step| { - step.state = .precheck_done; - any_dirty = true; - } - } - } - }, - else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}), - } - } - } -} - const Run = struct { max_rss: u64, max_rss_is_default: bool, @@ -1319,7 +1282,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void { \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss \\ --fetch Exit after fetching dependency tree \\ --watch Continuously rebuild when source files are modified - \\ --debounce Delay before rebuilding after watched file detection + \\ --debounce Delay before rebuilding after changed file detected \\ \\Project-Specific Options: \\ diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index fcab831f676d..4b958be284e0 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -637,6 +637,31 @@ fn addWatchInputFromPath(step: *Step, path: Build.Cache.Path, basename: []const try gop.value_ptr.append(gpa, basename); } +fn reset(step: *Step, gpa: Allocator) void { + assert(step.state == .precheck_done); + + step.result_error_msgs.clearRetainingCapacity(); + step.result_stderr = ""; + step.result_cached = false; + step.result_duration_ns = null; + step.result_peak_rss = 0; + step.test_results = .{}; + + step.result_error_bundle.deinit(gpa); + step.result_error_bundle = std.zig.ErrorBundle.empty; +} + +/// Implementation detail of file watching. Prepares the step for being re-evaluated. +pub fn recursiveReset(step: *Step, gpa: Allocator) void { + assert(step.state != .precheck_done); + step.state = .precheck_done; + step.reset(gpa); + for (step.dependants.items) |dep| { + if (dep.state == .precheck_done) continue; + dep.recursiveReset(gpa); + } +} + test { _ = CheckFile; _ = CheckObject; diff --git a/lib/std/Build/Watch.zig b/lib/std/Build/Watch.zig index b04bfcb475d1..b58e524cab38 100644 --- a/lib/std/Build/Watch.zig +++ b/lib/std/Build/Watch.zig @@ -2,6 +2,7 @@ const std = @import("../std.zig"); const Watch = @This(); const Step = std.Build.Step; const Allocator = std.mem.Allocator; +const assert = std.debug.assert; dir_table: DirTable, /// Keyed differently but indexes correspond 1:1 with `dir_table`. @@ -117,3 +118,56 @@ pub fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path) !LinuxFileHandle const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr }; return stack_lfh.clone(gpa); } + +pub fn markDirtySteps(w: *Watch, gpa: Allocator) !bool { + const fanotify = std.os.linux.fanotify; + const M = fanotify.event_metadata; + var events_buf: [256 + 4096]u8 = undefined; + var any_dirty = false; + while (true) { + var len = std.posix.read(w.fan_fd, &events_buf) catch |err| switch (err) { + error.WouldBlock => return any_dirty, + else => |e| return e, + }; + var meta: [*]align(1) M = @ptrCast(&events_buf); + while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({ + len -= meta[0].event_len; + meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); + }) { + assert(meta[0].vers == M.VERSION); + const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); + switch (fid.hdr.info_type) { + .DFID_NAME => { + const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); + const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes); + const file_name = std.mem.span(file_name_z); + const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; + if (w.handle_table.getPtr(lfh)) |reaction_set| { + if (reaction_set.getPtr(file_name)) |step_set| { + for (step_set.keys()) |step| { + if (step.state != .precheck_done) { + step.recursiveReset(gpa); + any_dirty = true; + } + } + } + } + }, + else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}), + } + } + } +} + +pub fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { + for (all_steps) |step| switch (step.state) { + .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), + else => continue, + }; + // Now that all dirty steps have been found, the remaining steps that + // succeeded from last run shall be marked "cached". + for (all_steps) |step| switch (step.state) { + .success => step.result_cached = true, + else => continue, + }; +} From e6b6a728b36335d9ac32dd216082aaf1faa6327e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 20:06:56 -0700 Subject: [PATCH 12/38] build runner: fix build summary painting over CLI progress by obtaining the stderr lock when printing the build summary --- lib/compiler/build_runner.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index ea1cfda2668a..7cc3d2bcd7d0 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -682,9 +682,12 @@ fn runStepNames( } const ttyconf = run.ttyconf; - const stderr = run.stderr; if (run.summary != .none) { + std.debug.lockStdErr(); + defer std.debug.unlockStdErr(); + const stderr = run.stderr; + const total_count = success_count + failure_count + pending_count + skipped_count; ttyconf.setColor(stderr, .cyan) catch {}; stderr.writeAll("Build Summary:") catch {}; From e712ca595f3ac2e25911fd0d647bfb8c97019ce8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 20:25:53 -0700 Subject: [PATCH 13/38] std.os.linux: type safety for fanotify metadata event mask field --- lib/std/os/linux.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 56930c6b290f..43abbcc1a94d 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -4261,7 +4261,7 @@ pub const fanotify = struct { vers: u8, reserved: u8, metadata_len: u16, - mask: u64 align(8), + mask: MarkMask align(8), fd: i32, pid: i32, From 956f1ebc707f8a2530e49b80357768f3bf1235ac Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 20:26:16 -0700 Subject: [PATCH 14/38] std.Build.Watch: gracefully handle fanotify queue overflow --- lib/std/Build/Watch.zig | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/std/Build/Watch.zig b/lib/std/Build/Watch.zig index b58e524cab38..2ffcd27ca7a8 100644 --- a/lib/std/Build/Watch.zig +++ b/lib/std/Build/Watch.zig @@ -135,6 +135,12 @@ pub fn markDirtySteps(w: *Watch, gpa: Allocator) !bool { meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); }) { assert(meta[0].vers == M.VERSION); + if (meta[0].mask.Q_OVERFLOW) { + any_dirty = true; + std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); + markAllFilesDirty(w, gpa); + return true; + } const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); switch (fid.hdr.info_type) { .DFID_NAME => { @@ -171,3 +177,13 @@ pub fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { else => continue, }; } + +fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { + for (w.handle_table.values()) |reaction_set| { + for (reaction_set.values()) |step_set| { + for (step_set.keys()) |step| { + step.recursiveReset(gpa); + } + } + } +} From 0cc492a272ef9c03a34b57a26bf570b242615ddf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 21:08:20 -0700 Subject: [PATCH 15/38] make more build steps integrate with the watch system --- lib/std/Build.zig | 2 +- lib/std/Build/Step.zig | 15 +++++++++------ lib/std/Build/Step/CheckFile.zig | 1 + lib/std/Build/Step/CheckObject.zig | 1 + lib/std/Build/Step/ConfigHeader.zig | 2 ++ lib/std/Build/Step/InstallFile.zig | 4 +--- lib/std/Build/Step/ObjCopy.zig | 1 + lib/std/Build/Step/Options.zig | 3 +++ lib/std/Build/Step/RemoveDir.zig | 20 +++++++++++++------- test/tests.zig | 6 +++--- 10 files changed, 35 insertions(+), 20 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 30aa5b72db1e..72108adaf52b 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1052,7 +1052,7 @@ pub fn addWriteFiles(b: *Build) *Step.WriteFile { return Step.WriteFile.create(b); } -pub fn addRemoveDirTree(b: *Build, dir_path: []const u8) *Step.RemoveDir { +pub fn addRemoveDirTree(b: *Build, dir_path: LazyPath) *Step.RemoveDir { return Step.RemoveDir.create(b, dir_path); } diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 4b958be284e0..91d3924611ae 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -598,14 +598,17 @@ pub fn writeManifest(s: *Step, man: *Build.Cache.Manifest) !void { } } -fn oom(err: anytype) noreturn { - switch (err) { - error.OutOfMemory => @panic("out of memory"), - } +/// For steps that have a single input that never changes when re-running `make`. +pub fn singleUnchangingWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { + if (!step.inputs.populated()) try step.addWatchInput(lazy_path); +} + +pub fn clearWatchInputs(step: *Step) void { + const gpa = step.owner.allocator; + step.inputs.clear(gpa); } -pub fn addWatchInput(step: *Step, lazy_path: Build.LazyPath) void { - errdefer |err| oom(err); +pub fn addWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { switch (lazy_path) { .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), diff --git a/lib/std/Build/Step/CheckFile.zig b/lib/std/Build/Step/CheckFile.zig index b7ce2ded6136..c7a2046c1f63 100644 --- a/lib/std/Build/Step/CheckFile.zig +++ b/lib/std/Build/Step/CheckFile.zig @@ -50,6 +50,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { _ = prog_node; const b = step.owner; const check_file: *CheckFile = @fieldParentPtr("step", step); + try step.singleUnchangingWatchInput(check_file.source); const src_path = check_file.source.getPath2(b, step); const contents = fs.cwd().readFileAlloc(b.allocator, src_path, check_file.max_bytes) catch |err| { diff --git a/lib/std/Build/Step/CheckObject.zig b/lib/std/Build/Step/CheckObject.zig index f25708c55b36..93ee57e3b47b 100644 --- a/lib/std/Build/Step/CheckObject.zig +++ b/lib/std/Build/Step/CheckObject.zig @@ -555,6 +555,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { const b = step.owner; const gpa = b.allocator; const check_object: *CheckObject = @fieldParentPtr("step", step); + try step.singleUnchangingWatchInput(check_object.source); const src_path = check_object.source.getPath2(b, step); const contents = fs.cwd().readFileAllocOptions( diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index 6390a88da79d..fd655125cf11 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -168,6 +168,8 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { _ = prog_node; const b = step.owner; const config_header: *ConfigHeader = @fieldParentPtr("step", step); + if (config_header.style.getPath()) |lp| try step.singleUnchangingWatchInput(lp); + const gpa = b.allocator; const arena = b.allocator; diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index f68e6e8aa023..d29ac21c1cd4 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -39,9 +39,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { _ = prog_node; const b = step.owner; const install_file: *InstallFile = @fieldParentPtr("step", step); - - // Inputs never change when re-running `make`. - if (!step.inputs.populated()) step.addWatchInput(install_file.source); + try step.singleUnchangingWatchInput(install_file.source); const full_src_path = install_file.source.getPath2(b, step); const full_dest_path = b.getInstallPath(install_file.dir, install_file.dest_rel_path); diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index 06c9a8ef0d8d..908341aefb47 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -93,6 +93,7 @@ pub fn getOutputSeparatedDebug(objcopy: *const ObjCopy) ?std.Build.LazyPath { fn make(step: *Step, prog_node: std.Progress.Node) !void { const b = step.owner; const objcopy: *ObjCopy = @fieldParentPtr("step", step); + try step.singleUnchangingWatchInput(objcopy.input_file); var man = b.graph.cache.obtain(); defer man.deinit(); diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index 2937cf70e1ee..9ce23e0802a6 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -424,6 +424,9 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { item.path.getPath2(b, step), ); } + if (!step.inputs.populated()) for (options.args.items) |item| { + try step.addWatchInput(item.path); + }; const basename = "options.zig"; diff --git a/lib/std/Build/Step/RemoveDir.zig b/lib/std/Build/Step/RemoveDir.zig index 6483a684aaea..1b7dc7feb8be 100644 --- a/lib/std/Build/Step/RemoveDir.zig +++ b/lib/std/Build/Step/RemoveDir.zig @@ -2,22 +2,23 @@ const std = @import("std"); const fs = std.fs; const Step = std.Build.Step; const RemoveDir = @This(); +const LazyPath = std.Build.LazyPath; pub const base_id: Step.Id = .remove_dir; step: Step, -dir_path: []const u8, +doomed_path: LazyPath, -pub fn create(owner: *std.Build, dir_path: []const u8) *RemoveDir { +pub fn create(owner: *std.Build, doomed_path: LazyPath) *RemoveDir { const remove_dir = owner.allocator.create(RemoveDir) catch @panic("OOM"); remove_dir.* = .{ .step = Step.init(.{ .id = base_id, - .name = owner.fmt("RemoveDir {s}", .{dir_path}), + .name = owner.fmt("RemoveDir {s}", .{doomed_path.getDisplayName()}), .owner = owner, .makeFn = make, }), - .dir_path = owner.dupePath(dir_path), + .doomed_path = doomed_path.dupe(owner), }; return remove_dir; } @@ -30,14 +31,19 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { const b = step.owner; const remove_dir: *RemoveDir = @fieldParentPtr("step", step); - b.build_root.handle.deleteTree(remove_dir.dir_path) catch |err| { + step.clearWatchInputs(); + try step.addWatchInput(remove_dir.doomed_path); + + const full_doomed_path = remove_dir.doomed_path.getPath2(b, step); + + b.build_root.handle.deleteTree(full_doomed_path) catch |err| { if (b.build_root.path) |base| { return step.fail("unable to recursively delete path '{s}/{s}': {s}", .{ - base, remove_dir.dir_path, @errorName(err), + base, full_doomed_path, @errorName(err), }); } else { return step.fail("unable to recursively delete path '{s}': {s}", .{ - remove_dir.dir_path, @errorName(err), + full_doomed_path, @errorName(err), }); } }; diff --git a/test/tests.zig b/test/tests.zig index 95a86c68f66d..e19a9efccfde 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -771,7 +771,7 @@ pub fn addCliTests(b: *std.Build) *Step { run_run.expectStdErrEqual("All your codebase are belong to us.\n"); run_run.step.dependOn(&init_exe.step); - const cleanup = b.addRemoveDirTree(tmp_path); + const cleanup = b.addRemoveDirTree(.{ .cwd_relative = tmp_path }); cleanup.step.dependOn(&run_test.step); cleanup.step.dependOn(&run_run.step); cleanup.step.dependOn(&run_bad.step); @@ -816,7 +816,7 @@ pub fn addCliTests(b: *std.Build) *Step { }); checkfile.setName("check godbolt.org CLI usage generating valid asm"); - const cleanup = b.addRemoveDirTree(tmp_path); + const cleanup = b.addRemoveDirTree(.{ .cwd_relative = tmp_path }); cleanup.step.dependOn(&checkfile.step); step.dependOn(&cleanup.step); @@ -902,7 +902,7 @@ pub fn addCliTests(b: *std.Build) *Step { }); check6.step.dependOn(&run6.step); - const cleanup = b.addRemoveDirTree(tmp_path); + const cleanup = b.addRemoveDirTree(.{ .cwd_relative = tmp_path }); cleanup.step.dependOn(&check6.step); step.dependOn(&cleanup.step); From d1c14f2f52ddec476eca6d605b985a27f4d4fe28 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 21:47:26 -0700 Subject: [PATCH 16/38] std.Build.Step.WriteFile: extract UpdateSourceFiles This has been planned for quite some time; this commit finally does it. Also implements file system watching integration in the make() implementation for UpdateSourceFiles and fixes the reporting of step caching for both. WriteFile does not yet have file system watching integration. --- build.zig | 2 +- lib/std/Build.zig | 4 + lib/std/Build/Step.zig | 4 + lib/std/Build/Step/UpdateSourceFiles.zig | 114 +++++++++++++++++++++++ lib/std/Build/Step/WriteFile.zig | 89 +----------------- test/tests.zig | 2 +- 6 files changed, 128 insertions(+), 87 deletions(-) create mode 100644 lib/std/Build/Step/UpdateSourceFiles.zig diff --git a/build.zig b/build.zig index 0f0d7d4d67a2..a364982ce993 100644 --- a/build.zig +++ b/build.zig @@ -595,7 +595,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void { run_opt.addArg("-o"); run_opt.addFileArg(b.path("stage1/zig1.wasm")); - const copy_zig_h = b.addWriteFiles(); + const copy_zig_h = b.addUpdateSourceFiles(); copy_zig_h.addCopyFileToSource(b.path("lib/zig.h"), "stage1/zig.h"); const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm"); diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 72108adaf52b..87bb0eeeda68 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1052,6 +1052,10 @@ pub fn addWriteFiles(b: *Build) *Step.WriteFile { return Step.WriteFile.create(b); } +pub fn addUpdateSourceFiles(b: *Build) *Step.UpdateSourceFiles { + return Step.UpdateSourceFiles.create(b); +} + pub fn addRemoveDirTree(b: *Build, dir_path: LazyPath) *Step.RemoveDir { return Step.RemoveDir.create(b, dir_path); } diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 91d3924611ae..e41912d54843 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -102,6 +102,7 @@ pub const Id = enum { fmt, translate_c, write_file, + update_source_files, run, check_file, check_object, @@ -122,6 +123,7 @@ pub const Id = enum { .fmt => Fmt, .translate_c => TranslateC, .write_file => WriteFile, + .update_source_files => UpdateSourceFiles, .run => Run, .check_file => CheckFile, .check_object => CheckObject, @@ -148,6 +150,7 @@ pub const RemoveDir = @import("Step/RemoveDir.zig"); pub const Run = @import("Step/Run.zig"); pub const TranslateC = @import("Step/TranslateC.zig"); pub const WriteFile = @import("Step/WriteFile.zig"); +pub const UpdateSourceFiles = @import("Step/UpdateSourceFiles.zig"); pub const Inputs = struct { table: Table, @@ -680,4 +683,5 @@ test { _ = Run; _ = TranslateC; _ = WriteFile; + _ = UpdateSourceFiles; } diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig new file mode 100644 index 000000000000..9d1c8e20fe62 --- /dev/null +++ b/lib/std/Build/Step/UpdateSourceFiles.zig @@ -0,0 +1,114 @@ +//! Writes data to paths relative to the package root, effectively mutating the +//! package's source files. Be careful with the latter functionality; it should +//! not be used during the normal build process, but as a utility run by a +//! developer with intention to update source files, which will then be +//! committed to version control. +const std = @import("std"); +const Step = std.Build.Step; +const fs = std.fs; +const ArrayList = std.ArrayList; +const UpdateSourceFiles = @This(); + +step: Step, +output_source_files: std.ArrayListUnmanaged(OutputSourceFile), + +pub const base_id: Step.Id = .update_source_files; + +pub const OutputSourceFile = struct { + contents: Contents, + sub_path: []const u8, +}; + +pub const Contents = union(enum) { + bytes: []const u8, + copy: std.Build.LazyPath, +}; + +pub fn create(owner: *std.Build) *UpdateSourceFiles { + const usf = owner.allocator.create(UpdateSourceFiles) catch @panic("OOM"); + usf.* = .{ + .step = Step.init(.{ + .id = base_id, + .name = "UpdateSourceFiles", + .owner = owner, + .makeFn = make, + }), + .output_source_files = .{}, + }; + return usf; +} + +/// A path relative to the package root. +/// +/// Be careful with this because it updates source files. This should not be +/// used as part of the normal build process, but as a utility occasionally +/// run by a developer with intent to modify source files and then commit +/// those changes to version control. +pub fn addCopyFileToSource(usf: *UpdateSourceFiles, source: std.Build.LazyPath, sub_path: []const u8) void { + const b = usf.step.owner; + usf.output_source_files.append(b.allocator, .{ + .contents = .{ .copy = source }, + .sub_path = sub_path, + }) catch @panic("OOM"); + source.addStepDependencies(&usf.step); +} + +/// A path relative to the package root. +/// +/// Be careful with this because it updates source files. This should not be +/// used as part of the normal build process, but as a utility occasionally +/// run by a developer with intent to modify source files and then commit +/// those changes to version control. +pub fn addBytesToSource(usf: *UpdateSourceFiles, bytes: []const u8, sub_path: []const u8) void { + const b = usf.step.owner; + usf.output_source_files.append(b.allocator, .{ + .contents = .{ .bytes = bytes }, + .sub_path = sub_path, + }) catch @panic("OOM"); +} + +fn make(step: *Step, prog_node: std.Progress.Node) !void { + _ = prog_node; + const b = step.owner; + const usf: *UpdateSourceFiles = @fieldParentPtr("step", step); + + var any_miss = false; + for (usf.output_source_files.items) |output_source_file| { + if (fs.path.dirname(output_source_file.sub_path)) |dirname| { + b.build_root.handle.makePath(dirname) catch |err| { + return step.fail("unable to make path '{}{s}': {s}", .{ + b.build_root, dirname, @errorName(err), + }); + }; + } + switch (output_source_file.contents) { + .bytes => |bytes| { + b.build_root.handle.writeFile(.{ .sub_path = output_source_file.sub_path, .data = bytes }) catch |err| { + return step.fail("unable to write file '{}{s}': {s}", .{ + b.build_root, output_source_file.sub_path, @errorName(err), + }); + }; + any_miss = true; + }, + .copy => |file_source| { + if (!step.inputs.populated()) try step.addWatchInput(file_source); + + const source_path = file_source.getPath2(b, step); + const prev_status = fs.Dir.updateFile( + fs.cwd(), + source_path, + b.build_root.handle, + output_source_file.sub_path, + .{}, + ) catch |err| { + return step.fail("unable to update file from '{s}' to '{}{s}': {s}", .{ + source_path, b.build_root, output_source_file.sub_path, @errorName(err), + }); + }; + any_miss = any_miss or prev_status == .stale; + }, + } + } + + step.result_cached = !any_miss; +} diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 013c58890a2d..f35bf09b7e83 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -1,13 +1,6 @@ -//! WriteFile is primarily used to create a directory in an appropriate -//! location inside the local cache which has a set of files that have either -//! been generated during the build, or are copied from the source package. -//! -//! However, this step has an additional capability of writing data to paths -//! relative to the package root, effectively mutating the package's source -//! files. Be careful with the latter functionality; it should not be used -//! during the normal build process, but as a utility run by a developer with -//! intention to update source files, which will then be committed to version -//! control. +//! WriteFile is used to create a directory in an appropriate location inside +//! the local cache which has a set of files that have either been generated +//! during the build, or are copied from the source package. const std = @import("std"); const Step = std.Build.Step; const fs = std.fs; @@ -19,8 +12,6 @@ step: Step, // The elements here are pointers because we need stable pointers for the GeneratedFile field. files: std.ArrayListUnmanaged(File), directories: std.ArrayListUnmanaged(Directory), - -output_source_files: std.ArrayListUnmanaged(OutputSourceFile), generated_directory: std.Build.GeneratedFile, pub const base_id: Step.Id = .write_file; @@ -52,11 +43,6 @@ pub const Directory = struct { }; }; -pub const OutputSourceFile = struct { - contents: Contents, - sub_path: []const u8, -}; - pub const Contents = union(enum) { bytes: []const u8, copy: std.Build.LazyPath, @@ -73,7 +59,6 @@ pub fn create(owner: *std.Build) *WriteFile { }), .files = .{}, .directories = .{}, - .output_source_files = .{}, .generated_directory = .{ .step = &write_file.step }, }; return write_file; @@ -150,33 +135,6 @@ pub fn addCopyDirectory( }; } -/// A path relative to the package root. -/// Be careful with this because it updates source files. This should not be -/// used as part of the normal build process, but as a utility occasionally -/// run by a developer with intent to modify source files and then commit -/// those changes to version control. -pub fn addCopyFileToSource(write_file: *WriteFile, source: std.Build.LazyPath, sub_path: []const u8) void { - const b = write_file.step.owner; - write_file.output_source_files.append(b.allocator, .{ - .contents = .{ .copy = source }, - .sub_path = sub_path, - }) catch @panic("OOM"); - source.addStepDependencies(&write_file.step); -} - -/// A path relative to the package root. -/// Be careful with this because it updates source files. This should not be -/// used as part of the normal build process, but as a utility occasionally -/// run by a developer with intent to modify source files and then commit -/// those changes to version control. -pub fn addBytesToSource(write_file: *WriteFile, bytes: []const u8, sub_path: []const u8) void { - const b = write_file.step.owner; - write_file.output_source_files.append(b.allocator, .{ - .contents = .{ .bytes = bytes }, - .sub_path = sub_path, - }) catch @panic("OOM"); -} - /// Returns a `LazyPath` representing the base directory that contains all the /// files from this `WriteFile`. pub fn getDirectory(write_file: *WriteFile) std.Build.LazyPath { @@ -202,46 +160,6 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { const b = step.owner; const write_file: *WriteFile = @fieldParentPtr("step", step); - // Writing to source files is kind of an extra capability of this - // WriteFile - arguably it should be a different step. But anyway here - // it is, it happens unconditionally and does not interact with the other - // files here. - var any_miss = false; - for (write_file.output_source_files.items) |output_source_file| { - if (fs.path.dirname(output_source_file.sub_path)) |dirname| { - b.build_root.handle.makePath(dirname) catch |err| { - return step.fail("unable to make path '{}{s}': {s}", .{ - b.build_root, dirname, @errorName(err), - }); - }; - } - switch (output_source_file.contents) { - .bytes => |bytes| { - b.build_root.handle.writeFile(.{ .sub_path = output_source_file.sub_path, .data = bytes }) catch |err| { - return step.fail("unable to write file '{}{s}': {s}", .{ - b.build_root, output_source_file.sub_path, @errorName(err), - }); - }; - any_miss = true; - }, - .copy => |file_source| { - const source_path = file_source.getPath2(b, step); - const prev_status = fs.Dir.updateFile( - fs.cwd(), - source_path, - b.build_root.handle, - output_source_file.sub_path, - .{}, - ) catch |err| { - return step.fail("unable to update file from '{s}' to '{}{s}': {s}", .{ - source_path, b.build_root, output_source_file.sub_path, @errorName(err), - }); - }; - any_miss = any_miss or prev_status == .stale; - }, - } - } - // The cache is used here not really as a way to speed things up - because writing // the data to a file would probably be very fast - but as a way to find a canonical // location to put build artifacts. @@ -278,6 +196,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { if (try step.cacheHit(&man)) { const digest = man.final(); write_file.generated_directory.path = try b.cache_root.join(b.allocator, &.{ "o", &digest }); + step.result_cached = true; return; } diff --git a/test/tests.zig b/test/tests.zig index e19a9efccfde..0862f8deb0e5 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -882,7 +882,7 @@ pub fn addCliTests(b: *std.Build) *Step { const unformatted_code_utf16 = "\xff\xfe \x00 \x00 \x00 \x00/\x00/\x00 \x00n\x00o\x00 \x00r\x00e\x00a\x00s\x00o\x00n\x00"; const fmt6_path = std.fs.path.join(b.allocator, &.{ tmp_path, "fmt6.zig" }) catch @panic("OOM"); - const write6 = b.addWriteFiles(); + const write6 = b.addUpdateSourceFiles(); write6.addBytesToSource(unformatted_code_utf16, fmt6_path); write6.step.dependOn(&run5.step); From dcbb3aa1f329ba0f12f8a039df62cd347b8f2559 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 23:13:03 -0700 Subject: [PATCH 17/38] std.Build.Cache.Path: fix format function for absolute paths --- lib/std/Build/Cache/Path.zig | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/std/Build/Cache/Path.zig b/lib/std/Build/Cache/Path.zig index 89dba6b577dc..f6e469c254ad 100644 --- a/lib/std/Build/Cache/Path.zig +++ b/lib/std/Build/Cache/Path.zig @@ -58,6 +58,20 @@ pub fn openFile( return p.root_dir.handle.openFile(joined_path, flags); } +pub fn openDir( + p: Path, + sub_path: []const u8, + args: fs.Dir.OpenOptions, +) fs.Dir.OpenError!fs.Dir { + var buf: [fs.max_path_bytes]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.openDir(joined_path, args); +} + pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.Dir.OpenOptions) !fs.Dir { var buf: [fs.max_path_bytes]u8 = undefined; const joined_path = if (p.sub_path.len == 0) sub_path else p: { @@ -137,6 +151,10 @@ pub fn format( } if (fmt_string.len > 0) std.fmt.invalidFmtError(fmt_string, self); + if (std.fs.path.isAbsolute(self.sub_path)) { + try writer.writeAll(self.sub_path); + return; + } if (self.root_dir.path) |p| { try writer.writeAll(p); try writer.writeAll(fs.path.sep_str); From 0994e22a646a797aa91602a56a6a0cddf8cdbd28 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 23:13:28 -0700 Subject: [PATCH 18/38] build runner: more useful failure handling for fanotify_mark --- lib/compiler/build_runner.zig | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 7cc3d2bcd7d0..990eb1b4ba1e 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -416,10 +416,12 @@ pub fn main() !void { const reaction_set = rs: { const gop = try w.dir_table.getOrPut(gpa, path); if (!gop.found_existing) { - try std.posix.fanotify_mark(w.fan_fd, .{ + std.posix.fanotify_mark(w.fan_fd, .{ .ADD = true, .ONLYDIR = true, - }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt()); + }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt()) catch |err| { + fatal("unable to watch {}: {s}", .{ path, @errorName(err) }); + }; const dir_handle = try Watch.getDirHandle(gpa, path); try w.handle_table.putNoClobber(gpa, dir_handle, .{}); @@ -489,7 +491,7 @@ pub fn main() !void { .revents = undefined, }, }; - var caption_buf: [40]u8 = undefined; + var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; const caption = std.fmt.bufPrint(&caption_buf, "Watching {d} Directories", .{ w.dir_table.entries.len, }) catch &caption_buf; From 26bdc836d2d9b2654f7f95fec34c6276070f2a59 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 23:14:18 -0700 Subject: [PATCH 19/38] std.Build.LazyPath: add getPath3; deprecate getPath2 and getPath The goal is to move towards using `std.Build.Cache.Path` instead of absolute path names. This was helpful for implementing file watching integration to the InstallDir Step --- lib/std/Build.zig | 53 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 87bb0eeeda68..556ed89e8d5a 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -2327,36 +2327,52 @@ pub const LazyPath = union(enum) { } } - /// Returns an absolute path. - /// Intended to be used during the make phase only. + /// Deprecated, see `getPath3`. pub fn getPath(lazy_path: LazyPath, src_builder: *Build) []const u8 { return getPath2(lazy_path, src_builder, null); } - /// Returns an absolute path. + /// Deprecated, see `getPath3`. + pub fn getPath2(lazy_path: LazyPath, src_builder: *Build, asking_step: ?*Step) []const u8 { + const p = getPath3(lazy_path, src_builder, asking_step); + return src_builder.pathResolve(&.{ p.root_dir.path orelse ".", p.sub_path }); + } + /// Intended to be used during the make phase only. /// /// `asking_step` is only used for debugging purposes; it's the step being /// run that is asking for the path. - pub fn getPath2(lazy_path: LazyPath, src_builder: *Build, asking_step: ?*Step) []const u8 { + pub fn getPath3(lazy_path: LazyPath, src_builder: *Build, asking_step: ?*Step) Cache.Path { switch (lazy_path) { - .src_path => |sp| return sp.owner.pathFromRoot(sp.sub_path), - .cwd_relative => |p| return src_builder.pathFromCwd(p), + .src_path => |sp| return .{ + .root_dir = sp.owner.build_root, + .sub_path = sp.sub_path, + }, + .cwd_relative => |sub_path| return .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = sub_path, + }, .generated => |gen| { - var file_path: []const u8 = gen.file.step.owner.pathFromRoot(gen.file.path orelse { - std.debug.lockStdErr(); - const stderr = std.io.getStdErr(); - dumpBadGetPathHelp(gen.file.step, stderr, src_builder, asking_step) catch {}; - std.debug.unlockStdErr(); - @panic("misconfigured build script"); - }); + // TODO make gen.file.path not be absolute and use that as the + // basis for not traversing up too many directories. + + var file_path: Cache.Path = .{ + .root_dir = gen.file.step.owner.build_root, + .sub_path = gen.file.path orelse { + std.debug.lockStdErr(); + const stderr = std.io.getStdErr(); + dumpBadGetPathHelp(gen.file.step, stderr, src_builder, asking_step) catch {}; + std.debug.unlockStdErr(); + @panic("misconfigured build script"); + }, + }; if (gen.up > 0) { const cache_root_path = src_builder.cache_root.path orelse (src_builder.cache_root.join(src_builder.allocator, &.{"."}) catch @panic("OOM")); for (0..gen.up) |_| { - if (mem.eql(u8, file_path, cache_root_path)) { + if (mem.eql(u8, file_path.sub_path, cache_root_path)) { // If we hit the cache root and there's still more to go, // the script attempted to go too far. dumpBadDirnameHelp(gen.file.step, asking_step, @@ -2370,7 +2386,7 @@ pub const LazyPath = union(enum) { // path is absolute. // dirname will return null only if we're at root. // Typically, we'll stop well before that at the cache root. - file_path = fs.path.dirname(file_path) orelse { + file_path.sub_path = fs.path.dirname(file_path.sub_path) orelse { dumpBadDirnameHelp(gen.file.step, asking_step, \\dirname() reached root. \\No more directories left to go up. @@ -2381,9 +2397,12 @@ pub const LazyPath = union(enum) { } } - return src_builder.pathResolve(&.{ file_path, gen.sub_path }); + return file_path.join(src_builder.allocator, gen.sub_path) catch @panic("OOM"); + }, + .dependency => |dep| return .{ + .root_dir = dep.dependency.builder.build_root, + .sub_path = dep.sub_path, }, - .dependency => |dep| return dep.dependency.builder.pathFromRoot(dep.sub_path), } } From 7bccef3e4e4d48c2e1d34ec28754d8c33902f2bb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 23:15:22 -0700 Subject: [PATCH 20/38] std.Build.Watch: introduce special file "." to watch entire dir And use it to implement InstallDir Step watch integration. I'm not seeing any events triggered when I run `mkdir` in the watched directory, however, and I have not yet figured out why. --- lib/std/Build/Step.zig | 41 +++++++++++++++++++++++++++++-- lib/std/Build/Step/InstallDir.zig | 26 ++++++++++++-------- lib/std/Build/Watch.zig | 29 +++++++++++++++------- 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index e41912d54843..a2640492ba8d 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -160,6 +160,7 @@ pub const Inputs = struct { }; pub const Table = std.ArrayHashMapUnmanaged(Build.Cache.Path, Files, Build.Cache.Path.TableAdapter, false); + /// The special file name "." means any changes inside the directory. pub const Files = std.ArrayListUnmanaged([]const u8); pub fn populated(inputs: *Inputs) bool { @@ -611,8 +612,9 @@ pub fn clearWatchInputs(step: *Step) void { step.inputs.clear(gpa); } -pub fn addWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { - switch (lazy_path) { +/// Places a *file* dependency on the path. +pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!void { + switch (lazy_file) { .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), .cwd_relative => |path_string| { @@ -629,6 +631,34 @@ pub fn addWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!voi } } +/// Any changes inside the directory will trigger invalidation. +/// +/// See also `addDirectoryWatchInputFromPath` which takes a `Build.Cache.Path` instead. +pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!void { + switch (lazy_directory) { + .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), + .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), + .cwd_relative => |path_string| { + try addDirectoryWatchInputFromPath(step, .{ + .root_dir = .{ + .path = null, + .handle = std.fs.cwd(), + }, + .sub_path = path_string, + }); + }, + // Nothing to watch because this dependency edge is modeled instead via `dependants`. + .generated => {}, + } +} + +/// Any changes inside the directory will trigger invalidation. +/// +/// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead. +pub fn addDirectoryWatchInputFromPath(step: *Step, path: Build.Cache.Path) !void { + return addWatchInputFromPath(step, path, "."); +} + fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { return addWatchInputFromPath(step, .{ .root_dir = builder.build_root, @@ -636,6 +666,13 @@ fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) }, std.fs.path.basename(sub_path)); } +fn addDirectoryWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void { + return addDirectoryWatchInputFromPath(step, .{ + .root_dir = builder.build_root, + .sub_path = sub_path, + }); +} + fn addWatchInputFromPath(step: *Step, path: Build.Cache.Path, basename: []const u8) !void { const gpa = step.owner.allocator; const gop = try step.inputs.table.getOrPut(gpa, path); diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index ca38e09ec199..9cd71e7828db 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -59,12 +59,14 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { _ = prog_node; const b = step.owner; const install_dir: *InstallDir = @fieldParentPtr("step", step); + step.clearWatchInputs(); const arena = b.allocator; const dest_prefix = b.getInstallPath(install_dir.options.install_dir, install_dir.options.install_subdir); - const src_dir_path = install_dir.options.source_dir.getPath2(b, step); - var src_dir = b.build_root.handle.openDir(src_dir_path, .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{}{s}': {s}", .{ - b.build_root, src_dir_path, @errorName(err), + const src_dir_path = install_dir.options.source_dir.getPath3(b, step); + try step.addDirectoryWatchInput(install_dir.options.source_dir); + var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.subPathOpt() orelse ".", .{ .iterate = true }) catch |err| { + return step.fail("unable to open source directory '{}': {s}", .{ + src_dir_path, @errorName(err), }); }; defer src_dir.close(); @@ -88,12 +90,16 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { } // relative to src build root - const src_sub_path = b.pathJoin(&.{ src_dir_path, entry.path }); + const src_sub_path = try src_dir_path.join(arena, entry.path); const dest_path = b.pathJoin(&.{ dest_prefix, entry.path }); const cwd = fs.cwd(); switch (entry.kind) { - .directory => try cwd.makePath(dest_path), + .directory => { + const subdir_path = try src_dir_path.join(arena, entry.path); + try step.addDirectoryWatchInputFromPath(subdir_path); + try cwd.makePath(dest_path); + }, .file => { for (install_dir.options.blank_extensions) |ext| { if (mem.endsWith(u8, entry.path, ext)) { @@ -103,14 +109,14 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { } const prev_status = fs.Dir.updateFile( - b.build_root.handle, - src_sub_path, + src_sub_path.root_dir.handle, + src_sub_path.sub_path, cwd, dest_path, .{}, ) catch |err| { - return step.fail("unable to update file from '{}{s}' to '{s}': {s}", .{ - b.build_root, src_sub_path, dest_path, @errorName(err), + return step.fail("unable to update file from '{}' to '{s}': {s}", .{ + src_sub_path, dest_path, @errorName(err), }); }; all_cached = all_cached and prev_status == .fresh; diff --git a/lib/std/Build/Watch.zig b/lib/std/Build/Watch.zig index 2ffcd27ca7a8..e9b4449748b6 100644 --- a/lib/std/Build/Watch.zig +++ b/lib/std/Build/Watch.zig @@ -12,10 +12,13 @@ generation: Generation, pub const fan_mask: std.os.linux.fanotify.MarkMask = .{ .CLOSE_WRITE = true, + .CREATE = true, .DELETE = true, + .DELETE_SELF = true, + .EVENT_ON_CHILD = true, .MOVED_FROM = true, .MOVED_TO = true, - .EVENT_ON_CHILD = true, + .MOVE_SELF = true, }; pub const init: Watch = .{ @@ -32,6 +35,7 @@ pub const init: Watch = .{ const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAdapter, false); const HandleTable = std.ArrayHashMapUnmanaged(LinuxFileHandle, ReactionSet, LinuxFileHandle.Adapter, false); +/// Special key of "." means any changes in this directory trigger the steps. const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet); const StepSet = std.AutoArrayHashMapUnmanaged(*Step, Generation); @@ -149,14 +153,10 @@ pub fn markDirtySteps(w: *Watch, gpa: Allocator) !bool { const file_name = std.mem.span(file_name_z); const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; if (w.handle_table.getPtr(lfh)) |reaction_set| { - if (reaction_set.getPtr(file_name)) |step_set| { - for (step_set.keys()) |step| { - if (step.state != .precheck_done) { - step.recursiveReset(gpa); - any_dirty = true; - } - } - } + if (reaction_set.getPtr(".")) |glob_set| + any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); + if (reaction_set.getPtr(file_name)) |step_set| + any_dirty = markStepSetDirty(gpa, step_set, any_dirty); } }, else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}), @@ -187,3 +187,14 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { } } } + +fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool { + var this_any_dirty = false; + for (step_set.keys()) |step| { + if (step.state != .precheck_done) { + step.recursiveReset(gpa); + this_any_dirty = true; + } + } + return any_dirty or this_any_dirty; +} From 2ebf021061b8789226fa8bafe36f7827925d7022 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 9 Jul 2024 23:34:08 -0700 Subject: [PATCH 21/38] build runner: don't pass a dirfd + null to fanotify_mark Otherwise it reports EBADF. --- lib/compiler/build_runner.zig | 4 ++-- lib/std/Build/Cache/Path.zig | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 990eb1b4ba1e..b11ca783cc64 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -419,7 +419,7 @@ pub fn main() !void { std.posix.fanotify_mark(w.fan_fd, .{ .ADD = true, .ONLYDIR = true, - }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt()) catch |err| { + }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOrDot()) catch |err| { fatal("unable to watch {}: {s}", .{ path, @errorName(err) }); }; @@ -471,7 +471,7 @@ pub fn main() !void { try std.posix.fanotify_mark(w.fan_fd, .{ .REMOVE = true, .ONLYDIR = true, - }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt()); + }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOrDot()); w.dir_table.swapRemoveAt(i); w.handle_table.swapRemoveAt(i); diff --git a/lib/std/Build/Cache/Path.zig b/lib/std/Build/Cache/Path.zig index f6e469c254ad..c7160ea14350 100644 --- a/lib/std/Build/Cache/Path.zig +++ b/lib/std/Build/Cache/Path.zig @@ -173,6 +173,10 @@ pub fn subPathOpt(self: Path) ?[]const u8 { return if (self.sub_path.len == 0) null else self.sub_path; } +pub fn subPathOrDot(self: Path) []const u8 { + return if (self.sub_path.len == 0) "." else self.sub_path; +} + /// Useful to make `Path` a key in `std.ArrayHashMap`. pub const TableAdapter = struct { pub const Hash = std.hash.Wyhash; From b6ed833083e5174d7b616de7c9205596d1aa0a9f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 10 Jul 2024 00:41:35 -0700 Subject: [PATCH 22/38] build runner: ignore ENOENT of fanotify_mark REMOVE This happens when deleting watched directories and is harmless. --- lib/compiler/build_runner.zig | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index b11ca783cc64..b3bdd8804d93 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -468,10 +468,13 @@ pub fn main() !void { const path = w.dir_table.keys()[i]; - try std.posix.fanotify_mark(w.fan_fd, .{ + std.posix.fanotify_mark(w.fan_fd, .{ .REMOVE = true, .ONLYDIR = true, - }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOrDot()); + }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOrDot()) catch |err| switch (err) { + error.FileNotFound => {}, // Expected, harmless. + else => |e| std.log.warn("unable to unwatch '{}': {s}", .{ path, @errorName(e) }), + }; w.dir_table.swapRemoveAt(i); w.handle_table.swapRemoveAt(i); From 61d2234743e192bbee1d05c1d231427c3338d2af Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 10 Jul 2024 00:42:16 -0700 Subject: [PATCH 23/38] std.Build.Watch: add ONDIR to fanotify event mask This makes mkdir/rmdir events show up. --- lib/std/Build/Watch.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/Build/Watch.zig b/lib/std/Build/Watch.zig index e9b4449748b6..e4bef7ca4ef6 100644 --- a/lib/std/Build/Watch.zig +++ b/lib/std/Build/Watch.zig @@ -19,6 +19,7 @@ pub const fan_mask: std.os.linux.fanotify.MarkMask = .{ .MOVED_FROM = true, .MOVED_TO = true, .MOVE_SELF = true, + .ONDIR = true, }; pub const init: Watch = .{ From 5c3fae3a329bed39f780827b4c5bfc494dee1c6b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 10 Jul 2024 00:42:41 -0700 Subject: [PATCH 24/38] td.Build.Step.InstallDir: leave hint for wrong cached status Since I spent a couple minutes debugging this, hopefully this saves someone some future trouble doing the same. --- lib/std/Build/Step/InstallDir.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index 9cd71e7828db..8cb06641ec5f 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -99,6 +99,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { const subdir_path = try src_dir_path.join(arena, entry.path); try step.addDirectoryWatchInputFromPath(subdir_path); try cwd.makePath(dest_path); + // TODO: set result_cached=false if the directory did not already exist. }, .file => { for (install_dir.options.blank_extensions) |ext| { From f2856403c6997ff1317c968abed0871df9586c7c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 10 Jul 2024 15:08:23 -0700 Subject: [PATCH 25/38] introduce std.Build.Cache.Manifest.addFilePath and deprecate `addFile`. Part of an effort to move towards using `std.Build.Cache.Path` abstraction in more places, which makes it easier to avoid absolute paths and path resolution. --- lib/std/Build/Cache.zig | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 6d43361ae457..e78353fa455b 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -354,6 +354,19 @@ pub const Manifest = struct { /// ``` /// var file_contents = cache_hash.files.keys()[file_index].contents.?; /// ``` + pub fn addFilePath(m: *Manifest, file_path: Path, max_file_size: ?usize) !usize { + const gpa = m.cache.gpa; + try m.files.ensureUnusedCapacity(gpa, 1); + const resolved_path = try fs.path.resolve(gpa, &.{ + file_path.root_dir.path orelse ".", + file_path.subPathOrDot(), + }); + errdefer gpa.free(resolved_path); + const prefixed_path = try m.cache.findPrefixResolved(resolved_path); + return addFileInner(m, prefixed_path, max_file_size); + } + + /// Deprecated; use `addFilePath`. pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize { assert(self.manifest_file == null); @@ -362,6 +375,10 @@ pub const Manifest = struct { const prefixed_path = try self.cache.findPrefix(file_path); errdefer gpa.free(prefixed_path.sub_path); + return addFileInner(self, prefixed_path, max_file_size); + } + + fn addFileInner(self: *Manifest, prefixed_path: PrefixedPath, max_file_size: ?usize) !usize { const gop = self.files.getOrPutAssumeCapacityAdapted(prefixed_path, FilesAdapter{}); if (gop.found_existing) { gop.key_ptr.updateMaxSize(max_file_size); From a966eee090d55c7d61484333af675c80115bf188 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 10 Jul 2024 15:09:46 -0700 Subject: [PATCH 26/38] std.Build.Step.WriteFile: fix handling of directories and add file system watching integration. `addDirectoryWatchInput` now returns a `bool` which helps remind the caller to 1. call addDirectoryWatchInputFromPath on any derived paths 2. but only if the dependency is not already captured by a step dependency edge. The make function now recursively walks all directories and adds the found files to the cache hash rather than incorrectly only adding the directory name to the cache hash. closes #20571 --- lib/std/Build/Step.zig | 14 +++- lib/std/Build/Step/InstallDir.zig | 7 +- lib/std/Build/Step/WriteFile.zig | 117 ++++++++++++++++++++---------- 3 files changed, 95 insertions(+), 43 deletions(-) diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index a2640492ba8d..5c77bd3367b6 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -634,7 +634,11 @@ pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!voi /// Any changes inside the directory will trigger invalidation. /// /// See also `addDirectoryWatchInputFromPath` which takes a `Build.Cache.Path` instead. -pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!void { +/// +/// Paths derived from this directory should also be manually added via +/// `addDirectoryWatchInputFromPath` if and only if this function returns +/// `true`. +pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!bool { switch (lazy_directory) { .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path), .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path), @@ -648,13 +652,19 @@ pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Alloc }); }, // Nothing to watch because this dependency edge is modeled instead via `dependants`. - .generated => {}, + .generated => return false, } + return true; } /// Any changes inside the directory will trigger invalidation. /// /// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead. +/// +/// This function should only be called when it has been verified that the +/// dependency on `path` is not already accounted for by a `Step` dependency. +/// In other words, before calling this function, first check that the +/// `Build.LazyPath` which this `path` is derived from is not `generated`. pub fn addDirectoryWatchInputFromPath(step: *Step, path: Build.Cache.Path) !void { return addWatchInputFromPath(step, path, "."); } diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index 8cb06641ec5f..78281e56d9df 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -63,8 +63,8 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { const arena = b.allocator; const dest_prefix = b.getInstallPath(install_dir.options.install_dir, install_dir.options.install_subdir); const src_dir_path = install_dir.options.source_dir.getPath3(b, step); - try step.addDirectoryWatchInput(install_dir.options.source_dir); - var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.subPathOpt() orelse ".", .{ .iterate = true }) catch |err| { + const need_derived_inputs = try step.addDirectoryWatchInput(install_dir.options.source_dir); + var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { return step.fail("unable to open source directory '{}': {s}", .{ src_dir_path, @errorName(err), }); @@ -96,8 +96,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { switch (entry.kind) { .directory => { - const subdir_path = try src_dir_path.join(arena, entry.path); - try step.addDirectoryWatchInputFromPath(subdir_path); + if (need_derived_inputs) try step.addDirectoryWatchInputFromPath(src_sub_path); try cwd.makePath(dest_path); // TODO: set result_cached=false if the directory did not already exist. }, diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index f35bf09b7e83..6c0770e45866 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -40,6 +40,22 @@ pub const Directory = struct { .include_extensions = if (opts.include_extensions) |incs| b.dupeStrings(incs) else null, }; } + + pub fn pathIncluded(opts: Options, path: []const u8) bool { + for (opts.exclude_extensions) |ext| { + if (std.mem.endsWith(u8, path, ext)) + return false; + } + if (opts.include_extensions) |incs| { + for (incs) |inc| { + if (std.mem.endsWith(u8, path, inc)) + return true; + } else { + return false; + } + } + return true; + } }; }; @@ -158,7 +174,10 @@ fn maybeUpdateName(write_file: *WriteFile) void { fn make(step: *Step, prog_node: std.Progress.Node) !void { _ = prog_node; const b = step.owner; + const arena = b.allocator; + const gpa = arena; const write_file: *WriteFile = @fieldParentPtr("step", step); + step.clearWatchInputs(); // The cache is used here not really as a way to speed things up - because writing // the data to a file would probably be very fast - but as a way to find a canonical @@ -173,29 +192,67 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { // Random bytes to make WriteFile unique. Refresh this with // new random bytes when WriteFile implementation is modified // in a non-backwards-compatible way. - man.hash.add(@as(u32, 0xd767ee59)); + man.hash.add(@as(u32, 0xc2a287d0)); for (write_file.files.items) |file| { man.hash.addBytes(file.sub_path); + switch (file.contents) { .bytes => |bytes| { man.hash.addBytes(bytes); }, - .copy => |file_source| { - _ = try man.addFile(file_source.getPath2(b, step), null); + .copy => |lazy_path| { + const path = lazy_path.getPath3(b, step); + _ = try man.addFilePath(path, null); + try step.addWatchInput(lazy_path); }, } } - for (write_file.directories.items) |dir| { - man.hash.addBytes(dir.source.getPath2(b, step)); + + const open_dir_cache = try arena.alloc(fs.Dir, write_file.directories.items.len); + var open_dirs_count: usize = 0; + defer closeDirs(open_dir_cache[0..open_dirs_count]); + + for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| { man.hash.addBytes(dir.sub_path); for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext); if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc); + + const need_derived_inputs = try step.addDirectoryWatchInput(dir.source); + const src_dir_path = dir.source.getPath3(b, step); + + var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| { + return step.fail("unable to open source directory '{}': {s}", .{ + src_dir_path, @errorName(err), + }); + }; + open_dir_cache_elem.* = src_dir; + open_dirs_count += 1; + + var it = try src_dir.walk(gpa); + defer it.deinit(); + while (try it.next()) |entry| { + if (!dir.options.pathIncluded(entry.path)) continue; + + switch (entry.kind) { + .directory => { + if (need_derived_inputs) { + const entry_path = try src_dir_path.join(arena, entry.path); + try step.addDirectoryWatchInputFromPath(entry_path); + } + }, + .file => { + const entry_path = try src_dir_path.join(arena, entry.path); + _ = try man.addFilePath(entry_path, null); + }, + else => continue, + } + } } if (try step.cacheHit(&man)) { const digest = man.final(); - write_file.generated_directory.path = try b.cache_root.join(b.allocator, &.{ "o", &digest }); + write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); step.result_cached = true; return; } @@ -203,7 +260,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { const digest = man.final(); const cache_path = "o" ++ fs.path.sep_str ++ digest; - write_file.generated_directory.path = try b.cache_root.join(b.allocator, &.{ "o", &digest }); + write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest }); var cache_dir = b.cache_root.handle.makeOpenPath(cache_path, .{}) catch |err| { return step.fail("unable to make path '{}{s}': {s}", .{ @@ -256,8 +313,9 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { }, } } - for (write_file.directories.items) |dir| { - const full_src_dir_path = dir.source.getPath2(b, step); + + for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| { + const src_dir_path = dir.source.getPath3(b, step); const dest_dirname = dir.sub_path; if (dest_dirname.len != 0) { @@ -268,44 +326,25 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { }; } - var src_dir = b.build_root.handle.openDir(full_src_dir_path, .{ .iterate = true }) catch |err| { - return step.fail("unable to open source directory '{s}': {s}", .{ - full_src_dir_path, @errorName(err), - }); - }; - defer src_dir.close(); + var it = try already_open_dir.walk(gpa); + defer it.deinit(); + while (try it.next()) |entry| { + if (!dir.options.pathIncluded(entry.path)) continue; - var it = try src_dir.walk(b.allocator); - next_entry: while (try it.next()) |entry| { - for (dir.options.exclude_extensions) |ext| { - if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry; - } - if (dir.options.include_extensions) |incs| { - for (incs) |inc| { - if (std.mem.endsWith(u8, entry.path, inc)) break; - } else { - continue :next_entry; - } - } - const full_src_entry_path = b.pathJoin(&.{ full_src_dir_path, entry.path }); + const src_entry_path = try src_dir_path.join(arena, entry.path); const dest_path = b.pathJoin(&.{ dest_dirname, entry.path }); switch (entry.kind) { .directory => try cache_dir.makePath(dest_path), .file => { const prev_status = fs.Dir.updateFile( - cwd, - full_src_entry_path, + src_entry_path.root_dir.handle, + src_entry_path.sub_path, cache_dir, dest_path, .{}, ) catch |err| { - return step.fail("unable to update file from '{s}' to '{}{s}{c}{s}': {s}", .{ - full_src_entry_path, - b.cache_root, - cache_path, - fs.path.sep, - dest_path, - @errorName(err), + return step.fail("unable to update file from '{}' to '{}{s}{c}{s}': {s}", .{ + src_entry_path, b.cache_root, cache_path, fs.path.sep, dest_path, @errorName(err), }); }; _ = prev_status; @@ -317,3 +356,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { try step.writeManifest(&man); } + +fn closeDirs(dirs: []fs.Dir) void { + for (dirs) |*d| d.close(); +} From 6fcb1897d263130b5ff7a25dd12f027bddd75b2f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 10 Jul 2024 17:11:53 -0700 Subject: [PATCH 27/38] std.Build.Step.WriteFile: remove random bytes from cache hash The cache hash already has the zig version in there, so it's not really needed. --- lib/std/Build/Step/WriteFile.zig | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index 6c0770e45866..c1488a23d2ed 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -189,11 +189,6 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { var man = b.graph.cache.obtain(); defer man.deinit(); - // Random bytes to make WriteFile unique. Refresh this with - // new random bytes when WriteFile implementation is modified - // in a non-backwards-compatible way. - man.hash.add(@as(u32, 0xc2a287d0)); - for (write_file.files.items) |file| { man.hash.addBytes(file.sub_path); From 2e429697865ebcadc001a7d167ef964b3f1393a2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 10 Jul 2024 17:14:54 -0700 Subject: [PATCH 28/38] std.Build.Step.Run: integrate with --watch --- lib/std/Build/Step.zig | 40 ++++++++++++++++++++++++++++++++++++++ lib/std/Build/Step/Run.zig | 8 ++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 5c77bd3367b6..13cd47981bb2 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -582,11 +582,26 @@ pub fn allocPrintCmd2( return buf.toOwnedSlice(arena); } +/// Prefer `cacheHitAndWatch` unless you already added watch inputs +/// separately from using the cache system. pub fn cacheHit(s: *Step, man: *Build.Cache.Manifest) !bool { s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err); return s.result_cached; } +/// Clears previous watch inputs, if any, and then populates watch inputs from +/// the full set of files picked up by the cache manifest. +/// +/// Must be accompanied with `writeManifestAndWatch`. +pub fn cacheHitAndWatch(s: *Step, man: *Build.Cache.Manifest) !bool { + const is_hit = man.hit() catch |err| return failWithCacheError(s, man, err); + s.result_cached = is_hit; + // The above call to hit() populates the manifest with files, so in case of + // a hit, we need to populate watch inputs. + if (is_hit) try setWatchInputsFromManifest(s, man); + return is_hit; +} + fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: anyerror) anyerror { const i = man.failed_file_index orelse return err; const pp = man.files.keys()[i].prefixed_path; @@ -594,6 +609,8 @@ fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: anyerror) return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path }); } +/// Prefer `writeManifestAndWatch` unless you already added watch inputs +/// separately from using the cache system. pub fn writeManifest(s: *Step, man: *Build.Cache.Manifest) !void { if (s.test_results.isSuccess()) { man.writeManifest() catch |err| { @@ -602,6 +619,29 @@ pub fn writeManifest(s: *Step, man: *Build.Cache.Manifest) !void { } } +/// Clears previous watch inputs, if any, and then populates watch inputs from +/// the full set of files picked up by the cache manifest. +/// +/// Must be accompanied with `cacheHitAndWatch`. +pub fn writeManifestAndWatch(s: *Step, man: *Build.Cache.Manifest) !void { + try writeManifest(s, man); + try setWatchInputsFromManifest(s, man); +} + +fn setWatchInputsFromManifest(s: *Step, man: *Build.Cache.Manifest) !void { + const arena = s.owner.allocator; + const prefixes = man.cache.prefixes(); + clearWatchInputs(s); + for (man.files.keys()) |file| { + // The file path data is freed when the cache manifest is cleaned up at the end of `make`. + const sub_path = try arena.dupe(u8, file.prefixed_path.sub_path); + try addWatchInputFromPath(s, .{ + .root_dir = prefixes[file.prefixed_path.prefix], + .sub_path = std.fs.path.dirname(sub_path) orelse "", + }, std.fs.path.basename(sub_path)); + } +} + /// For steps that have a single input that never changes when re-running `make`. pub fn singleUnchangingWatchInput(step: *Step, lazy_path: Build.LazyPath) Allocator.Error!void { if (!step.inputs.populated()) try step.addWatchInput(lazy_path); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 69d6b393fd41..b1e7060f3e95 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -615,7 +615,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { // On Windows we don't have rpaths so we have to add .dll search paths to PATH run.addPathForDynLibs(artifact); } - const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; // the path is guaranteed to be set + const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; try argv_list.append(file_path); @@ -665,7 +665,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { _ = try man.addFile(lazy_path.getPath2(b, step), null); } - if (!has_side_effects and try step.cacheHit(&man)) { + if (!has_side_effects and try step.cacheHitAndWatch(&man)) { // cache hit, skip running command const digest = man.final(); @@ -719,7 +719,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { } try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node); - if (!has_side_effects) try step.writeManifest(&man); + if (!has_side_effects) try step.writeManifestAndWatch(&man); return; }; @@ -795,7 +795,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { }; } - if (!has_side_effects) try step.writeManifest(&man); + if (!has_side_effects) try step.writeManifestAndWatch(&man); try populateGeneratedPaths( arena, From dad07fb6f33d9024dc6528ca3de37715fbc13182 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 10 Jul 2024 17:27:30 -0700 Subject: [PATCH 29/38] std.Build.Cache.Path: fix hash impl on windows --- lib/std/Build/Cache/Path.zig | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/std/Build/Cache/Path.zig b/lib/std/Build/Cache/Path.zig index c7160ea14350..0abe79d373d0 100644 --- a/lib/std/Build/Cache/Path.zig +++ b/lib/std/Build/Cache/Path.zig @@ -183,7 +183,11 @@ pub const TableAdapter = struct { pub fn hash(self: TableAdapter, a: Cache.Path) u32 { _ = self; - const seed: u32 = @bitCast(a.root_dir.handle.fd); + const seed = switch (@typeInfo(@TypeOf(a.root_dir.handle.fd))) { + .Pointer => @intFromPtr(a.root_dir.handle.fd), + .Int => @as(u32, @bitCast(a.root_dir.handle.fd)), + else => @compileError("unimplemented hash function"), + }; return @truncate(Hash.hash(seed, a.sub_path)); } pub fn eql(self: TableAdapter, a: Cache.Path, b: Cache.Path, b_index: usize) bool { From 768cb7e406fe26711167cbb2d67efdc10f830fdc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 11 Jul 2024 16:23:49 -0700 Subject: [PATCH 30/38] objcopy: use the fatal helper method --- lib/compiler/objcopy.zig | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/compiler/objcopy.zig b/lib/compiler/objcopy.zig index ff69c4b28750..f3360c8108ea 100644 --- a/lib/compiler/objcopy.zig +++ b/lib/compiler/objcopy.zig @@ -198,20 +198,14 @@ fn cmdObjCopy( return std.process.cleanExit(); }, .update => { - if (seen_update) { - std.debug.print("zig objcopy only supports 1 update for now\n", .{}); - std.process.exit(1); - } + if (seen_update) fatal("zig objcopy only supports 1 update for now", .{}); seen_update = true; try server.serveEmitBinPath(output, .{ .flags = .{ .cache_hit = false }, }); }, - else => { - std.debug.print("unsupported message: {s}", .{@tagName(hdr.tag)}); - std.process.exit(1); - }, + else => fatal("unsupported message: {s}", .{@tagName(hdr.tag)}), } } } From 818f9cb5a07bec1273d3fc1ecb54e7189a9e6106 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 11 Jul 2024 16:24:48 -0700 Subject: [PATCH 31/38] std.Build.Step.ObjCopy: remove random bytes from cache hash The cache hash already has the zig version in there, so it's not really needed. --- lib/std/Build/Step/ObjCopy.zig | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index 908341aefb47..d314550f601d 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -98,10 +98,6 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { var man = b.graph.cache.obtain(); defer man.deinit(); - // Random bytes to make ObjCopy unique. Refresh this with new random - // bytes when ObjCopy implementation is modified incompatibly. - man.hash.add(@as(u32, 0xe18b7baf)); - const full_src_path = objcopy.input_file.getPath2(b, step); _ = try man.addFile(full_src_path, null); man.hash.addOptionalBytes(objcopy.only_section); From fd4d366009e92c79137ee681334f216bbfc9b5f5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 11 Jul 2024 16:25:21 -0700 Subject: [PATCH 32/38] std.Build.Cache.Path: fix the format method This function previously wrote a trailing directory separator, but that's not correct if the path refers to a file. --- lib/std/Build/Cache/Path.zig | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/std/Build/Cache/Path.zig b/lib/std/Build/Cache/Path.zig index 0abe79d373d0..b81786d0a8b6 100644 --- a/lib/std/Build/Cache/Path.zig +++ b/lib/std/Build/Cache/Path.zig @@ -157,12 +157,17 @@ pub fn format( } if (self.root_dir.path) |p| { try writer.writeAll(p); - try writer.writeAll(fs.path.sep_str); + if (self.sub_path.len > 0) { + try writer.writeAll(fs.path.sep_str); + try writer.writeAll(self.sub_path); + } + return; } if (self.sub_path.len > 0) { try writer.writeAll(self.sub_path); - try writer.writeAll(fs.path.sep_str); + return; } + try writer.writeByte('.'); } pub fn eql(self: Path, other: Path) bool { From a3c20dffaed77727494d34f7b4b03c0d10771270 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 11 Jul 2024 16:26:04 -0700 Subject: [PATCH 33/38] integrate Compile steps with file watching Updates the build runner to unconditionally require a zig lib directory parameter. This parameter is needed in order to correctly understand file system inputs from zig compiler subprocesses, since they will refer to "the zig lib directory", and the build runner needs to place file system watches on directories in there. The build runner's fanotify file watching implementation now accounts for when two or more Cache.Path instances compare unequal but ultimately refer to the same directory in the file system. Breaking change: std.Build no longer has a zig_lib_dir field. Instead, there is the Graph zig_lib_directory field, and individual Compile steps can still have their zig lib directories overridden. I think this is unlikely to break anyone's build in practice. The compiler now sends a "file_system_inputs" message to the build runner which shares the full set of files that were added to the cache system with the build system, so that the build runner can watch properly and redo the Compile step. This is implemented for whole cache mode but not yet for incremental cache mode. --- build.zig | 9 +++--- lib/compiler/build_runner.zig | 55 +++++++++++++++++++---------------- lib/std/Build.zig | 16 +++++----- lib/std/Build/Cache.zig | 16 ++++++++++ lib/std/Build/Step.zig | 38 ++++++++++++++++++++++++ lib/std/zig/Server.zig | 16 +++++++++- src/Compilation.zig | 14 +++++++++ src/main.zig | 32 +++++++++++++++++--- 8 files changed, 152 insertions(+), 44 deletions(-) diff --git a/build.zig b/build.zig index a364982ce993..d234c68c8eea 100644 --- a/build.zig +++ b/build.zig @@ -1261,7 +1261,9 @@ fn generateLangRef(b: *std.Build) std.Build.LazyPath { }); var dir = b.build_root.handle.openDir("doc/langref", .{ .iterate = true }) catch |err| { - std.debug.panic("unable to open 'doc/langref' directory: {s}", .{@errorName(err)}); + std.debug.panic("unable to open '{}doc/langref' directory: {s}", .{ + b.build_root, @errorName(err), + }); }; defer dir.close(); @@ -1280,10 +1282,7 @@ fn generateLangRef(b: *std.Build) std.Build.LazyPath { // in a temporary directory "--cache-root", b.cache_root.path orelse ".", }); - if (b.zig_lib_dir) |p| { - cmd.addArg("--zig-lib-dir"); - cmd.addDirectoryArg(p); - } + cmd.addArgs(&.{ "--zig-lib-dir", b.fmt("{}", .{b.graph.zig_lib_directory}) }); cmd.addArgs(&.{"-i"}); cmd.addFileArg(b.path(b.fmt("doc/langref/{s}", .{entry.name}))); diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index b3bdd8804d93..384c16438046 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -31,21 +31,15 @@ pub fn main() !void { // skip my own exe name var arg_idx: usize = 1; - const zig_exe = nextArg(args, &arg_idx) orelse { - std.debug.print("Expected path to zig compiler\n", .{}); - return error.InvalidArgs; - }; - const build_root = nextArg(args, &arg_idx) orelse { - std.debug.print("Expected build root directory path\n", .{}); - return error.InvalidArgs; - }; - const cache_root = nextArg(args, &arg_idx) orelse { - std.debug.print("Expected cache root directory path\n", .{}); - return error.InvalidArgs; - }; - const global_cache_root = nextArg(args, &arg_idx) orelse { - std.debug.print("Expected global cache root directory path\n", .{}); - return error.InvalidArgs; + const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{}); + const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{}); + const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{}); + const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{}); + const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{}); + + const zig_lib_directory: std.Build.Cache.Directory = .{ + .path = zig_lib_dir, + .handle = try std.fs.cwd().openDir(zig_lib_dir, .{}), }; const build_root_directory: std.Build.Cache.Directory = .{ @@ -72,6 +66,7 @@ pub fn main() !void { .zig_exe = zig_exe, .env_map = try process.getEnvMap(arena), .global_cache_root = global_cache_directory, + .zig_lib_directory = zig_lib_directory, .host = .{ .query = .{}, .result = try std.zig.system.resolveTargetQuery(.{}), @@ -189,8 +184,6 @@ pub fn main() !void { arg, next_arg, }); }; - } else if (mem.eql(u8, arg, "--zig-lib-dir")) { - builder.zig_lib_dir = .{ .cwd_relative = nextArgOrFatal(args, &arg_idx) }; } else if (mem.eql(u8, arg, "--seed")) { const next_arg = nextArg(args, &arg_idx) orelse fatalWithHint("expected u32 after '{s}'", .{arg}); @@ -416,15 +409,27 @@ pub fn main() !void { const reaction_set = rs: { const gop = try w.dir_table.getOrPut(gpa, path); if (!gop.found_existing) { - std.posix.fanotify_mark(w.fan_fd, .{ - .ADD = true, - .ONLYDIR = true, - }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOrDot()) catch |err| { - fatal("unable to watch {}: {s}", .{ path, @errorName(err) }); - }; - const dir_handle = try Watch.getDirHandle(gpa, path); - try w.handle_table.putNoClobber(gpa, dir_handle, .{}); + // `dir_handle` may already be present in the table in + // the case that we have multiple Cache.Path instances + // that compare inequal but ultimately point to the same + // directory on the file system. + // In such case, we must revert adding this directory, but keep + // the additions to the step set. + const dh_gop = try w.handle_table.getOrPut(gpa, dir_handle); + if (dh_gop.found_existing) { + _ = w.dir_table.pop(); + } else { + assert(dh_gop.index == gop.index); + dh_gop.value_ptr.* = .{}; + std.posix.fanotify_mark(w.fan_fd, .{ + .ADD = true, + .ONLYDIR = true, + }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOrDot()) catch |err| { + fatal("unable to watch {}: {s}", .{ path, @errorName(err) }); + }; + } + break :rs dh_gop.value_ptr; } break :rs &w.handle_table.values()[gop.index]; }; diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 556ed89e8d5a..36f7396c8eb6 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -54,7 +54,6 @@ libc_file: ?[]const u8 = null, /// Path to the directory containing build.zig. build_root: Cache.Directory, cache_root: Cache.Directory, -zig_lib_dir: ?LazyPath, pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, args: ?[]const []const u8 = null, debug_log_scopes: []const []const u8 = &.{}, @@ -117,6 +116,7 @@ pub const Graph = struct { zig_exe: [:0]const u8, env_map: EnvMap, global_cache_root: Cache.Directory, + zig_lib_directory: Cache.Directory, needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{}, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, @@ -293,7 +293,6 @@ pub fn create( }), .description = "Remove build artifacts from prefix path", }, - .zig_lib_dir = null, .install_path = undefined, .args = null, .host = graph.host, @@ -379,7 +378,6 @@ fn createChildOnly( .libc_file = parent.libc_file, .build_root = build_root, .cache_root = parent.cache_root, - .zig_lib_dir = parent.zig_lib_dir, .debug_log_scopes = parent.debug_log_scopes, .debug_compile_errors = parent.debug_compile_errors, .debug_pkg_config = parent.debug_pkg_config, @@ -687,7 +685,7 @@ pub fn addExecutable(b: *Build, options: ExecutableOptions) *Step.Compile { .max_rss = options.max_rss, .use_llvm = options.use_llvm, .use_lld = options.use_lld, - .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, + .zig_lib_dir = options.zig_lib_dir, .win32_manifest = options.win32_manifest, }); } @@ -735,7 +733,7 @@ pub fn addObject(b: *Build, options: ObjectOptions) *Step.Compile { .max_rss = options.max_rss, .use_llvm = options.use_llvm, .use_lld = options.use_lld, - .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, + .zig_lib_dir = options.zig_lib_dir, }); } @@ -791,7 +789,7 @@ pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *Step.Compile .max_rss = options.max_rss, .use_llvm = options.use_llvm, .use_lld = options.use_lld, - .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, + .zig_lib_dir = options.zig_lib_dir, .win32_manifest = options.win32_manifest, }); } @@ -842,7 +840,7 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *Step.Compile .max_rss = options.max_rss, .use_llvm = options.use_llvm, .use_lld = options.use_lld, - .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, + .zig_lib_dir = options.zig_lib_dir, }); } @@ -905,7 +903,7 @@ pub fn addTest(b: *Build, options: TestOptions) *Step.Compile { .test_runner = options.test_runner, .use_llvm = options.use_llvm, .use_lld = options.use_lld, - .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, + .zig_lib_dir = options.zig_lib_dir, }); } @@ -929,7 +927,7 @@ pub fn addAssembly(b: *Build, options: AssemblyOptions) *Step.Compile { .optimize = options.optimize, }, .max_rss = options.max_rss, - .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, + .zig_lib_dir = options.zig_lib_dir, }); obj_step.addAssemblyFile(options.source_file); return obj_step; diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index e78353fa455b..8ba2f9f12899 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1007,6 +1007,22 @@ pub const Manifest = struct { } self.files.deinit(self.cache.gpa); } + + pub fn populateFileSystemInputs(man: *Manifest, buf: *std.ArrayListUnmanaged(u8)) Allocator.Error!void { + assert(@typeInfo(std.zig.Server.Message.PathPrefix).Enum.fields.len == man.cache.prefixes_len); + const gpa = man.cache.gpa; + const files = man.files.keys(); + if (files.len > 0) { + for (files) |file| { + try buf.ensureUnusedCapacity(gpa, file.prefixed_path.sub_path.len + 2); + buf.appendAssumeCapacity(file.prefixed_path.prefix + 1); + buf.appendSliceAssumeCapacity(file.prefixed_path.sub_path); + buf.appendAssumeCapacity(0); + } + // The null byte is a separator, not a terminator. + buf.items.len -= 1; + } + } }; /// On operating systems that support symlinks, does a readlink. On other operating systems, diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 13cd47981bb2..3c6cd660fff1 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -435,6 +435,44 @@ pub fn evalZigProcess( s.result_cached = ebp_hdr.flags.cache_hit; result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]); }, + .file_system_inputs => { + s.clearWatchInputs(); + var it = std.mem.splitScalar(u8, body, 0); + while (it.next()) |prefixed_path| { + const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); + const sub_path = try arena.dupe(u8, prefixed_path[1..]); + const sub_path_dirname = std.fs.path.dirname(sub_path) orelse ""; + switch (prefix_index) { + .cwd => { + const path: Build.Cache.Path = .{ + .root_dir = Build.Cache.Directory.cwd(), + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .zig_lib => zl: { + if (s.cast(Step.Compile)) |compile| { + if (compile.zig_lib_dir) |lp| { + try addWatchInput(s, lp); + break :zl; + } + } + const path: Build.Cache.Path = .{ + .root_dir = s.owner.graph.zig_lib_directory, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + .local_cache => { + const path: Build.Cache.Path = .{ + .root_dir = b.cache_root, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, + } + } + }, else => {}, // ignore other messages } diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 7f8de00b4aeb..f1d3bc7b61d3 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -20,10 +20,24 @@ pub const Message = struct { test_metadata, /// Body is a TestResults test_results, + /// Body is a series of strings, delimited by null bytes. + /// Each string is a prefixed file path. + /// The first byte indicates the file prefix path (see prefixes fields + /// of Cache). This byte is sent over the wire incremented so that null + /// bytes are not confused with string terminators. + /// The remaining bytes is the file path relative to that prefix. + /// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir) + file_system_inputs, _, }; + pub const PathPrefix = enum(u8) { + cwd, + zig_lib, + local_cache, + }; + /// Trailing: /// * extra: [extra_len]u32, /// * string_bytes: [string_bytes_len]u8, @@ -58,7 +72,7 @@ pub const Message = struct { }; /// Trailing: - /// * the file system path the emitted binary can be found + /// * file system path where the emitted binary can be found pub const EmitBinPath = extern struct { flags: Flags, diff --git a/src/Compilation.zig b/src/Compilation.zig index cc5fd1a9eba2..49d4b041ae8b 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -235,6 +235,8 @@ astgen_wait_group: WaitGroup = .{}, llvm_opt_bisect_limit: c_int, +file_system_inputs: ?*std.ArrayListUnmanaged(u8), + pub const Emit = struct { /// Where the output will go. directory: Directory, @@ -1157,6 +1159,9 @@ pub const CreateOptions = struct { error_limit: ?Zcu.ErrorInt = null, global_cc_argv: []const []const u8 = &.{}, + /// Tracks all files that can cause the Compilation to be invalidated and need a rebuild. + file_system_inputs: ?*std.ArrayListUnmanaged(u8) = null, + pub const Entry = link.File.OpenOptions.Entry; }; @@ -1332,6 +1337,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil .gpa = gpa, .manifest_dir = try options.local_cache_directory.handle.makeOpenPath("h", .{}), }; + // These correspond to std.zig.Server.Message.PathPrefix. cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); cache.addPrefix(options.zig_lib_directory); cache.addPrefix(options.local_cache_directory); @@ -1508,6 +1514,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil .force_undefined_symbols = options.force_undefined_symbols, .link_eh_frame_hdr = link_eh_frame_hdr, .global_cc_argv = options.global_cc_argv, + .file_system_inputs = options.file_system_inputs, }; // Prevent some footguns by making the "any" fields of config reflect @@ -2044,6 +2051,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { ); }; if (is_hit) { + if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf); + comp.last_update_was_cache_hit = true; log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name}); const digest = man.final(); @@ -2170,6 +2179,11 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { try comp.performAllTheWork(main_progress_node); + switch (comp.cache_use) { + .whole => if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf), + .incremental => {}, + } + if (comp.module) |zcu| { const pt: Zcu.PerThread = .{ .zcu = zcu, .tid = .main }; diff --git a/src/main.zig b/src/main.zig index e00442f399e5..2fb49b74bcfa 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3227,6 +3227,9 @@ fn buildOutputType( process.raiseFileDescriptorLimit(); + var file_system_inputs: std.ArrayListUnmanaged(u8) = .{}; + defer file_system_inputs.deinit(gpa); + const comp = Compilation.create(gpa, arena, .{ .zig_lib_directory = zig_lib_directory, .local_cache_directory = local_cache_directory, @@ -3350,6 +3353,7 @@ fn buildOutputType( // than to any particular module. This feature can greatly reduce CLI // noise when --search-prefix and --mod are combined. .global_cc_argv = try cc_argv.toOwnedSlice(arena), + .file_system_inputs = &file_system_inputs, }) catch |err| switch (err) { error.LibCUnavailable => { const triple_name = try target.zigTriple(arena); @@ -3433,7 +3437,7 @@ fn buildOutputType( defer root_prog_node.end(); if (arg_mode == .translate_c) { - return cmdTranslateC(comp, arena, null, root_prog_node); + return cmdTranslateC(comp, arena, null, null, root_prog_node); } updateModule(comp, color, root_prog_node) catch |err| switch (err) { @@ -4059,6 +4063,7 @@ fn serve( var child_pid: ?std.process.Child.Id = null; const main_progress_node = std.Progress.start(.{}); + const file_system_inputs = comp.file_system_inputs.?; while (true) { const hdr = try server.receiveMessage(); @@ -4067,14 +4072,16 @@ fn serve( .exit => return cleanExit(), .update => { tracy.frameMark(); + file_system_inputs.clearRetainingCapacity(); if (arg_mode == .translate_c) { var arena_instance = std.heap.ArenaAllocator.init(gpa); defer arena_instance.deinit(); const arena = arena_instance.allocator(); var output: Compilation.CImportResult = undefined; - try cmdTranslateC(comp, arena, &output, main_progress_node); + try cmdTranslateC(comp, arena, &output, file_system_inputs, main_progress_node); defer output.deinit(gpa); + try server.serveStringMessage(.file_system_inputs, file_system_inputs.items); if (output.errors.errorMessageCount() != 0) { try server.serveErrorBundle(output.errors); } else { @@ -4116,6 +4123,7 @@ fn serve( }, .hot_update => { tracy.frameMark(); + file_system_inputs.clearRetainingCapacity(); if (child_pid) |pid| { try comp.hotCodeSwap(main_progress_node, pid); try serveUpdateResults(&server, comp); @@ -4147,6 +4155,12 @@ fn serve( fn serveUpdateResults(s: *Server, comp: *Compilation) !void { const gpa = comp.gpa; + + if (comp.file_system_inputs) |file_system_inputs| { + assert(file_system_inputs.items.len > 0); + try s.serveStringMessage(.file_system_inputs, file_system_inputs.items); + } + var error_bundle = try comp.getAllErrorsAlloc(); defer error_bundle.deinit(gpa); if (error_bundle.errorMessageCount() > 0) { @@ -4434,6 +4448,7 @@ fn cmdTranslateC( comp: *Compilation, arena: Allocator, fancy_output: ?*Compilation.CImportResult, + file_system_inputs: ?*std.ArrayListUnmanaged(u8), prog_node: std.Progress.Node, ) !void { if (build_options.only_core_functionality) @panic("@translate-c is not available in a zig2.c build"); @@ -4454,7 +4469,10 @@ fn cmdTranslateC( }; if (fancy_output) |p| p.cache_hit = true; - const digest = if (try man.hit()) man.final() else digest: { + const digest = if (try man.hit()) digest: { + if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf); + break :digest man.final(); + } else digest: { if (fancy_output) |p| p.cache_hit = false; var argv = std.ArrayList([]const u8).init(arena); switch (comp.config.c_frontend) { @@ -4566,6 +4584,8 @@ fn cmdTranslateC( @errorName(err), }); + if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf); + break :digest digest; }; @@ -4678,6 +4698,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { const self_exe_path = try introspect.findZigExePath(arena); try child_argv.append(self_exe_path); + const argv_index_zig_lib_dir = child_argv.items.len; + _ = try child_argv.addOne(); + const argv_index_build_file = child_argv.items.len; _ = try child_argv.addOne(); @@ -4727,7 +4750,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; override_lib_dir = args[i]; - try child_argv.appendSlice(&.{ arg, args[i] }); continue; } else if (mem.eql(u8, arg, "--build-runner")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); @@ -4865,6 +4887,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { defer zig_lib_directory.handle.close(); const cwd_path = try process.getCwdAlloc(arena); + child_argv.items[argv_index_zig_lib_dir] = zig_lib_directory.path orelse cwd_path; + const build_root = try findBuildRoot(arena, .{ .cwd_path = cwd_path, .build_file = build_file, From 5a34e6c3e608e1f526bababd3a2a146f6216d045 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 11 Jul 2024 18:28:05 -0700 Subject: [PATCH 34/38] frontend: add file system inputs for incremental cache mode These are also used for whole cache mode in the case that any compile errors are emitted. --- lib/std/Build/Cache.zig | 1 + src/Compilation.zig | 67 +++++++++++++++++++++++++++++++++++++---- src/Zcu.zig | 2 +- src/Zcu/PerThread.zig | 2 +- 4 files changed, 64 insertions(+), 8 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 8ba2f9f12899..fe2a6b0a830d 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -1010,6 +1010,7 @@ pub const Manifest = struct { pub fn populateFileSystemInputs(man: *Manifest, buf: *std.ArrayListUnmanaged(u8)) Allocator.Error!void { assert(@typeInfo(std.zig.Server.Message.PathPrefix).Enum.fields.len == man.cache.prefixes_len); + buf.clearRetainingCapacity(); const gpa = man.cache.gpa; const files = man.files.keys(); if (files.len > 0) { diff --git a/src/Compilation.zig b/src/Compilation.zig index 49d4b041ae8b..a0cf2378dad5 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2051,6 +2051,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { ); }; if (is_hit) { + // In this case the cache hit contains the full set of file system inputs. Nice! if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf); comp.last_update_was_cache_hit = true; @@ -2112,12 +2113,24 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { .incremental => {}, } + // From this point we add a preliminary set of file system inputs that + // affects both incremental and whole cache mode. For incremental cache + // mode, the long-lived compiler state will track additional file system + // inputs discovered after this point. For whole cache mode, we rely on + // these inputs to make it past AstGen, and once there, we can rely on + // learning file system inputs from the Cache object. + // For compiling C objects, we rely on the cache hash system to avoid duplicating work. // Add a Job for each C object. try comp.c_object_work_queue.ensureUnusedCapacity(comp.c_object_table.count()); for (comp.c_object_table.keys()) |key| { comp.c_object_work_queue.writeItemAssumeCapacity(key); } + if (comp.file_system_inputs) |fsi| { + for (comp.c_object_table.keys()) |c_object| { + try comp.appendFileSystemInput(fsi, c_object.src.owner.root, c_object.src.src_path); + } + } // For compiling Win32 resources, we rely on the cache hash system to avoid duplicating work. // Add a Job for each Win32 resource file. @@ -2126,6 +2139,12 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { for (comp.win32_resource_table.keys()) |key| { comp.win32_resource_work_queue.writeItemAssumeCapacity(key); } + if (comp.file_system_inputs) |fsi| { + for (comp.win32_resource_table.keys()) |win32_resource| switch (win32_resource.src) { + .rc => |f| try comp.appendFileSystemInput(fsi, f.owner.root, f.src_path), + .manifest => continue, + }; + } } if (comp.module) |zcu| { @@ -2160,12 +2179,24 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { if (zcu.fileByIndex(file_index).mod.isBuiltin()) continue; comp.astgen_work_queue.writeItemAssumeCapacity(file_index); } + if (comp.file_system_inputs) |fsi| { + for (zcu.import_table.values()) |file| { + try comp.appendFileSystemInput(fsi, file.mod.root, file.sub_file_path); + } + } // Put a work item in for checking if any files used with `@embedFile` changed. try comp.embed_file_work_queue.ensureUnusedCapacity(zcu.embed_table.count()); for (zcu.embed_table.values()) |embed_file| { comp.embed_file_work_queue.writeItemAssumeCapacity(embed_file); } + if (comp.file_system_inputs) |fsi| { + const ip = &zcu.intern_pool; + for (zcu.embed_table.values()) |embed_file| { + const sub_file_path = embed_file.sub_file_path.toSlice(ip); + try comp.appendFileSystemInput(fsi, embed_file.owner.root, sub_file_path); + } + } try comp.work_queue.writeItem(.{ .analyze_mod = std_mod }); if (comp.config.is_test) { @@ -2179,11 +2210,6 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { try comp.performAllTheWork(main_progress_node); - switch (comp.cache_use) { - .whole => if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf), - .incremental => {}, - } - if (comp.module) |zcu| { const pt: Zcu.PerThread = .{ .zcu = zcu, .tid = .main }; @@ -2224,6 +2250,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { switch (comp.cache_use) { .whole => |whole| { + if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf); + const digest = man.final(); // Rename the temporary directory into place. @@ -2311,6 +2339,30 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { } } +fn appendFileSystemInput( + comp: *Compilation, + file_system_inputs: *std.ArrayListUnmanaged(u8), + root: Cache.Path, + sub_file_path: []const u8, +) Allocator.Error!void { + const gpa = comp.gpa; + const prefixes = comp.cache_parent.prefixes(); + try file_system_inputs.ensureUnusedCapacity(gpa, root.sub_path.len + sub_file_path.len + 3); + if (file_system_inputs.items.len > 0) file_system_inputs.appendAssumeCapacity(0); + for (prefixes, 1..) |prefix_directory, i| { + if (prefix_directory.eql(root.root_dir)) { + file_system_inputs.appendAssumeCapacity(@intCast(i)); + if (root.sub_path.len > 0) { + file_system_inputs.appendSliceAssumeCapacity(root.sub_path); + file_system_inputs.appendAssumeCapacity(std.fs.path.sep); + } + file_system_inputs.appendSliceAssumeCapacity(sub_file_path); + return; + } + } + std.debug.panic("missing prefix directory: {}, {s}", .{ root, sub_file_path }); +} + fn flush(comp: *Compilation, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: std.Progress.Node) !void { if (comp.bin_file) |lf| { // This is needed before reading the error flags. @@ -4218,6 +4270,9 @@ fn workerAstGenFile( .token = item.data.token, } }) catch continue; } + if (res.is_new) if (comp.file_system_inputs) |fsi| { + comp.appendFileSystemInput(fsi, res.file.mod.root, res.file.sub_file_path) catch continue; + }; const imported_path_digest = pt.zcu.filePathDigest(res.file_index); const imported_root_decl = pt.zcu.fileRootDecl(res.file_index); break :blk .{ res, imported_path_digest, imported_root_decl }; @@ -4588,7 +4643,7 @@ fn reportRetryableEmbedFileError( const gpa = mod.gpa; const src_loc = embed_file.src_loc; const ip = &mod.intern_pool; - const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{ + const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}/{s}': {s}", .{ embed_file.owner.root, embed_file.sub_file_path.toSlice(ip), @errorName(err), diff --git a/src/Zcu.zig b/src/Zcu.zig index a9d80b4fdf04..fd1e2f9d612f 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -728,7 +728,7 @@ pub const File = struct { source_loaded: bool, tree_loaded: bool, zir_loaded: bool, - /// Relative to the owning package's root_src_dir. + /// Relative to the owning package's root source directory. /// Memory is stored in gpa, owned by File. sub_file_path: []const u8, /// Whether this is populated depends on `source_loaded`. diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index 4a1f257ddfe2..f6a47f626b13 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -2666,7 +2666,7 @@ pub fn reportRetryableAstGenError( }, }; - const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{ + const err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "unable to load '{}/{s}': {s}", .{ file.mod.root, file.sub_file_path, @errorName(err), }); errdefer err_msg.destroy(gpa); From 67e3e4989d5dc654ce5fef87d5a974caa05b9733 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 12 Jul 2024 00:18:29 -0700 Subject: [PATCH 35/38] Compilation: fix rebase conflict --- src/Compilation.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index a0cf2378dad5..94ce0ee26777 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2180,7 +2180,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { comp.astgen_work_queue.writeItemAssumeCapacity(file_index); } if (comp.file_system_inputs) |fsi| { - for (zcu.import_table.values()) |file| { + for (zcu.import_table.values()) |file_index| { + const file = zcu.fileByIndex(file_index); try comp.appendFileSystemInput(fsi, file.mod.root, file.sub_file_path); } } From 4f9a8b68430b9b44cba78664bd0f60d5c5db5fe3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 12 Jul 2024 11:00:52 -0700 Subject: [PATCH 36/38] update build system unit test need to add another field to initialize now --- lib/std/Build/Step/Options.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index 9ce23e0802a6..b67acd408697 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -523,6 +523,7 @@ test Options { .query = .{}, .result = try std.zig.system.resolveTargetQuery(.{}), }, + .zig_lib_directory = std.Build.Cache.Directory.cwd(), }; var builder = try std.Build.create( From f77b43dad39a5140f3e39f32e98e9624368c16d6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 12 Jul 2024 14:19:17 -0700 Subject: [PATCH 37/38] zig build: add a --debug-target CLI flag it's not advertised in the usage and only available in debug builds of the compiler. Makes it easier to test changes to the build runner that might affect targets differently. --- src/main.zig | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/main.zig b/src/main.zig index 2fb49b74bcfa..0d942ea6c453 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4691,6 +4691,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { var verbose_llvm_cpu_features = false; var fetch_only = false; var system_pkg_dir_path: ?[]const u8 = null; + var debug_target: ?[]const u8 = null; const argv_index_exe = child_argv.items.len; _ = try child_argv.addOne(); @@ -4799,6 +4800,14 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } else { warn("Zig was compiled without debug extensions. --debug-compile-errors has no effect.", .{}); } + } else if (mem.eql(u8, arg, "--debug-target")) { + if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); + i += 1; + if (build_options.enable_debug_extensions) { + debug_target = args[i]; + } else { + warn("Zig was compiled without debug extensions. --debug-target has no effect.", .{}); + } } else if (mem.eql(u8, arg, "--verbose-link")) { verbose_link = true; } else if (mem.eql(u8, arg, "--verbose-cc")) { @@ -4857,11 +4866,27 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { }); defer root_prog_node.end(); - const target_query: std.Target.Query = .{}; - const resolved_target: Package.Module.ResolvedTarget = .{ - .result = std.zig.resolveTargetQueryOrFatal(target_query), - .is_native_os = true, - .is_native_abi = true, + // Normally the build runner is compiled for the host target but here is + // some code to help when debugging edits to the build runner so that you + // can make sure it compiles successfully on other targets. + const resolved_target: Package.Module.ResolvedTarget = t: { + if (build_options.enable_debug_extensions) { + if (debug_target) |triple| { + const target_query = try std.Target.Query.parse(.{ + .arch_os_abi = triple, + }); + break :t .{ + .result = std.zig.resolveTargetQueryOrFatal(target_query), + .is_native_os = false, + .is_native_abi = false, + }; + } + } + break :t .{ + .result = std.zig.resolveTargetQueryOrFatal(.{}), + .is_native_os = true, + .is_native_abi = true, + }; }; const exe_basename = try std.zig.binNameAlloc(arena, .{ From 5efcc2e9e7c84893b9e418ca82d8d2d4366dde7c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 12 Jul 2024 14:20:20 -0700 Subject: [PATCH 38/38] build runner: refactor fs watch logic for OS abstraction Makes the build runner compile successfully for non-linux targets; printing an error if you ask for --watch rather than making build scripts fail to compile. --- lib/compiler/build_runner.zig | 151 +++--------- lib/std/Build/Watch.zig | 432 +++++++++++++++++++++++----------- 2 files changed, 328 insertions(+), 255 deletions(-) diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 384c16438046..703571eb56a6 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -10,6 +10,7 @@ const File = std.fs.File; const Step = std.Build.Step; const Watch = std.Build.Watch; const Allocator = std.mem.Allocator; +const fatal = std.zig.fatal; pub const root = @import("@build"); pub const dependencies = @import("@dependencies"); @@ -371,18 +372,7 @@ pub fn main() !void { else => return err, }; - var w = Watch.init; - if (watch) { - w.fan_fd = try std.posix.fanotify_init(.{ - .CLASS = .NOTIF, - .CLOEXEC = true, - .NONBLOCK = true, - .REPORT_NAME = true, - .REPORT_DIR_FID = true, - .REPORT_FID = true, - .REPORT_TARGET_FID = true, - }, 0); - } + var w = if (watch) try Watch.init() else undefined; try run.thread_pool.init(thread_pool_options); defer run.thread_pool.deinit(); @@ -403,127 +393,53 @@ pub fn main() !void { }; if (!watch) return cleanExit(); - // Add missing marks and note persisted ones. - for (run.step_stack.keys()) |step| { - for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { - const reaction_set = rs: { - const gop = try w.dir_table.getOrPut(gpa, path); - if (!gop.found_existing) { - const dir_handle = try Watch.getDirHandle(gpa, path); - // `dir_handle` may already be present in the table in - // the case that we have multiple Cache.Path instances - // that compare inequal but ultimately point to the same - // directory on the file system. - // In such case, we must revert adding this directory, but keep - // the additions to the step set. - const dh_gop = try w.handle_table.getOrPut(gpa, dir_handle); - if (dh_gop.found_existing) { - _ = w.dir_table.pop(); - } else { - assert(dh_gop.index == gop.index); - dh_gop.value_ptr.* = .{}; - std.posix.fanotify_mark(w.fan_fd, .{ - .ADD = true, - .ONLYDIR = true, - }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOrDot()) catch |err| { - fatal("unable to watch {}: {s}", .{ path, @errorName(err) }); - }; - } - break :rs dh_gop.value_ptr; - } - break :rs &w.handle_table.values()[gop.index]; - }; - for (files.items) |basename| { - const gop = try reaction_set.getOrPut(gpa, basename); - if (!gop.found_existing) gop.value_ptr.* = .{}; - try gop.value_ptr.put(gpa, step, w.generation); - } - } + switch (builtin.os.tag) { + .linux => {}, + else => fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)}), } - { - // Remove marks for files that are no longer inputs. - var i: usize = 0; - while (i < w.handle_table.entries.len) { - { - const reaction_set = &w.handle_table.values()[i]; - var step_set_i: usize = 0; - while (step_set_i < reaction_set.entries.len) { - const step_set = &reaction_set.values()[step_set_i]; - var dirent_i: usize = 0; - while (dirent_i < step_set.entries.len) { - const generations = step_set.values(); - if (generations[dirent_i] == w.generation) { - dirent_i += 1; - continue; - } - step_set.swapRemoveAt(dirent_i); - } - if (step_set.entries.len > 0) { - step_set_i += 1; - continue; - } - reaction_set.swapRemoveAt(step_set_i); - } - if (reaction_set.entries.len > 0) { - i += 1; - continue; - } - } - - const path = w.dir_table.keys()[i]; - - std.posix.fanotify_mark(w.fan_fd, .{ - .REMOVE = true, - .ONLYDIR = true, - }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOrDot()) catch |err| switch (err) { - error.FileNotFound => {}, // Expected, harmless. - else => |e| std.log.warn("unable to unwatch '{}': {s}", .{ path, @errorName(e) }), - }; - - w.dir_table.swapRemoveAt(i); - w.handle_table.swapRemoveAt(i); - } - w.generation +%= 1; - } + try w.update(gpa, run.step_stack.keys()); // Wait until a file system notification arrives. Read all such events // until the buffer is empty. Then wait for a debounce interval, resetting // if any more events come in. After the debounce interval has passed, // trigger a rebuild on all steps with modified inputs, as well as their // recursive dependants. - var poll_fds: [1]std.posix.pollfd = .{ - .{ - .fd = w.fan_fd, - .events = std.posix.POLL.IN, - .revents = undefined, - }, - }; var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; const caption = std.fmt.bufPrint(&caption_buf, "Watching {d} Directories", .{ w.dir_table.entries.len, }) catch &caption_buf; var debouncing_node = main_progress_node.start(caption, 0); - var debouncing = false; - while (true) { - const timeout: i32 = if (debouncing) debounce_interval_ms else -1; - const events_len = try std.posix.poll(&poll_fds, timeout); - if (events_len == 0) { + var debounce_timeout: Watch.Timeout = .none; + while (true) switch (try w.wait(gpa, debounce_timeout)) { + .timeout => { debouncing_node.end(); - Watch.markFailedStepsDirty(gpa, run.step_stack.keys()); + markFailedStepsDirty(gpa, run.step_stack.keys()); continue :rebuild; - } - if (try w.markDirtySteps(gpa)) { - if (!debouncing) { - debouncing = true; - debouncing_node.end(); - debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); - } - } - } + }, + .dirty => if (debounce_timeout == .none) { + debounce_timeout = .{ .ms = debounce_interval_ms }; + debouncing_node.end(); + debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); + }, + .clean => {}, + }; } } +fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { + for (all_steps) |step| switch (step.state) { + .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), + else => continue, + }; + // Now that all dirty steps have been found, the remaining steps that + // succeeded from last run shall be marked "cached". + for (all_steps) |step| switch (step.state) { + .success => step.result_cached = true, + else => continue, + }; +} + const Run = struct { max_rss: u64, max_rss_is_default: bool, @@ -1430,11 +1346,6 @@ fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { process.exit(1); } -fn fatal(comptime f: []const u8, args: anytype) noreturn { - std.debug.print(f ++ "\n", args); - process.exit(1); -} - fn validateSystemLibraryOptions(b: *std.Build) void { var bad = false; for (b.graph.system_library_options.keys(), b.graph.system_library_options.values()) |k, v| { diff --git a/lib/std/Build/Watch.zig b/lib/std/Build/Watch.zig index e4bef7ca4ef6..534e9814de24 100644 --- a/lib/std/Build/Watch.zig +++ b/lib/std/Build/Watch.zig @@ -1,41 +1,21 @@ +const builtin = @import("builtin"); const std = @import("../std.zig"); const Watch = @This(); const Step = std.Build.Step; const Allocator = std.mem.Allocator; const assert = std.debug.assert; +const fatal = std.zig.fatal; dir_table: DirTable, -/// Keyed differently but indexes correspond 1:1 with `dir_table`. -handle_table: HandleTable, -fan_fd: std.posix.fd_t, +os: Os, generation: Generation, -pub const fan_mask: std.os.linux.fanotify.MarkMask = .{ - .CLOSE_WRITE = true, - .CREATE = true, - .DELETE = true, - .DELETE_SELF = true, - .EVENT_ON_CHILD = true, - .MOVED_FROM = true, - .MOVED_TO = true, - .MOVE_SELF = true, - .ONDIR = true, -}; - -pub const init: Watch = .{ - .dir_table = .{}, - .handle_table = .{}, - .fan_fd = -1, - .generation = 0, -}; - /// Key is the directory to watch which contains one or more files we are /// interested in noticing changes to. /// /// Value is generation. const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAdapter, false); -const HandleTable = std.ArrayHashMapUnmanaged(LinuxFileHandle, ReactionSet, LinuxFileHandle.Adapter, false); /// Special key of "." means any changes in this directory trigger the steps. const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet); const StepSet = std.AutoArrayHashMapUnmanaged(*Step, Generation); @@ -45,6 +25,255 @@ const Generation = u8; const Hash = std.hash.Wyhash; const Cache = std.Build.Cache; +const Os = switch (builtin.os.tag) { + .linux => struct { + const posix = std.posix; + + /// Keyed differently but indexes correspond 1:1 with `dir_table`. + handle_table: HandleTable, + poll_fds: [1]posix.pollfd, + + const HandleTable = std.ArrayHashMapUnmanaged(FileHandle, ReactionSet, FileHandle.Adapter, false); + + const fan_mask: std.os.linux.fanotify.MarkMask = .{ + .CLOSE_WRITE = true, + .CREATE = true, + .DELETE = true, + .DELETE_SELF = true, + .EVENT_ON_CHILD = true, + .MOVED_FROM = true, + .MOVED_TO = true, + .MOVE_SELF = true, + .ONDIR = true, + }; + + const FileHandle = struct { + handle: *align(1) std.os.linux.file_handle, + + fn clone(lfh: FileHandle, gpa: Allocator) Allocator.Error!FileHandle { + const bytes = lfh.slice(); + const new_ptr = try gpa.alignedAlloc( + u8, + @alignOf(std.os.linux.file_handle), + @sizeOf(std.os.linux.file_handle) + bytes.len, + ); + const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr); + new_header.* = lfh.handle.*; + const new: FileHandle = .{ .handle = new_header }; + @memcpy(new.slice(), lfh.slice()); + return new; + } + + fn destroy(lfh: FileHandle, gpa: Allocator) void { + const ptr: [*]u8 = @ptrCast(lfh.handle); + const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes]; + return gpa.free(allocated_slice); + } + + fn slice(lfh: FileHandle) []u8 { + const ptr: [*]u8 = &lfh.handle.f_handle; + return ptr[0..lfh.handle.handle_bytes]; + } + + const Adapter = struct { + pub fn hash(self: Adapter, a: FileHandle) u32 { + _ = self; + const unsigned_type: u32 = @bitCast(a.handle.handle_type); + return @truncate(Hash.hash(unsigned_type, a.slice())); + } + pub fn eql(self: Adapter, a: FileHandle, b: FileHandle, b_index: usize) bool { + _ = self; + _ = b_index; + return a.handle.handle_type == b.handle.handle_type and std.mem.eql(u8, a.slice(), b.slice()); + } + }; + }; + + fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path) !FileHandle { + var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; + var mount_id: i32 = undefined; + var buf: [std.fs.max_path_bytes]u8 = undefined; + const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{ + path.sub_path, + }) catch return error.NameTooLong; + const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); + stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); + try posix.name_to_handle_at(path.root_dir.handle.fd, adjusted_path, stack_ptr, &mount_id, std.os.linux.AT.HANDLE_FID); + const stack_lfh: FileHandle = .{ .handle = stack_ptr }; + return stack_lfh.clone(gpa); + } + + fn markDirtySteps(w: *Watch, gpa: Allocator) !bool { + const fan_fd = w.os.getFanFd(); + const fanotify = std.os.linux.fanotify; + const M = fanotify.event_metadata; + var events_buf: [256 + 4096]u8 = undefined; + var any_dirty = false; + while (true) { + var len = posix.read(fan_fd, &events_buf) catch |err| switch (err) { + error.WouldBlock => return any_dirty, + else => |e| return e, + }; + var meta: [*]align(1) M = @ptrCast(&events_buf); + while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({ + len -= meta[0].event_len; + meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); + }) { + assert(meta[0].vers == M.VERSION); + if (meta[0].mask.Q_OVERFLOW) { + any_dirty = true; + std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); + markAllFilesDirty(w, gpa); + return true; + } + const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); + switch (fid.hdr.info_type) { + .DFID_NAME => { + const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); + const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes); + const file_name = std.mem.span(file_name_z); + const lfh: FileHandle = .{ .handle = file_handle }; + if (w.os.handle_table.getPtr(lfh)) |reaction_set| { + if (reaction_set.getPtr(".")) |glob_set| + any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); + if (reaction_set.getPtr(file_name)) |step_set| + any_dirty = markStepSetDirty(gpa, step_set, any_dirty); + } + }, + else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}), + } + } + } + } + + fn getFanFd(os: *const @This()) posix.fd_t { + return os.poll_fds[0].fd; + } + + fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { + const fan_fd = w.os.getFanFd(); + // Add missing marks and note persisted ones. + for (steps) |step| { + for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| { + const reaction_set = rs: { + const gop = try w.dir_table.getOrPut(gpa, path); + if (!gop.found_existing) { + const dir_handle = try Os.getDirHandle(gpa, path); + // `dir_handle` may already be present in the table in + // the case that we have multiple Cache.Path instances + // that compare inequal but ultimately point to the same + // directory on the file system. + // In such case, we must revert adding this directory, but keep + // the additions to the step set. + const dh_gop = try w.os.handle_table.getOrPut(gpa, dir_handle); + if (dh_gop.found_existing) { + _ = w.dir_table.pop(); + } else { + assert(dh_gop.index == gop.index); + dh_gop.value_ptr.* = .{}; + posix.fanotify_mark(fan_fd, .{ + .ADD = true, + .ONLYDIR = true, + }, fan_mask, path.root_dir.handle.fd, path.subPathOrDot()) catch |err| { + fatal("unable to watch {}: {s}", .{ path, @errorName(err) }); + }; + } + break :rs dh_gop.value_ptr; + } + break :rs &w.os.handle_table.values()[gop.index]; + }; + for (files.items) |basename| { + const gop = try reaction_set.getOrPut(gpa, basename); + if (!gop.found_existing) gop.value_ptr.* = .{}; + try gop.value_ptr.put(gpa, step, w.generation); + } + } + } + + { + // Remove marks for files that are no longer inputs. + var i: usize = 0; + while (i < w.os.handle_table.entries.len) { + { + const reaction_set = &w.os.handle_table.values()[i]; + var step_set_i: usize = 0; + while (step_set_i < reaction_set.entries.len) { + const step_set = &reaction_set.values()[step_set_i]; + var dirent_i: usize = 0; + while (dirent_i < step_set.entries.len) { + const generations = step_set.values(); + if (generations[dirent_i] == w.generation) { + dirent_i += 1; + continue; + } + step_set.swapRemoveAt(dirent_i); + } + if (step_set.entries.len > 0) { + step_set_i += 1; + continue; + } + reaction_set.swapRemoveAt(step_set_i); + } + if (reaction_set.entries.len > 0) { + i += 1; + continue; + } + } + + const path = w.dir_table.keys()[i]; + + posix.fanotify_mark(fan_fd, .{ + .REMOVE = true, + .ONLYDIR = true, + }, fan_mask, path.root_dir.handle.fd, path.subPathOrDot()) catch |err| switch (err) { + error.FileNotFound => {}, // Expected, harmless. + else => |e| std.log.warn("unable to unwatch '{}': {s}", .{ path, @errorName(e) }), + }; + + w.dir_table.swapRemoveAt(i); + w.os.handle_table.swapRemoveAt(i); + } + w.generation +%= 1; + } + } + }, + else => void, +}; + +pub fn init() !Watch { + switch (builtin.os.tag) { + .linux => { + const fan_fd = try std.posix.fanotify_init(.{ + .CLASS = .NOTIF, + .CLOEXEC = true, + .NONBLOCK = true, + .REPORT_NAME = true, + .REPORT_DIR_FID = true, + .REPORT_FID = true, + .REPORT_TARGET_FID = true, + }, 0); + return .{ + .dir_table = .{}, + .os = switch (builtin.os.tag) { + .linux => .{ + .handle_table = .{}, + .poll_fds = .{ + .{ + .fd = fan_fd, + .events = std.posix.POLL.IN, + .revents = undefined, + }, + }, + }, + else => {}, + }, + .generation = 0, + }; + }, + else => @panic("unimplemented"), + } +} + pub const Match = struct { /// Relative to the watched directory, the file path that triggers this /// match. @@ -68,119 +297,8 @@ pub const Match = struct { }; }; -pub const LinuxFileHandle = struct { - handle: *align(1) std.os.linux.file_handle, - - pub fn clone(lfh: LinuxFileHandle, gpa: Allocator) Allocator.Error!LinuxFileHandle { - const bytes = lfh.slice(); - const new_ptr = try gpa.alignedAlloc( - u8, - @alignOf(std.os.linux.file_handle), - @sizeOf(std.os.linux.file_handle) + bytes.len, - ); - const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr); - new_header.* = lfh.handle.*; - const new: LinuxFileHandle = .{ .handle = new_header }; - @memcpy(new.slice(), lfh.slice()); - return new; - } - - pub fn destroy(lfh: LinuxFileHandle, gpa: Allocator) void { - const ptr: [*]u8 = @ptrCast(lfh.handle); - const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes]; - return gpa.free(allocated_slice); - } - - pub fn slice(lfh: LinuxFileHandle) []u8 { - const ptr: [*]u8 = &lfh.handle.f_handle; - return ptr[0..lfh.handle.handle_bytes]; - } - - pub const Adapter = struct { - pub fn hash(self: Adapter, a: LinuxFileHandle) u32 { - _ = self; - const unsigned_type: u32 = @bitCast(a.handle.handle_type); - return @truncate(Hash.hash(unsigned_type, a.slice())); - } - pub fn eql(self: Adapter, a: LinuxFileHandle, b: LinuxFileHandle, b_index: usize) bool { - _ = self; - _ = b_index; - return a.handle.handle_type == b.handle.handle_type and std.mem.eql(u8, a.slice(), b.slice()); - } - }; -}; - -pub fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path) !LinuxFileHandle { - var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined; - var mount_id: i32 = undefined; - var buf: [std.fs.max_path_bytes]u8 = undefined; - const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{ - path.sub_path, - }) catch return error.NameTooLong; - const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer); - stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle); - try std.posix.name_to_handle_at(path.root_dir.handle.fd, adjusted_path, stack_ptr, &mount_id, std.os.linux.AT.HANDLE_FID); - const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr }; - return stack_lfh.clone(gpa); -} - -pub fn markDirtySteps(w: *Watch, gpa: Allocator) !bool { - const fanotify = std.os.linux.fanotify; - const M = fanotify.event_metadata; - var events_buf: [256 + 4096]u8 = undefined; - var any_dirty = false; - while (true) { - var len = std.posix.read(w.fan_fd, &events_buf) catch |err| switch (err) { - error.WouldBlock => return any_dirty, - else => |e| return e, - }; - var meta: [*]align(1) M = @ptrCast(&events_buf); - while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({ - len -= meta[0].event_len; - meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len); - }) { - assert(meta[0].vers == M.VERSION); - if (meta[0].mask.Q_OVERFLOW) { - any_dirty = true; - std.log.warn("file system watch queue overflowed; falling back to fstat", .{}); - markAllFilesDirty(w, gpa); - return true; - } - const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1); - switch (fid.hdr.info_type) { - .DFID_NAME => { - const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle); - const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes); - const file_name = std.mem.span(file_name_z); - const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle }; - if (w.handle_table.getPtr(lfh)) |reaction_set| { - if (reaction_set.getPtr(".")) |glob_set| - any_dirty = markStepSetDirty(gpa, glob_set, any_dirty); - if (reaction_set.getPtr(file_name)) |step_set| - any_dirty = markStepSetDirty(gpa, step_set, any_dirty); - } - }, - else => |t| std.log.warn("unexpected fanotify event '{s}'", .{@tagName(t)}), - } - } - } -} - -pub fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { - for (all_steps) |step| switch (step.state) { - .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), - else => continue, - }; - // Now that all dirty steps have been found, the remaining steps that - // succeeded from last run shall be marked "cached". - for (all_steps) |step| switch (step.state) { - .success => step.result_cached = true, - else => continue, - }; -} - fn markAllFilesDirty(w: *Watch, gpa: Allocator) void { - for (w.handle_table.values()) |reaction_set| { + for (w.os.handle_table.values()) |reaction_set| { for (reaction_set.values()) |step_set| { for (step_set.keys()) |step| { step.recursiveReset(gpa); @@ -199,3 +317,47 @@ fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool { } return any_dirty or this_any_dirty; } + +pub fn update(w: *Watch, gpa: Allocator, steps: []const *Step) !void { + switch (builtin.os.tag) { + .linux => return Os.update(w, gpa, steps), + else => @compileError("unimplemented"), + } +} + +pub const Timeout = union(enum) { + none, + ms: u16, + + pub fn to_i32_ms(t: Timeout) i32 { + return switch (t) { + .none => -1, + .ms => |ms| ms, + }; + } +}; + +pub const WaitResult = enum { + timeout, + /// File system watching triggered on files that were marked as inputs to at least one Step. + /// Relevant steps have been marked dirty. + dirty, + /// File system watching triggered but none of the events were relevant to + /// what we are listening to. There is nothing to do. + clean, +}; + +pub fn wait(w: *Watch, gpa: Allocator, timeout: Timeout) !WaitResult { + switch (builtin.os.tag) { + .linux => { + const events_len = try std.posix.poll(&w.os.poll_fds, timeout.to_i32_ms()); + return if (events_len == 0) + .timeout + else if (try Os.markDirtySteps(w, gpa)) + .dirty + else + .clean; + }, + else => @compileError("unimplemented"), + } +}