From 892ce7ef527c863d84f3085f79f1a5aec7161c2c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Sep 2024 19:36:52 -0700 Subject: [PATCH 1/7] rework fuzzing API The previous API used `std.testing.fuzzInput(.{})` however that has the problem that users call it multiple times incorrectly, and there might be work happening to obtain the corpus which should not be included in coverage analysis, and which must not slow down iteration speed. This commit restructures it so that the main loop lives in libfuzzer and directly calls the "test one" function. In this commit I was a little too aggressive because I made the test runner export `fuzzer_one` for this purpose. This was motivated by performance, but it causes "exported symbol collision: fuzzer_one" to occur when more than one fuzz test is provided. There are three ways to solve this: 1. libfuzzer needs to be passed a function pointer instead. Possible performance downside. 2. build runner needs to build a different process per fuzz test. Potentially wasteful and unclear how to isolate them. 3. test runner needs to perform a relocation at runtime to point the function call to the relevant unit test. Portability issues and dubious performance gains. --- lib/compiler/test_runner.zig | 106 +++++++++++++++++++++++++---------- lib/fuzzer.zig | 84 ++++++++++++++------------- lib/std/testing.zig | 8 ++- lib/std/zig/tokenizer.zig | 7 ++- 4 files changed, 131 insertions(+), 74 deletions(-) diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index ac9629a57dd6..2468c12645cc 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -145,31 +145,27 @@ fn mainServer() !void { .start_fuzzing => { if (!builtin.fuzz) unreachable; const index = try server.receiveBody_u32(); - var first = true; const test_fn = builtin.test_functions[index]; - while (true) { - testing.allocator_instance = .{}; - defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); - log_err_count = 0; - is_fuzz_test = false; - test_fn.func() catch |err| switch (err) { - error.SkipZigTest => continue, - else => { - if (@errorReturnTrace()) |trace| { - std.debug.dumpStackTrace(trace.*); - } - std.debug.print("failed with error.{s}\n", .{@errorName(err)}); - std.process.exit(1); - }, - }; - if (!is_fuzz_test) @panic("missed call to std.testing.fuzzInput"); - if (log_err_count != 0) @panic("error logs detected"); - if (first) { - first = false; - const entry_addr = @intFromPtr(test_fn.func); - try server.serveU64Message(.fuzz_start_addr, entry_addr); - } + const entry_addr = @intFromPtr(test_fn.func); + try server.serveU64Message(.fuzz_start_addr, entry_addr); + const prev_allocator_state = testing.allocator_instance; + defer { + testing.allocator_instance = prev_allocator_state; + if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); } + is_fuzz_test = false; + test_fn.func() catch |err| switch (err) { + error.SkipZigTest => return, + else => { + if (@errorReturnTrace()) |trace| { + std.debug.dumpStackTrace(trace.*); + } + std.debug.print("failed with error.{s}\n", .{@errorName(err)}); + std.process.exit(1); + }, + }; + if (!is_fuzz_test) @panic("missed call to std.testing.fuzz"); + if (log_err_count != 0) @panic("error logs detected"); }, else => { @@ -349,19 +345,67 @@ const FuzzerSlice = extern struct { var is_fuzz_test: bool = undefined; -extern fn fuzzer_next() FuzzerSlice; +extern fn fuzzer_start() void; extern fn fuzzer_init(cache_dir: FuzzerSlice) void; extern fn fuzzer_coverage_id() u64; -pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 { +pub fn fuzz( + comptime testOne: fn ([]const u8) anyerror!void, + options: testing.FuzzInputOptions, +) anyerror!void { + // Prevent this function from confusing the fuzzer by omitting its own code + // coverage from being considered. @disableInstrumentation(); - if (crippled) return ""; + + // Some compiler backends are not capable of handling fuzz testing yet but + // we still want CI test coverage enabled. + if (crippled) return; + + // Smoke test to ensure the test did not use conditional compilation to + // contradict itself by making it not actually be a fuzz test when the test + // is built in fuzz mode. is_fuzz_test = true; + + // Ensure no test failure occurred before starting fuzzing. + if (log_err_count != 0) @panic("error logs detected"); + + // libfuzzer is in a separate compilation unit so that its own code can be + // excluded from code coverage instrumentation. It needs a function pointer + // it can call for checking exactly one input. Inside this function we do + // our standard unit test checks such as memory leaks, and interaction with + // error logs. + const global = struct { + fn fuzzer_one(input_ptr: [*]const u8, input_len: usize) callconv(.C) void { + @disableInstrumentation(); + testing.allocator_instance = .{}; + defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); + log_err_count = 0; + testOne(input_ptr[0..input_len]) catch |err| switch (err) { + error.SkipZigTest => return, + else => { + if (@errorReturnTrace()) |trace| { + std.debug.dumpStackTrace(trace.*); + } + std.debug.print("failed with error.{s}\n", .{@errorName(err)}); + std.process.exit(1); + }, + }; + if (log_err_count != 0) @panic("error logs detected"); + } + }; if (builtin.fuzz) { - return fuzzer_next().toSlice(); + @export(&global.fuzzer_one, .{ .name = "fuzzer_one" }); + fuzzer_start(); + return; + } + + // When the unit test executable is not built in fuzz mode, only run the + // provided corpus. + for (options.corpus) |input| { + try testOne(input); } - if (options.corpus.len == 0) return ""; - var prng = std.Random.DefaultPrng.init(testing.random_seed); - const random = prng.random(); - return options.corpus[random.uintLessThan(usize, options.corpus.len)]; + + // In case there is no provided corpus, also use an empty + // string as a smoke test. + try testOne(""); } diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 9c67756a6d97..2aa974427589 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -235,22 +235,41 @@ const Fuzzer = struct { }; } - fn next(f: *Fuzzer) ![]const u8 { + fn start(f: *Fuzzer) !void { const gpa = f.gpa; const rng = fuzzer.rng.random(); - if (f.recent_cases.entries.len == 0) { - // Prepare initial input. - try f.recent_cases.ensureUnusedCapacity(gpa, 100); - const len = rng.uintLessThanBiased(usize, 80); - try f.input.resize(gpa, len); - rng.bytes(f.input.items); - f.recent_cases.putAssumeCapacity(.{ - .id = 0, - .input = try gpa.dupe(u8, f.input.items), - .score = 0, - }, {}); - } else { + // Prepare initial input. + assert(f.recent_cases.entries.len == 0); + assert(f.n_runs == 0); + try f.recent_cases.ensureUnusedCapacity(gpa, 100); + const len = rng.uintLessThanBiased(usize, 80); + try f.input.resize(gpa, len); + rng.bytes(f.input.items); + f.recent_cases.putAssumeCapacity(.{ + .id = 0, + .input = try gpa.dupe(u8, f.input.items), + .score = 0, + }, {}); + + const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]); + + while (true) { + const chosen_index = rng.uintLessThanBiased(usize, f.recent_cases.entries.len); + const run = &f.recent_cases.keys()[chosen_index]; + f.input.clearRetainingCapacity(); + f.input.appendSliceAssumeCapacity(run.input); + try f.mutate(); + + _ = @atomicRmw(usize, &header.lowest_stack, .Min, __sancov_lowest_stack, .monotonic); + @memset(f.pc_counters, 0); + f.coverage.reset(); + + fuzzer_one(f.input.items.ptr, f.input.items.len); + + f.n_runs += 1; + _ = @atomicRmw(usize, &header.n_runs, .Add, 1, .monotonic); + if (f.n_runs % 10000 == 0) f.dumpStats(); const analysis = f.analyzeLastRun(); @@ -301,7 +320,6 @@ const Fuzzer = struct { } } - const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]); _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic); } @@ -317,26 +335,12 @@ const Fuzzer = struct { // This has to be done before deinitializing the deleted items. const doomed_runs = f.recent_cases.keys()[cap..]; f.recent_cases.shrinkRetainingCapacity(cap); - for (doomed_runs) |*run| { - std.log.info("culling score={d} id={d}", .{ run.score, run.id }); - run.deinit(gpa); + for (doomed_runs) |*doomed_run| { + std.log.info("culling score={d} id={d}", .{ doomed_run.score, doomed_run.id }); + doomed_run.deinit(gpa); } } } - - const chosen_index = rng.uintLessThanBiased(usize, f.recent_cases.entries.len); - const run = &f.recent_cases.keys()[chosen_index]; - f.input.clearRetainingCapacity(); - f.input.appendSliceAssumeCapacity(run.input); - try f.mutate(); - - f.n_runs += 1; - const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]); - _ = @atomicRmw(usize, &header.n_runs, .Add, 1, .monotonic); - _ = @atomicRmw(usize, &header.lowest_stack, .Min, __sancov_lowest_stack, .monotonic); - @memset(f.pc_counters, 0); - f.coverage.reset(); - return f.input.items; } fn visitPc(f: *Fuzzer, pc: usize) void { @@ -419,10 +423,12 @@ export fn fuzzer_coverage_id() u64 { return fuzzer.coverage_id; } -export fn fuzzer_next() Fuzzer.Slice { - return Fuzzer.Slice.fromZig(fuzzer.next() catch |err| switch (err) { - error.OutOfMemory => @panic("out of memory"), - }); +extern fn fuzzer_one(input_ptr: [*]const u8, input_len: usize) callconv(.C) void; + +export fn fuzzer_start() void { + fuzzer.start() catch |err| switch (err) { + error.OutOfMemory => fatal("out of memory", .{}), + }; } export fn fuzzer_init(cache_dir_struct: Fuzzer.Slice) void { @@ -432,24 +438,24 @@ export fn fuzzer_init(cache_dir_struct: Fuzzer.Slice) void { const pc_counters_start = @extern([*]u8, .{ .name = "__start___sancov_cntrs", .linkage = .weak, - }) orelse fatal("missing __start___sancov_cntrs symbol"); + }) orelse fatal("missing __start___sancov_cntrs symbol", .{}); const pc_counters_end = @extern([*]u8, .{ .name = "__stop___sancov_cntrs", .linkage = .weak, - }) orelse fatal("missing __stop___sancov_cntrs symbol"); + }) orelse fatal("missing __stop___sancov_cntrs symbol", .{}); const pc_counters = pc_counters_start[0 .. pc_counters_end - pc_counters_start]; const pcs_start = @extern([*]usize, .{ .name = "__start___sancov_pcs1", .linkage = .weak, - }) orelse fatal("missing __start___sancov_pcs1 symbol"); + }) orelse fatal("missing __start___sancov_pcs1 symbol", .{}); const pcs_end = @extern([*]usize, .{ .name = "__stop___sancov_pcs1", .linkage = .weak, - }) orelse fatal("missing __stop___sancov_pcs1 symbol"); + }) orelse fatal("missing __stop___sancov_pcs1 symbol", .{}); const pcs = pcs_start[0 .. pcs_end - pcs_start]; diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 35bb13bf0d9a..2cc38749eb5e 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -1141,6 +1141,10 @@ pub const FuzzInputOptions = struct { corpus: []const []const u8 = &.{}, }; -pub inline fn fuzzInput(options: FuzzInputOptions) []const u8 { - return @import("root").fuzzInput(options); +/// Inline to avoid coverage instrumentation. +pub inline fn fuzz( + comptime testOne: fn (input: []const u8) anyerror!void, + options: FuzzInputOptions, +) anyerror!void { + return @import("root").fuzz(testOne, options); } diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 06c6b859ac68..db69693a93b9 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -1708,6 +1708,10 @@ test "invalid tabs and carriage returns" { try testTokenize("\rpub\rswitch\r", &.{ .keyword_pub, .keyword_switch }); } +test "fuzzable properties upheld" { + return std.testing.fuzz(testPropertiesUpheld, .{}); +} + fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void { var tokenizer = Tokenizer.init(source); for (expected_token_tags) |expected_token_tag| { @@ -1723,8 +1727,7 @@ fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !v try std.testing.expectEqual(source.len, last_token.loc.end); } -test "fuzzable properties upheld" { - const source = std.testing.fuzzInput(.{}); +fn testPropertiesUpheld(source: []const u8) anyerror!void { const source0 = try std.testing.allocator.dupeZ(u8, source); defer std.testing.allocator.free(source0); var tokenizer = Tokenizer.init(source0); From 2b76221a468d1d4556b8f512a069b703f621cc2c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Sep 2024 19:57:31 -0700 Subject: [PATCH 2/7] libfuzzer: use a function pointer instead of extern solves the problem presented in the previous commit message --- lib/compiler/test_runner.zig | 5 ++--- lib/fuzzer.zig | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 2468c12645cc..3d2187748e51 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -345,7 +345,7 @@ const FuzzerSlice = extern struct { var is_fuzz_test: bool = undefined; -extern fn fuzzer_start() void; +extern fn fuzzer_start(testOne: *const fn ([*]const u8, usize) callconv(.C) void) void; extern fn fuzzer_init(cache_dir: FuzzerSlice) void; extern fn fuzzer_coverage_id() u64; @@ -394,8 +394,7 @@ pub fn fuzz( } }; if (builtin.fuzz) { - @export(&global.fuzzer_one, .{ .name = "fuzzer_one" }); - fuzzer_start(); + fuzzer_start(&global.fuzzer_one); return; } diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 2aa974427589..6cc8f9cc2879 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -423,9 +423,10 @@ export fn fuzzer_coverage_id() u64 { return fuzzer.coverage_id; } -extern fn fuzzer_one(input_ptr: [*]const u8, input_len: usize) callconv(.C) void; +var fuzzer_one: *const fn (input_ptr: [*]const u8, input_len: usize) callconv(.C) void = undefined; -export fn fuzzer_start() void { +export fn fuzzer_start(testOne: @TypeOf(fuzzer_one)) void { + fuzzer_one = testOne; fuzzer.start() catch |err| switch (err) { error.OutOfMemory => fatal("out of memory", .{}), }; From 9bc731b30a0be771a8128bab25d873f9212643a9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Sep 2024 20:32:41 -0700 Subject: [PATCH 3/7] fuzzing: better std.testing.allocator lifetime management --- lib/compiler/test_runner.zig | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 3d2187748e51..83d53626c3f0 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -148,11 +148,7 @@ fn mainServer() !void { const test_fn = builtin.test_functions[index]; const entry_addr = @intFromPtr(test_fn.func); try server.serveU64Message(.fuzz_start_addr, entry_addr); - const prev_allocator_state = testing.allocator_instance; - defer { - testing.allocator_instance = prev_allocator_state; - if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); - } + defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); is_fuzz_test = false; test_fn.func() catch |err| switch (err) { error.SkipZigTest => return, @@ -383,18 +379,24 @@ pub fn fuzz( testOne(input_ptr[0..input_len]) catch |err| switch (err) { error.SkipZigTest => return, else => { - if (@errorReturnTrace()) |trace| { - std.debug.dumpStackTrace(trace.*); - } + std.debug.lockStdErr(); + if (@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace.*); std.debug.print("failed with error.{s}\n", .{@errorName(err)}); std.process.exit(1); }, }; - if (log_err_count != 0) @panic("error logs detected"); + if (log_err_count != 0) { + std.debug.lockStdErr(); + std.debug.print("error logs detected\n", .{}); + std.process.exit(1); + } } }; if (builtin.fuzz) { + const prev_allocator_state = testing.allocator_instance; + testing.allocator_instance = .{}; fuzzer_start(&global.fuzzer_one); + testing.allocator_instance = prev_allocator_state; return; } From 0cdccff51912359b7ec5afa57fbbd5bb69d8f3a2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Sep 2024 20:37:03 -0700 Subject: [PATCH 4/7] fuzzer: move web files into separate directory --- lib/fuzzer/{ => web}/index.html | 0 lib/fuzzer/{ => web}/main.js | 0 lib/fuzzer/{wasm => web}/main.zig | 0 lib/std/Build/Fuzz/WebServer.zig | 6 +++--- 4 files changed, 3 insertions(+), 3 deletions(-) rename lib/fuzzer/{ => web}/index.html (100%) rename lib/fuzzer/{ => web}/main.js (100%) rename lib/fuzzer/{wasm => web}/main.zig (100%) diff --git a/lib/fuzzer/index.html b/lib/fuzzer/web/index.html similarity index 100% rename from lib/fuzzer/index.html rename to lib/fuzzer/web/index.html diff --git a/lib/fuzzer/main.js b/lib/fuzzer/web/main.js similarity index 100% rename from lib/fuzzer/main.js rename to lib/fuzzer/web/main.js diff --git a/lib/fuzzer/wasm/main.zig b/lib/fuzzer/web/main.zig similarity index 100% rename from lib/fuzzer/wasm/main.zig rename to lib/fuzzer/web/main.zig diff --git a/lib/std/Build/Fuzz/WebServer.zig b/lib/std/Build/Fuzz/WebServer.zig index a0ab018cf57c..b5ad86af15fd 100644 --- a/lib/std/Build/Fuzz/WebServer.zig +++ b/lib/std/Build/Fuzz/WebServer.zig @@ -128,11 +128,11 @@ fn serveRequest(ws: *WebServer, request: *std.http.Server.Request) !void { std.mem.eql(u8, request.head.target, "/debug") or std.mem.eql(u8, request.head.target, "/debug/")) { - try serveFile(ws, request, "fuzzer/index.html", "text/html"); + try serveFile(ws, request, "fuzzer/web/index.html", "text/html"); } else if (std.mem.eql(u8, request.head.target, "/main.js") or std.mem.eql(u8, request.head.target, "/debug/main.js")) { - try serveFile(ws, request, "fuzzer/main.js", "application/javascript"); + try serveFile(ws, request, "fuzzer/web/main.js", "application/javascript"); } else if (std.mem.eql(u8, request.head.target, "/main.wasm")) { try serveWasm(ws, request, .ReleaseFast); } else if (std.mem.eql(u8, request.head.target, "/debug/main.wasm")) { @@ -217,7 +217,7 @@ fn buildWasmBinary( const main_src_path: Build.Cache.Path = .{ .root_dir = ws.zig_lib_directory, - .sub_path = "fuzzer/wasm/main.zig", + .sub_path = "fuzzer/web/main.zig", }; const walk_src_path: Build.Cache.Path = .{ .root_dir = ws.zig_lib_directory, From 2d005827b874f27535cda72c80b6558d9d4cd30c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Sep 2024 21:16:06 -0700 Subject: [PATCH 5/7] make lowest stack an internal libfuzzer detail This value is useful to help determine run uniqueness in the face of recursion, however it is not valuable to expose to the fuzzing UI. --- lib/fuzzer.zig | 6 +++--- lib/fuzzer/web/index.html | 1 - lib/fuzzer/web/main.js | 2 -- lib/fuzzer/web/main.zig | 7 ------- lib/std/Build/Fuzz/WebServer.zig | 2 -- lib/std/Build/Fuzz/abi.zig | 2 -- 6 files changed, 3 insertions(+), 17 deletions(-) diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 6cc8f9cc2879..3f8a99114876 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -28,7 +28,8 @@ fn logOverride( f.writer().print(prefix1 ++ prefix2 ++ format ++ "\n", args) catch @panic("failed to write to fuzzer log"); } -export threadlocal var __sancov_lowest_stack: usize = std.math.maxInt(usize); +/// Helps determine run uniqueness in the face of recursion. +export threadlocal var __sancov_lowest_stack: usize = 0; export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void { handleCmp(@returnAddress(), arg1, arg2); @@ -220,7 +221,6 @@ const Fuzzer = struct { .n_runs = 0, .unique_runs = 0, .pcs_len = pcs.len, - .lowest_stack = std.math.maxInt(usize), }; f.seen_pcs.appendSliceAssumeCapacity(std.mem.asBytes(&header)); f.seen_pcs.appendNTimesAssumeCapacity(0, n_bitset_elems * @sizeOf(usize)); @@ -261,8 +261,8 @@ const Fuzzer = struct { f.input.appendSliceAssumeCapacity(run.input); try f.mutate(); - _ = @atomicRmw(usize, &header.lowest_stack, .Min, __sancov_lowest_stack, .monotonic); @memset(f.pc_counters, 0); + __sancov_lowest_stack = std.math.maxInt(usize); f.coverage.reset(); fuzzer_one(f.input.items.ptr, f.input.items.len); diff --git a/lib/fuzzer/web/index.html b/lib/fuzzer/web/index.html index 16fa87991377..0addd9f88288 100644 --- a/lib/fuzzer/web/index.html +++ b/lib/fuzzer/web/index.html @@ -147,7 +147,6 @@
  • Total Runs:
  • Unique Runs:
  • Coverage:
    • -
  • Lowest Stack:
  • Entry Points:
      • diff --git a/lib/fuzzer/web/main.js b/lib/fuzzer/web/main.js index ce02276f9819..9ee6b445e287 100644 --- a/lib/fuzzer/web/main.js +++ b/lib/fuzzer/web/main.js @@ -6,7 +6,6 @@ const domStatTotalRuns = document.getElementById("statTotalRuns"); const domStatUniqueRuns = document.getElementById("statUniqueRuns"); const domStatCoverage = document.getElementById("statCoverage"); - const domStatLowestStack = document.getElementById("statLowestStack"); const domEntryPointsList = document.getElementById("entryPointsList"); let wasm_promise = fetch("main.wasm"); @@ -158,7 +157,6 @@ domStatTotalRuns.innerText = totalRuns; domStatUniqueRuns.innerText = uniqueRuns + " (" + percent(uniqueRuns, totalRuns) + "%)"; domStatCoverage.innerText = coveredSourceLocations + " / " + totalSourceLocations + " (" + percent(coveredSourceLocations, totalSourceLocations) + "%)"; - domStatLowestStack.innerText = unwrapString(wasm_exports.lowestStack()); const entryPoints = unwrapInt32Array(wasm_exports.entryPoints()); resizeDomList(domEntryPointsList, entryPoints.length, "
    • "); diff --git a/lib/fuzzer/web/main.zig b/lib/fuzzer/web/main.zig index 342adc3b5608..94ea8cc92f2a 100644 --- a/lib/fuzzer/web/main.zig +++ b/lib/fuzzer/web/main.zig @@ -106,13 +106,6 @@ export fn decl_source_html(decl_index: Decl.Index) String { return String.init(string_result.items); } -export fn lowestStack() String { - const header: *abi.CoverageUpdateHeader = @ptrCast(recent_coverage_update.items[0..@sizeOf(abi.CoverageUpdateHeader)]); - string_result.clearRetainingCapacity(); - string_result.writer(gpa).print("0x{d}", .{header.lowest_stack}) catch @panic("OOM"); - return String.init(string_result.items); -} - export fn totalSourceLocations() usize { return coverage_source_locations.items.len; } diff --git a/lib/std/Build/Fuzz/WebServer.zig b/lib/std/Build/Fuzz/WebServer.zig index b5ad86af15fd..391f67e8231c 100644 --- a/lib/std/Build/Fuzz/WebServer.zig +++ b/lib/std/Build/Fuzz/WebServer.zig @@ -406,7 +406,6 @@ fn sendCoverageContext( const seen_pcs = cov_header.seenBits(); const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic); const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic); - const lowest_stack = @atomicLoad(usize, &cov_header.lowest_stack, .monotonic); if (prev_unique_runs.* != unique_runs) { // There has been an update. if (prev_unique_runs.* == 0) { @@ -431,7 +430,6 @@ fn sendCoverageContext( const header: abi.CoverageUpdateHeader = .{ .n_runs = n_runs, .unique_runs = unique_runs, - .lowest_stack = lowest_stack, }; const iovecs: [2]std.posix.iovec_const = .{ makeIov(std.mem.asBytes(&header)), diff --git a/lib/std/Build/Fuzz/abi.zig b/lib/std/Build/Fuzz/abi.zig index 0e16f0d5fa3f..c3f32d309b69 100644 --- a/lib/std/Build/Fuzz/abi.zig +++ b/lib/std/Build/Fuzz/abi.zig @@ -13,7 +13,6 @@ pub const SeenPcsHeader = extern struct { n_runs: usize, unique_runs: usize, pcs_len: usize, - lowest_stack: usize, /// Used for comptime assertions. Provides a mechanism for strategically /// causing compile errors. @@ -79,7 +78,6 @@ pub const CoverageUpdateHeader = extern struct { flags: Flags = .{}, n_runs: u64, unique_runs: u64, - lowest_stack: u64, pub const Flags = packed struct(u64) { tag: ToClientTag = .coverage_update, From 9dc75f03e26146cb81fc992baf172202fcd19b17 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Sep 2024 21:27:45 -0700 Subject: [PATCH 6/7] fix init template for new fuzz testing API --- lib/init/src/main.zig | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/init/src/main.zig b/lib/init/src/main.zig index 0c4bb73429c9..ba5a2ccef233 100644 --- a/lib/init/src/main.zig +++ b/lib/init/src/main.zig @@ -27,7 +27,11 @@ test "simple test" { } test "fuzz example" { - // Try passing `--fuzz` to `zig build test` and see if it manages to fail this test case! - const input_bytes = std.testing.fuzzInput(.{}); - try std.testing.expect(!std.mem.eql(u8, "canyoufindme", input_bytes)); + const global = struct { + fn testOne(input: []const u8) anyerror!void { + // Try passing `--fuzz` to `zig build test` and see if it manages to fail this test case! + try std.testing.expect(!std.mem.eql(u8, "canyoufindme", input)); + } + }; + try std.testing.fuzz(global.testOne, .{}); } From e3f58bd5515ffd0039c7f5afde8b9d74dc5a24b5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Sep 2024 19:53:14 -0700 Subject: [PATCH 7/7] add runs per second to fuzzing ui closes #21025 --- lib/fuzzer/web/index.html | 1 + lib/fuzzer/web/main.js | 5 +++++ lib/fuzzer/web/main.zig | 38 ++++++++++++++++++++++++++++++-- lib/std/Build/Fuzz.zig | 2 ++ lib/std/Build/Fuzz/WebServer.zig | 17 ++++++++++++++ lib/std/Build/Fuzz/abi.zig | 9 ++++++++ 6 files changed, 70 insertions(+), 2 deletions(-) diff --git a/lib/fuzzer/web/index.html b/lib/fuzzer/web/index.html index 0addd9f88288..325342e8ebf5 100644 --- a/lib/fuzzer/web/index.html +++ b/lib/fuzzer/web/index.html @@ -146,6 +146,7 @@
      • Total Runs:
      • Unique Runs:
        • +
      • Speed (Runs/Second):
      • Coverage:
      • Entry Points:
        diff --git a/lib/fuzzer/web/main.js b/lib/fuzzer/web/main.js index 9ee6b445e287..94f09391bb67 100644 --- a/lib/fuzzer/web/main.js +++ b/lib/fuzzer/web/main.js @@ -5,6 +5,7 @@ const domSourceText = document.getElementById("sourceText"); const domStatTotalRuns = document.getElementById("statTotalRuns"); const domStatUniqueRuns = document.getElementById("statUniqueRuns"); + const domStatSpeed = document.getElementById("statSpeed"); const domStatCoverage = document.getElementById("statCoverage"); const domEntryPointsList = document.getElementById("entryPointsList"); @@ -31,6 +32,9 @@ const msg = decodeString(ptr, len); throw new Error("panic: " + msg); }, + timestamp: function () { + return BigInt(new Date()); + }, emitSourceIndexChange: onSourceIndexChange, emitCoverageUpdate: onCoverageUpdate, emitEntryPointsUpdate: renderStats, @@ -157,6 +161,7 @@ domStatTotalRuns.innerText = totalRuns; domStatUniqueRuns.innerText = uniqueRuns + " (" + percent(uniqueRuns, totalRuns) + "%)"; domStatCoverage.innerText = coveredSourceLocations + " / " + totalSourceLocations + " (" + percent(coveredSourceLocations, totalSourceLocations) + "%)"; + domStatSpeed.innerText = wasm_exports.totalRunsPerSecond().toFixed(0); const entryPoints = unwrapInt32Array(wasm_exports.entryPoints()); resizeDomList(domEntryPointsList, entryPoints.length, "
      • "); diff --git a/lib/fuzzer/web/main.zig b/lib/fuzzer/web/main.zig index 94ea8cc92f2a..9c50704e8a71 100644 --- a/lib/fuzzer/web/main.zig +++ b/lib/fuzzer/web/main.zig @@ -10,9 +10,17 @@ const Walk = @import("Walk"); const Decl = Walk.Decl; const html_render = @import("html_render"); +/// Nanoseconds. +var server_base_timestamp: i64 = 0; +/// Milliseconds. +var client_base_timestamp: i64 = 0; +/// Relative to `server_base_timestamp`. +var start_fuzzing_timestamp: i64 = undefined; + const js = struct { extern "js" fn log(ptr: [*]const u8, len: usize) void; extern "js" fn panic(ptr: [*]const u8, len: usize) noreturn; + extern "js" fn timestamp() i64; extern "js" fn emitSourceIndexChange() void; extern "js" fn emitCoverageUpdate() void; extern "js" fn emitEntryPointsUpdate() void; @@ -64,6 +72,7 @@ export fn message_end() void { const tag: abi.ToClientTag = @enumFromInt(msg_bytes[0]); switch (tag) { + .current_time => return currentTimeMessage(msg_bytes), .source_index => return sourceIndexMessage(msg_bytes) catch @panic("OOM"), .coverage_update => return coverageUpdateMessage(msg_bytes) catch @panic("OOM"), .entry_points => return entryPointsMessage(msg_bytes) catch @panic("OOM"), @@ -117,16 +126,28 @@ export fn coveredSourceLocations() usize { return count; } +fn getCoverageUpdateHeader() *abi.CoverageUpdateHeader { + return @alignCast(@ptrCast(recent_coverage_update.items[0..@sizeOf(abi.CoverageUpdateHeader)])); +} + export fn totalRuns() u64 { - const header: *abi.CoverageUpdateHeader = @alignCast(@ptrCast(recent_coverage_update.items[0..@sizeOf(abi.CoverageUpdateHeader)])); + const header = getCoverageUpdateHeader(); return header.n_runs; } export fn uniqueRuns() u64 { - const header: *abi.CoverageUpdateHeader = @alignCast(@ptrCast(recent_coverage_update.items[0..@sizeOf(abi.CoverageUpdateHeader)])); + const header = getCoverageUpdateHeader(); return header.unique_runs; } +export fn totalRunsPerSecond() f64 { + @setFloatMode(.optimized); + const header = getCoverageUpdateHeader(); + const ns_elapsed: f64 = @floatFromInt(nsSince(start_fuzzing_timestamp)); + const n_runs: f64 = @floatFromInt(header.n_runs); + return n_runs / (ns_elapsed / std.time.ns_per_s); +} + const String = Slice(u8); fn Slice(T: type) type { @@ -189,6 +210,18 @@ fn fatal(comptime format: []const u8, args: anytype) noreturn { js.panic(line.ptr, line.len); } +fn currentTimeMessage(msg_bytes: []u8) void { + client_base_timestamp = js.timestamp(); + server_base_timestamp = @bitCast(msg_bytes[1..][0..8].*); +} + +/// Nanoseconds passed since a server timestamp. +fn nsSince(server_timestamp: i64) i64 { + const ms_passed = js.timestamp() - client_base_timestamp; + const ns_passed = server_base_timestamp - server_timestamp; + return ns_passed + ms_passed * std.time.ns_per_ms; +} + fn sourceIndexMessage(msg_bytes: []u8) error{OutOfMemory}!void { const Header = abi.SourceIndexHeader; const header: Header = @bitCast(msg_bytes[0..@sizeOf(Header)].*); @@ -205,6 +238,7 @@ fn sourceIndexMessage(msg_bytes: []u8) error{OutOfMemory}!void { const files: []const Coverage.File = @alignCast(std.mem.bytesAsSlice(Coverage.File, msg_bytes[files_start..files_end])); const source_locations: []const Coverage.SourceLocation = @alignCast(std.mem.bytesAsSlice(Coverage.SourceLocation, msg_bytes[source_locations_start..source_locations_end])); + start_fuzzing_timestamp = header.start_timestamp; try updateCoverage(directories, files, source_locations, string_bytes); js.emitSourceIndexChange(); } diff --git a/lib/std/Build/Fuzz.zig b/lib/std/Build/Fuzz.zig index 23f8a0269283..6258f4cddaee 100644 --- a/lib/std/Build/Fuzz.zig +++ b/lib/std/Build/Fuzz.zig @@ -66,6 +66,8 @@ pub fn start( .coverage_files = .{}, .coverage_mutex = .{}, .coverage_condition = .{}, + + .base_timestamp = std.time.nanoTimestamp(), }; // For accepting HTTP connections. diff --git a/lib/std/Build/Fuzz/WebServer.zig b/lib/std/Build/Fuzz/WebServer.zig index 391f67e8231c..fb78e96abb02 100644 --- a/lib/std/Build/Fuzz/WebServer.zig +++ b/lib/std/Build/Fuzz/WebServer.zig @@ -33,6 +33,9 @@ coverage_mutex: std.Thread.Mutex, /// Signaled when `coverage_files` changes. coverage_condition: std.Thread.Condition, +/// Time at initialization of WebServer. +base_timestamp: i128, + const fuzzer_bin_name = "fuzzer"; const fuzzer_arch_os_abi = "wasm32-freestanding"; const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; @@ -43,6 +46,7 @@ const CoverageMap = struct { source_locations: []Coverage.SourceLocation, /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. entry_points: std.ArrayListUnmanaged(u32), + start_timestamp: i64, fn deinit(cm: *CoverageMap, gpa: Allocator) void { std.posix.munmap(cm.mapped_memory); @@ -87,6 +91,10 @@ pub fn run(ws: *WebServer) void { } } +fn now(s: *const WebServer) i64 { + return @intCast(std.time.nanoTimestamp() - s.base_timestamp); +} + fn accept(ws: *WebServer, connection: std.net.Server.Connection) void { defer connection.stream.close(); @@ -381,6 +389,13 @@ fn serveWebSocket(ws: *WebServer, web_socket: *std.http.WebSocket) !void { ws.coverage_mutex.lock(); defer ws.coverage_mutex.unlock(); + // On first connection, the client needs to know what time the server + // thinks it is to rebase timestamps. + { + const timestamp_message: abi.CurrentTime = .{ .base = ws.now() }; + try web_socket.writeMessage(std.mem.asBytes(×tamp_message), .binary); + } + // On first connection, the client needs all the coverage information // so that subsequent updates can contain only the updated bits. var prev_unique_runs: usize = 0; @@ -416,6 +431,7 @@ fn sendCoverageContext( .files_len = @intCast(coverage_map.coverage.files.entries.len), .source_locations_len = @intCast(coverage_map.source_locations.len), .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len), + .start_timestamp = coverage_map.start_timestamp, }; const iovecs: [5]std.posix.iovec_const = .{ makeIov(std.mem.asBytes(&header)), @@ -582,6 +598,7 @@ fn prepareTables( .mapped_memory = undefined, // populated below .source_locations = undefined, // populated below .entry_points = .{}, + .start_timestamp = ws.now(), }; errdefer gop.value_ptr.coverage.deinit(gpa); diff --git a/lib/std/Build/Fuzz/abi.zig b/lib/std/Build/Fuzz/abi.zig index c3f32d309b69..a6abc13feebb 100644 --- a/lib/std/Build/Fuzz/abi.zig +++ b/lib/std/Build/Fuzz/abi.zig @@ -43,12 +43,19 @@ pub const SeenPcsHeader = extern struct { }; pub const ToClientTag = enum(u8) { + current_time, source_index, coverage_update, entry_points, _, }; +pub const CurrentTime = extern struct { + tag: ToClientTag = .current_time, + /// Number of nanoseconds that all other timestamps are in reference to. + base: i64 align(1), +}; + /// Sent to the fuzzer web client on first connection to the websocket URL. /// /// Trailing: @@ -62,6 +69,8 @@ pub const SourceIndexHeader = extern struct { files_len: u32, source_locations_len: u32, string_bytes_len: u32, + /// When, according to the server, fuzzing started. + start_timestamp: i64 align(4), pub const Flags = packed struct(u32) { tag: ToClientTag = .source_index,