Nixyri/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2025-02-08 21:50:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

build runner sends a start_fuzzing message to test runner

Browse Source
This commit is contained in:
Andrew Kelley 2024-07-23 22:59:25 -07:00
parent 711ed56ce3
commit bce3b1efb0
4 changed files with 167 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/compiler/build_runner.zig
View File

@ -401,7 +401,7 @@ pub fn main() !void {
else => return err,
};
if (fuzz) {
Fuzz.start(&run.thread_pool, run.step_stack.keys(), main_progress_node);
Fuzz.start(&run.thread_pool, run.step_stack.keys(), run.ttyconf, main_progress_node);
}
if (!watch) return cleanExit();
@ -1072,7 +1072,7 @@ fn workerMakeOneStep(
std.debug.lockStdErr();
defer std.debug.unlockStdErr();
printErrorMessages(b, s, run) catch {};
printErrorMessages(b, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {};
}
handle_result: {
@ -1125,10 +1125,14 @@ fn workerMakeOneStep(
}
}
fn printErrorMessages(b: *std.Build, failing_step: *Step, run: *const Run) !void {
pub fn printErrorMessages(
b: *std.Build,
failing_step: *Step,
ttyconf: std.io.tty.Config,
stderr: File,
prominent_compile_errors: bool,
) !void {
const gpa = b.allocator;
const stderr = run.stderr;
const ttyconf = run.ttyconf;
// Provide context for where these error messages are coming from by
// printing the corresponding Step subtree.
@ -1166,7 +1170,7 @@ fn printErrorMessages(b: *std.Build, failing_step: *Step, run: *const Run) !void
}
}
if (!run.prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0)
if (!prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0)
try failing_step.result_error_bundle.renderToWriter(renderOptions(ttyconf), stderr.writer());
for (failing_step.result_error_msgs.items) |msg| {

84
lib/std/Build/Fuzz.zig
View File

@ -3,9 +3,15 @@ const Fuzz = @This();
const Step = std.Build.Step;
const assert = std.debug.assert;
const fatal = std.process.fatal;
const build_runner = @import("root");
pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void {
{
pub fn start(
thread_pool: *std.Thread.Pool,
all_steps: []const *Step,
ttyconf: std.io.tty.Config,
prog_node: std.Progress.Node,
) void {
const count = block: {
const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0);
defer rebuild_node.end();
var count: usize = 0;
@ -14,13 +20,14 @@ pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node:
for (all_steps) |step| {
const run = step.cast(Step.Run) orelse continue;
if (run.fuzz_tests.items.len > 0 and run.producer != null) {
thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node });
thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, ttyconf, rebuild_node });
count += 1;
}
}
if (count == 0) fatal("no fuzz tests found", .{});
rebuild_node.setEstimatedTotalItems(count);
}
break :block count;
};
// Detect failure.
for (all_steps) |step| {
@ -29,18 +36,69 @@ pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node:
fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{});
}
@panic("TODO do something with the rebuilt unit tests");
{
const rebuild_node = prog_node.start("Fuzzing", count);
defer rebuild_node.end();
var wait_group: std.Thread.WaitGroup = .{};
defer wait_group.wait();
for (all_steps) |step| {
const run = step.cast(Step.Run) orelse continue;
for (run.fuzz_tests.items) |unit_test_index| {
assert(run.rebuilt_executable != null);
thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ run, unit_test_index, ttyconf, prog_node });
}
}
}
fatal("all fuzz workers crashed", .{});
}
fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void {
fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
const compile_step = run.producer.?;
const prog_node = parent_prog_node.start(compile_step.step.name, 0);
defer prog_node.end();
const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| {
std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{
compile_step.step.name, @errorName(err),
});
return;
};
run.rebuilt_executable = rebuilt_bin_path;
if (compile_step.rebuildInFuzzMode(prog_node)) |rebuilt_bin_path| {
run.rebuilt_executable = rebuilt_bin_path;
} else |err| switch (err) {
error.MakeFailed => {
const b = run.step.owner;
const stderr = std.io.getStdErr();
std.debug.lockStdErr();
defer std.debug.unlockStdErr();
build_runner.printErrorMessages(b, &compile_step.step, ttyconf, stderr, false) catch {};
},
else => {
std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{
compile_step.step.name, @errorName(err),
});
},
}
}
fn fuzzWorkerRun(
run: *Step.Run,
unit_test_index: u32,
ttyconf: std.io.tty.Config,
parent_prog_node: std.Progress.Node,
) void {
const test_name = run.cached_test_metadata.?.testName(unit_test_index);
const prog_node = parent_prog_node.start(test_name, 0);
defer prog_node.end();
run.rerunInFuzzMode(unit_test_index, prog_node) catch |err| switch (err) {
error.MakeFailed => {
const b = run.step.owner;
const stderr = std.io.getStdErr();
std.debug.lockStdErr();
defer std.debug.unlockStdErr();
build_runner.printErrorMessages(b, &run.step, ttyconf, stderr, false) catch {};
},
else => {
std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{
run.step.name, test_name, @errorName(err),
});
},
};
}

96
lib/std/Build/Step/Run.zig
View File

@ -89,6 +89,8 @@ has_side_effects: bool,
/// If this is a Zig unit test binary, this tracks the indexes of the unit
/// tests that are also fuzz tests.
fuzz_tests: std.ArrayListUnmanaged(u32),
cached_test_metadata: ?CachedTestMetadata = null,
/// Populated during the fuzz phase if this run step corresponds to a unit test
/// executable that contains fuzz tests.
rebuilt_executable: ?[]const u8,
@ -754,7 +756,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path });
}
try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node);
try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node, null);
if (!has_side_effects) try step.writeManifestAndWatch(&man);
return;
};
@ -784,7 +786,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path });
}
try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node);
try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, null);
const dep_file_dir = std.fs.cwd();
const dep_file_basename = dep_output_file.generated_file.getPath();
@ -843,6 +845,38 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
);
}
pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress.Node) !void {
const step = &run.step;
const b = step.owner;
const arena = b.allocator;
var argv_list: std.ArrayListUnmanaged([]const u8) = .{};
for (run.argv.items) |arg| {
switch (arg) {
.bytes => |bytes| {
try argv_list.append(arena, bytes);
},
.lazy_path => |file| {
const file_path = file.lazy_path.getPath2(b, step);
try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, file_path }));
},
.directory_source => |file| {
const file_path = file.lazy_path.getPath2(b, step);
try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, file_path }));
},
.artifact => |pa| {
const artifact = pa.artifact;
const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?;
try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path }));
},
.output_file, .output_directory => unreachable,
}
}
const has_side_effects = false;
const rand_int = std.crypto.random.int(u64);
const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, unit_test_index);
}
fn populateGeneratedPaths(
arena: std.mem.Allocator,
output_placeholders: []const IndexedOutput,
@ -921,6 +955,7 @@ fn runCommand(
has_side_effects: bool,
output_dir_path: []const u8,
prog_node: std.Progress.Node,
fuzz_unit_test_index: ?u32,
) !void {
const step = &run.step;
const b = step.owner;
@ -939,7 +974,7 @@ fn runCommand(
var interp_argv = std.ArrayList([]const u8).init(b.allocator);
defer interp_argv.deinit();
const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node) catch |err| term: {
const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node, fuzz_unit_test_index) catch |err| term: {
// InvalidExe: cpu arch mismatch
// FileNotFound: can happen with a wrong dynamic linker path
if (err == error.InvalidExe or err == error.FileNotFound) interpret: {
@ -1075,7 +1110,7 @@ fn runCommand(
try Step.handleVerbose2(step.owner, cwd, run.env_map, interp_argv.items);
break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node) catch |e| {
break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node, fuzz_unit_test_index) catch |e| {
if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped;
return step.fail("unable to spawn interpreter {s}: {s}", .{
@ -1090,6 +1125,15 @@ fn runCommand(
step.result_duration_ns = result.elapsed_ns;
step.result_peak_rss = result.peak_rss;
step.test_results = result.stdio.test_results;
if (result.stdio.test_metadata) |tm|
run.cached_test_metadata = tm.toCachedTestMetadata();
const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items;
if (fuzz_unit_test_index != null) {
try step.handleChildProcessTerm(result.term, cwd, final_argv);
return;
}
// Capture stdout and stderr to GeneratedFile objects.
const Stream = struct {
@ -1126,8 +1170,6 @@ fn runCommand(
}
}
const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items;
switch (run.stdio) {
.check => |checks| for (checks.items) |check| switch (check) {
.expect_stderr_exact => |expected_bytes| {
@ -1253,10 +1295,16 @@ fn spawnChildAndCollect(
argv: []const []const u8,
has_side_effects: bool,
prog_node: std.Progress.Node,
fuzz_unit_test_index: ?u32,
) !ChildProcResult {
const b = run.step.owner;
const arena = b.allocator;
if (fuzz_unit_test_index != null) {
assert(!has_side_effects);
assert(run.stdio == .zig_test);
}
var child = std.process.Child.init(argv, arena);
if (run.cwd) |lazy_cwd| {
child.cwd = lazy_cwd.getPath2(b, &run.step);
@ -1306,7 +1354,7 @@ fn spawnChildAndCollect(
var timer = try std.time.Timer.start();
const result = if (run.stdio == .zig_test)
evalZigTest(run, &child, prog_node)
evalZigTest(run, &child, prog_node, fuzz_unit_test_index)
else
evalGeneric(run, &child);
@ -1332,6 +1380,7 @@ fn evalZigTest(
run: *Run,
child: *std.process.Child,
prog_node: std.Progress.Node,
fuzz_unit_test_index: ?u32,
) !StdIoResult {
const gpa = run.step.owner.allocator;
const arena = run.step.owner.allocator;
@ -1342,7 +1391,12 @@ fn evalZigTest(
});
defer poller.deinit();
try sendMessage(child.stdin.?, .query_test_metadata);
if (fuzz_unit_test_index) |index| {
try sendRunTestMessage(child.stdin.?, .start_fuzzing, index);
} else {
run.fuzz_tests.clearRetainingCapacity();
try sendMessage(child.stdin.?, .query_test_metadata);
}
const Header = std.zig.Server.Message.Header;
@ -1360,8 +1414,6 @@ fn evalZigTest(
var sub_prog_node: ?std.Progress.Node = null;
defer if (sub_prog_node) |n| n.end();
run.fuzz_tests.clearRetainingCapacity();
poll: while (true) {
while (stdout.readableLength() < @sizeOf(Header)) {
if (!(try poller.poll())) break :poll;
@ -1382,6 +1434,7 @@ fn evalZigTest(
}
},
.test_metadata => {
assert(fuzz_unit_test_index == null);
const TmHdr = std.zig.Server.Message.TestMetadata;
const tm_hdr = @as(*align(1) const TmHdr, @ptrCast(body));
test_count = tm_hdr.tests_len;
@ -1410,6 +1463,7 @@ fn evalZigTest(
try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node);
},
.test_results => {
assert(fuzz_unit_test_index == null);
const md = metadata.?;
const TrHdr = std.zig.Server.Message.TestResults;
@ -1479,7 +1533,23 @@ const TestMetadata = struct {
next_index: u32,
prog_node: std.Progress.Node,
fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata {
return .{
.names = tm.names,
.string_bytes = tm.string_bytes,
};
}
fn testName(tm: TestMetadata, index: u32) []const u8 {
return tm.toCachedTestMetadata().testName(index);
}
};
pub const CachedTestMetadata = struct {
names: []const u32,
string_bytes: []const u8,
pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 {
return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0);
}
};
@ -1495,7 +1565,7 @@ fn requestNextTest(in: fs.File, metadata: *TestMetadata, sub_prog_node: *?std.Pr
if (sub_prog_node.*) |n| n.end();
sub_prog_node.* = metadata.prog_node.start(name, 0);
try sendRunTestMessage(in, i);
try sendRunTestMessage(in, .run_test, i);
return;
} else {
try sendMessage(in, .exit);
@ -1510,9 +1580,9 @@ fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
try file.writeAll(std.mem.asBytes(&header));
}
fn sendRunTestMessage(file: std.fs.File, index: u32) !void {
fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index: u32) !void {
const header: std.zig.Client.Message.Header = .{
.tag = .run_test,
.tag = tag,
.bytes_len = 4,
};
const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index);

3
lib/std/zig/Client.zig
View File

@ -33,6 +33,9 @@ pub const Message = struct {
/// Ask the test runner to run a particular test.
/// The message body is a u32 test index.
run_test,
/// Ask the test runner to start fuzzing a particular test.
/// The message body is a u32 test index.
start_fuzzing,
_,
};