Nixyri/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2025-02-01 14:55:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

rework fuzzing API

Browse Source 
The previous API used `std.testing.fuzzInput(.{})` however that has the
problem that users call it multiple times incorrectly, and there might
be work happening to obtain the corpus which should not be included in
coverage analysis, and which must not slow down iteration speed.

This commit restructures it so that the main loop lives in libfuzzer and
directly calls the "test one" function.

In this commit I was a little too aggressive because I made the test
runner export `fuzzer_one` for this purpose. This was motivated by
performance, but it causes "exported symbol collision: fuzzer_one" to
occur when more than one fuzz test is provided.

There are three ways to solve this:

1. libfuzzer needs to be passed a function pointer instead. Possible
   performance downside.

2. build runner needs to build a different process per fuzz test.
   Potentially wasteful and unclear how to isolate them.

3. test runner needs to perform a relocation at runtime to point the
   function call to the relevant unit test. Portability issues and
   dubious performance gains.
This commit is contained in:
Andrew Kelley 2024-09-09 19:36:52 -07:00
parent 218cf059dd
commit 892ce7ef52
4 changed files with 131 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
lib/compiler/test_runner.zig
View File

@ -145,31 +145,27 @@ fn mainServer() !void {
.start_fuzzing => {
if (!builtin.fuzz) unreachable;
const index = try server.receiveBody_u32();
var first = true;
const test_fn = builtin.test_functions[index];
while (true) {
testing.allocator_instance = .{};
defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
log_err_count = 0;
is_fuzz_test = false;
test_fn.func() catch |err| switch (err) {
error.SkipZigTest => continue,
else => {
if (@errorReturnTrace()) |trace| {
std.debug.dumpStackTrace(trace.*);
}
std.debug.print("failed with error.{s}\n", .{@errorName(err)});
std.process.exit(1);
},
};
if (!is_fuzz_test) @panic("missed call to std.testing.fuzzInput");
if (log_err_count != 0) @panic("error logs detected");
if (first) {
first = false;
const entry_addr = @intFromPtr(test_fn.func);
try server.serveU64Message(.fuzz_start_addr, entry_addr);
}
const entry_addr = @intFromPtr(test_fn.func);
try server.serveU64Message(.fuzz_start_addr, entry_addr);
const prev_allocator_state = testing.allocator_instance;
defer {
testing.allocator_instance = prev_allocator_state;
if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
}
is_fuzz_test = false;
test_fn.func() catch |err| switch (err) {
error.SkipZigTest => return,
else => {
if (@errorReturnTrace()) |trace| {
std.debug.dumpStackTrace(trace.*);
}
std.debug.print("failed with error.{s}\n", .{@errorName(err)});
std.process.exit(1);
},
};
if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
if (log_err_count != 0) @panic("error logs detected");
},
else => {
@ -349,19 +345,67 @@ const FuzzerSlice = extern struct {
var is_fuzz_test: bool = undefined;
extern fn fuzzer_next() FuzzerSlice;
extern fn fuzzer_start() void;
extern fn fuzzer_init(cache_dir: FuzzerSlice) void;
extern fn fuzzer_coverage_id() u64;
pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 {
pub fn fuzz(
comptime testOne: fn ([]const u8) anyerror!void,
options: testing.FuzzInputOptions,
) anyerror!void {
// Prevent this function from confusing the fuzzer by omitting its own code
// coverage from being considered.
@disableInstrumentation();
if (crippled) return "";
// Some compiler backends are not capable of handling fuzz testing yet but
// we still want CI test coverage enabled.
if (crippled) return;
// Smoke test to ensure the test did not use conditional compilation to
// contradict itself by making it not actually be a fuzz test when the test
// is built in fuzz mode.
is_fuzz_test = true;
// Ensure no test failure occurred before starting fuzzing.
if (log_err_count != 0) @panic("error logs detected");
// libfuzzer is in a separate compilation unit so that its own code can be
// excluded from code coverage instrumentation. It needs a function pointer
// it can call for checking exactly one input. Inside this function we do
// our standard unit test checks such as memory leaks, and interaction with
// error logs.
const global = struct {
fn fuzzer_one(input_ptr: [*]const u8, input_len: usize) callconv(.C) void {
@disableInstrumentation();
testing.allocator_instance = .{};
defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
log_err_count = 0;
testOne(input_ptr[0..input_len]) catch |err| switch (err) {
error.SkipZigTest => return,
else => {
if (@errorReturnTrace()) |trace| {
std.debug.dumpStackTrace(trace.*);
}
std.debug.print("failed with error.{s}\n", .{@errorName(err)});
std.process.exit(1);
},
};
if (log_err_count != 0) @panic("error logs detected");
}
};
if (builtin.fuzz) {
return fuzzer_next().toSlice();
@export(&global.fuzzer_one, .{ .name = "fuzzer_one" });
fuzzer_start();
return;
}
if (options.corpus.len == 0) return "";
var prng = std.Random.DefaultPrng.init(testing.random_seed);
const random = prng.random();
return options.corpus[random.uintLessThan(usize, options.corpus.len)];
// When the unit test executable is not built in fuzz mode, only run the
// provided corpus.
for (options.corpus) |input| {
try testOne(input);
}
// In case there is no provided corpus, also use an empty
// string as a smoke test.
try testOne("");
}

84
lib/fuzzer.zig
View File

@ -235,22 +235,41 @@ const Fuzzer = struct {
};
}
fn next(f: *Fuzzer) ![]const u8 {
fn start(f: *Fuzzer) !void {
const gpa = f.gpa;
const rng = fuzzer.rng.random();
if (f.recent_cases.entries.len == 0) {
// Prepare initial input.
try f.recent_cases.ensureUnusedCapacity(gpa, 100);
const len = rng.uintLessThanBiased(usize, 80);
try f.input.resize(gpa, len);
rng.bytes(f.input.items);
f.recent_cases.putAssumeCapacity(.{
.id = 0,
.input = try gpa.dupe(u8, f.input.items),
.score = 0,
}, {});
} else {
// Prepare initial input.
assert(f.recent_cases.entries.len == 0);
assert(f.n_runs == 0);
try f.recent_cases.ensureUnusedCapacity(gpa, 100);
const len = rng.uintLessThanBiased(usize, 80);
try f.input.resize(gpa, len);
rng.bytes(f.input.items);
f.recent_cases.putAssumeCapacity(.{
.id = 0,
.input = try gpa.dupe(u8, f.input.items),
.score = 0,
}, {});
const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
while (true) {
const chosen_index = rng.uintLessThanBiased(usize, f.recent_cases.entries.len);
const run = &f.recent_cases.keys()[chosen_index];
f.input.clearRetainingCapacity();
f.input.appendSliceAssumeCapacity(run.input);
try f.mutate();
_ = @atomicRmw(usize, &header.lowest_stack, .Min, __sancov_lowest_stack, .monotonic);
@memset(f.pc_counters, 0);
f.coverage.reset();
fuzzer_one(f.input.items.ptr, f.input.items.len);
f.n_runs += 1;
_ = @atomicRmw(usize, &header.n_runs, .Add, 1, .monotonic);
if (f.n_runs % 10000 == 0) f.dumpStats();
const analysis = f.analyzeLastRun();
@ -301,7 +320,6 @@ const Fuzzer = struct {
}
}
const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
_ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
}
@ -317,26 +335,12 @@ const Fuzzer = struct {
// This has to be done before deinitializing the deleted items.
const doomed_runs = f.recent_cases.keys()[cap..];
f.recent_cases.shrinkRetainingCapacity(cap);
for (doomed_runs) |*run| {
std.log.info("culling score={d} id={d}", .{ run.score, run.id });
run.deinit(gpa);
for (doomed_runs) |*doomed_run| {
std.log.info("culling score={d} id={d}", .{ doomed_run.score, doomed_run.id });
doomed_run.deinit(gpa);
}
}
}
const chosen_index = rng.uintLessThanBiased(usize, f.recent_cases.entries.len);
const run = &f.recent_cases.keys()[chosen_index];
f.input.clearRetainingCapacity();
f.input.appendSliceAssumeCapacity(run.input);
try f.mutate();
f.n_runs += 1;
const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]);
_ = @atomicRmw(usize, &header.n_runs, .Add, 1, .monotonic);
_ = @atomicRmw(usize, &header.lowest_stack, .Min, __sancov_lowest_stack, .monotonic);
@memset(f.pc_counters, 0);
f.coverage.reset();
return f.input.items;
}
fn visitPc(f: *Fuzzer, pc: usize) void {
@ -419,10 +423,12 @@ export fn fuzzer_coverage_id() u64 {
return fuzzer.coverage_id;
}
export fn fuzzer_next() Fuzzer.Slice {
return Fuzzer.Slice.fromZig(fuzzer.next() catch |err| switch (err) {
error.OutOfMemory => @panic("out of memory"),
});
extern fn fuzzer_one(input_ptr: [*]const u8, input_len: usize) callconv(.C) void;
export fn fuzzer_start() void {
fuzzer.start() catch |err| switch (err) {
error.OutOfMemory => fatal("out of memory", .{}),
};
}
export fn fuzzer_init(cache_dir_struct: Fuzzer.Slice) void {
@ -432,24 +438,24 @@ export fn fuzzer_init(cache_dir_struct: Fuzzer.Slice) void {
const pc_counters_start = @extern([*]u8, .{
.name = "__start___sancov_cntrs",
.linkage = .weak,
}) orelse fatal("missing __start___sancov_cntrs symbol");
}) orelse fatal("missing __start___sancov_cntrs symbol", .{});
const pc_counters_end = @extern([*]u8, .{
.name = "__stop___sancov_cntrs",
.linkage = .weak,
}) orelse fatal("missing __stop___sancov_cntrs symbol");
}) orelse fatal("missing __stop___sancov_cntrs symbol", .{});
const pc_counters = pc_counters_start[0 .. pc_counters_end - pc_counters_start];
const pcs_start = @extern([*]usize, .{
.name = "__start___sancov_pcs1",
.linkage = .weak,
}) orelse fatal("missing __start___sancov_pcs1 symbol");
}) orelse fatal("missing __start___sancov_pcs1 symbol", .{});
const pcs_end = @extern([*]usize, .{
.name = "__stop___sancov_pcs1",
.linkage = .weak,
}) orelse fatal("missing __stop___sancov_pcs1 symbol");
}) orelse fatal("missing __stop___sancov_pcs1 symbol", .{});
const pcs = pcs_start[0 .. pcs_end - pcs_start];

8
lib/std/testing.zig
View File

@ -1141,6 +1141,10 @@ pub const FuzzInputOptions = struct {
corpus: []const []const u8 = &.{},
};
pub inline fn fuzzInput(options: FuzzInputOptions) []const u8 {
return @import("root").fuzzInput(options);
/// Inline to avoid coverage instrumentation.
pub inline fn fuzz(
comptime testOne: fn (input: []const u8) anyerror!void,
options: FuzzInputOptions,
) anyerror!void {
return @import("root").fuzz(testOne, options);
}

7
lib/std/zig/tokenizer.zig
View File

@ -1708,6 +1708,10 @@ test "invalid tabs and carriage returns" {
try testTokenize("\rpub\rswitch\r", &.{ .keyword_pub, .keyword_switch });
}
test "fuzzable properties upheld" {
return std.testing.fuzz(testPropertiesUpheld, .{});
}
fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
var tokenizer = Tokenizer.init(source);
for (expected_token_tags) |expected_token_tag| {
@ -1723,8 +1727,7 @@ fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !v
try std.testing.expectEqual(source.len, last_token.loc.end);
}
test "fuzzable properties upheld" {
const source = std.testing.fuzzInput(.{});
fn testPropertiesUpheld(source: []const u8) anyerror!void {
const source0 = try std.testing.allocator.dupeZ(u8, source);
defer std.testing.allocator.free(source0);
var tokenizer = Tokenizer.init(source0);