zig build: promote qemu, wine, wasmtime, darling, and rosetta

from zig-specific options to generally recognized zig build options that
any project can take advantage of. See the updated usage text for more
details.
This commit is contained in:
Andrew Kelley 2021-12-02 15:42:59 -07:00
parent cbd653e1d6
commit b24cbecdb2
7 changed files with 86 additions and 235 deletions

View File

@ -296,13 +296,6 @@ pub fn build(b: *Builder) !void {
const test_filter = b.option([]const u8, "test-filter", "Skip tests that do not match filter");
const is_wine_enabled = b.option(bool, "enable-wine", "Use Wine to run cross compiled Windows tests") orelse false;
const is_qemu_enabled = b.option(bool, "enable-qemu", "Use QEMU to run cross compiled foreign architecture tests") orelse false;
const is_wasmtime_enabled = b.option(bool, "enable-wasmtime", "Use Wasmtime to enable and run WASI libstd tests") orelse false;
const is_darling_enabled = b.option(bool, "enable-darling", "[Experimental] Use Darling to run cross compiled macOS tests") orelse false;
const is_rosetta_enabled = b.option(bool, "enable-rosetta", "(Darwin) Use Rosetta to run x86_64 macOS tests on arm64 macOS") orelse false;
const glibc_multi_dir = b.option([]const u8, "enable-foreign-glibc", "Provide directory with glibc installations to run cross compiled tests that link glibc");
const test_stage2_options = b.addOptions();
test_stage2.addOptions("build_options", test_stage2_options);
@ -317,13 +310,13 @@ pub fn build(b: *Builder) !void {
test_stage2_options.addOption(bool, "llvm_has_csky", llvm_has_csky);
test_stage2_options.addOption(bool, "llvm_has_ve", llvm_has_ve);
test_stage2_options.addOption(bool, "llvm_has_arc", llvm_has_arc);
test_stage2_options.addOption(bool, "enable_qemu", is_qemu_enabled);
test_stage2_options.addOption(bool, "enable_wine", is_wine_enabled);
test_stage2_options.addOption(bool, "enable_wasmtime", is_wasmtime_enabled);
test_stage2_options.addOption(bool, "enable_rosetta", is_rosetta_enabled);
test_stage2_options.addOption(bool, "enable_qemu", b.enable_qemu);
test_stage2_options.addOption(bool, "enable_wine", b.enable_wine);
test_stage2_options.addOption(bool, "enable_wasmtime", b.enable_wasmtime);
test_stage2_options.addOption(bool, "enable_rosetta", b.enable_rosetta);
test_stage2_options.addOption(bool, "enable_darling", b.enable_darling);
test_stage2_options.addOption(u32, "mem_leak_frames", mem_leak_frames * 2);
test_stage2_options.addOption(bool, "enable_darling", is_darling_enabled);
test_stage2_options.addOption(?[]const u8, "glibc_multi_install_dir", glibc_multi_dir);
test_stage2_options.addOption(?[]const u8, "glibc_runtimes_dir", b.glibc_runtimes_dir);
test_stage2_options.addOption([:0]const u8, "version", try b.allocator.dupeZ(u8, version));
test_stage2_options.addOption(std.SemanticVersion, "semver", semver);
@ -368,12 +361,6 @@ pub fn build(b: *Builder) !void {
false, // skip_single_threaded
skip_non_native,
skip_libc,
is_wine_enabled,
is_qemu_enabled,
is_wasmtime_enabled,
is_darling_enabled,
is_rosetta_enabled,
glibc_multi_dir,
));
toolchain_step.dependOn(tests.addPkgTests(
@ -386,12 +373,6 @@ pub fn build(b: *Builder) !void {
true, // skip_single_threaded
skip_non_native,
true, // skip_libc
is_wine_enabled,
is_qemu_enabled,
is_wasmtime_enabled,
is_darling_enabled,
is_rosetta_enabled,
glibc_multi_dir,
));
toolchain_step.dependOn(tests.addPkgTests(
@ -404,12 +385,6 @@ pub fn build(b: *Builder) !void {
true, // skip_single_threaded
skip_non_native,
true, // skip_libc
is_wine_enabled,
is_qemu_enabled,
is_wasmtime_enabled,
is_darling_enabled,
is_rosetta_enabled,
glibc_multi_dir,
));
toolchain_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes));
@ -435,12 +410,6 @@ pub fn build(b: *Builder) !void {
false,
skip_non_native,
skip_libc,
is_wine_enabled,
is_qemu_enabled,
is_wasmtime_enabled,
is_darling_enabled,
is_rosetta_enabled,
glibc_multi_dir,
);
const test_step = b.step("test", "Run all the tests");

View File

@ -1,142 +0,0 @@
#!/bin/sh
set -x
set -e
sudo apt-get update -q
sudo apt-get install -y cmake s3cmd tidy
ZIGDIR="$(pwd)"
ARCH="$(uname -m)"
TARGET="$ARCH-linux-musl"
CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.9.0-dev.1243+456d7e5f5"
PREFIX="$HOME/$CACHE_BASENAME"
MCPU="baseline"
JOBS="-j$(nproc)"
rm -rf $PREFIX
cd $HOME
wget -nv "https://ziglang.org/deps/$CACHE_BASENAME.tar.xz"
tar xf "$CACHE_BASENAME.tar.xz"
QEMUBASE="qemu-linux-x86_64-6.1.0.1"
wget -nv "https://ziglang.org/deps/$QEMUBASE.tar.xz"
tar xf "$QEMUBASE.tar.xz"
export PATH="$(pwd)/$QEMUBASE/bin:$PATH"
WASMTIME="wasmtime-v0.26.1-x86_64-linux"
wget -nv "https://github.com/bytecodealliance/wasmtime/releases/download/v0.26.1/$WASMTIME.tar.xz"
tar xf "$WASMTIME.tar.xz"
export PATH="$(pwd)/$WASMTIME:$PATH"
ZIG="$PREFIX/bin/zig"
export CC="$ZIG cc -target $TARGET -mcpu=$MCPU"
export CXX="$ZIG c++ -target $TARGET -mcpu=$MCPU"
cd $ZIGDIR
# Make the `zig version` number consistent.
# This will affect the cmake command below.
git config core.abbrev 9
git fetch --unshallow || true
git fetch --tags
mkdir build
cd build
cmake .. \
-DCMAKE_INSTALL_PREFIX="$(pwd)/release" \
-DCMAKE_PREFIX_PATH="$PREFIX" \
-DCMAKE_BUILD_TYPE=Release \
-DZIG_TARGET_TRIPLE="$TARGET" \
-DZIG_TARGET_MCPU="$MCPU" \
-DZIG_STATIC=ON
# Now cmake will use zig as the C/C++ compiler. We reset the environment variables
# so that installation and testing do not get affected by them.
unset CC
unset CXX
make $JOBS install
# Look for non-conforming code formatting.
# Formatting errors can be fixed by running `zig fmt` on the files printed here.
release/bin/zig fmt --check ..
# Here we rebuild zig but this time using the Zig binary we just now produced to
# build zig1.o rather than relying on the one built with stage0. See
# https://github.com/ziglang/zig/issues/6830 for more details.
cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig"
make $JOBS install
release/bin/zig test ../test/behavior.zig -fno-stage1 -fLLVM -I ../test
release/bin/zig build test-behavior -Denable-qemu -Denable-wasmtime
release/bin/zig build test-compiler-rt -Denable-qemu -Denable-wasmtime
release/bin/zig build test-std -Denable-qemu -Denable-wasmtime
release/bin/zig build test-minilibc -Denable-qemu -Denable-wasmtime
release/bin/zig build test-compare-output -Denable-qemu -Denable-wasmtime
release/bin/zig build test-standalone -Denable-qemu -Denable-wasmtime
release/bin/zig build test-stack-traces -Denable-qemu -Denable-wasmtime
release/bin/zig build test-cli -Denable-qemu -Denable-wasmtime
release/bin/zig build test-asm-link -Denable-qemu -Denable-wasmtime
release/bin/zig build test-runtime-safety -Denable-qemu -Denable-wasmtime
release/bin/zig build test-translate-c -Denable-qemu -Denable-wasmtime
release/bin/zig build test-run-translated-c -Denable-qemu -Denable-wasmtime
release/bin/zig build docs -Denable-qemu -Denable-wasmtime
release/bin/zig build # test building self-hosted without LLVM
release/bin/zig build test-fmt -Denable-qemu -Denable-wasmtime
release/bin/zig build test-stage2 -Denable-qemu -Denable-wasmtime
# Look for HTML errors.
tidy --drop-empty-elements no -qe ../zig-cache/langref.html
if [ "${BUILD_REASON}" != "PullRequest" ]; then
# Produce the experimental std lib documentation.
mkdir -p release/docs/std
release/bin/zig test ../lib/std/std.zig \
--zig-lib-dir ../lib \
-femit-docs=release/docs/std \
-fno-emit-bin
mv ../LICENSE release/
mv ../zig-cache/langref.html release/docs/
# Remove the unnecessary bin dir in $prefix/bin/zig
mv release/bin/zig release/
rmdir release/bin
# Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig
mv release/lib/zig release/lib2
rmdir release/lib
mv release/lib2 release/lib
VERSION=$(release/zig version)
DIRNAME="zig-linux-$ARCH-$VERSION"
TARBALL="$DIRNAME.tar.xz"
mv release "$DIRNAME"
tar cfJ "$TARBALL" "$DIRNAME"
mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"
s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "$TARBALL" s3://ziglang.org/builds/
SHASUM=$(sha256sum $TARBALL | cut '-d ' -f1)
BYTESIZE=$(wc -c < $TARBALL)
JSONFILE="linux-$GITBRANCH.json"
touch $JSONFILE
echo "{\"tarball\": \"$TARBALL\"," >>$JSONFILE
echo "\"shasum\": \"$SHASUM\"," >>$JSONFILE
echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$ARCH-linux-$VERSION.json"
# `set -x` causes these variables to be mangled.
# See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
set +x
echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM"
echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE"
echo "##vso[task.setvariable variable=version;isOutput=true]$VERSION"
fi

View File

@ -7,23 +7,23 @@ ZIG=$DEBUG_STAGING/bin/zig
$ZIG test test/behavior.zig -fno-stage1 -fLLVM -I test
$ZIG test test/behavior.zig -fno-stage1 -ofmt=c -I test
$ZIG build test-behavior -Denable-qemu -Denable-wasmtime
$ZIG build test-compiler-rt -Denable-qemu -Denable-wasmtime
$ZIG build test-std -Denable-qemu -Denable-wasmtime
$ZIG build test-minilibc -Denable-qemu -Denable-wasmtime
$ZIG build test-compare-output -Denable-qemu -Denable-wasmtime
$ZIG build test-standalone -Denable-qemu -Denable-wasmtime
$ZIG build test-stack-traces -Denable-qemu -Denable-wasmtime
$ZIG build test-cli -Denable-qemu -Denable-wasmtime
$ZIG build test-asm-link -Denable-qemu -Denable-wasmtime
$ZIG build test-runtime-safety -Denable-qemu -Denable-wasmtime
$ZIG build test-translate-c -Denable-qemu -Denable-wasmtime
$ZIG build test-run-translated-c -Denable-qemu -Denable-wasmtime
$ZIG build docs -Denable-qemu -Denable-wasmtime
$ZIG build test-behavior -fqemu -fwasmtime
$ZIG build test-compiler-rt -fqemu -fwasmtime
$ZIG build test-std -fqemu -fwasmtime
$ZIG build test-minilibc -fqemu -fwasmtime
$ZIG build test-compare-output -fqemu -fwasmtime
$ZIG build test-standalone -fqemu -fwasmtime
$ZIG build test-stack-traces -fqemu -fwasmtime
$ZIG build test-cli -fqemu -fwasmtime
$ZIG build test-asm-link -fqemu -fwasmtime
$ZIG build test-runtime-safety -fqemu -fwasmtime
$ZIG build test-translate-c -fqemu -fwasmtime
$ZIG build test-run-translated-c -fqemu -fwasmtime
$ZIG build docs -fqemu -fwasmtime
$ZIG build # test building self-hosted without LLVM
$ZIG build -Dtarget=arm-linux-musleabihf # test building self-hosted for 32-bit arm
$ZIG build test-fmt -Denable-qemu -Denable-wasmtime
$ZIG build test-stage2 -Denable-qemu -Denable-wasmtime
$ZIG build test-fmt -fqemu -fwasmtime
$ZIG build test-stage2 -fqemu -fwasmtime
# Produce the experimental std lib documentation.
mkdir -p $RELEASE_STAGING/docs/std

View File

@ -70,6 +70,22 @@ pub const Builder = struct {
args: ?[][]const u8 = null,
debug_log_scopes: []const []const u8 = &.{},
/// Experimental. Use system Darling installation to run cross compiled macOS build artifacts.
enable_darling: bool = false,
/// Use system QEMU installation to run cross compiled foreign architecture build artifacts.
enable_qemu: bool = false,
/// Darwin. Use Rosetta to run x86_64 macOS build artifacts on arm64 macOS.
enable_rosetta: bool = false,
/// Use system Wasmtime installation to run cross compiled wasm/wasi build artifacts.
enable_wasmtime: bool = false,
/// Use system Wine installation to run cross compiled Windows build artifacts.
enable_wine: bool = false,
/// After following the steps in https://github.com/ziglang/zig/wiki/Updating-libc#glibc,
/// this will be the directory $glibc-build-dir/install/glibcs
/// Given the example of the aarch64 target, this is the directory
/// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`.
glibc_runtimes_dir: ?[]const u8 = null,
const PkgConfigError = error{
PkgConfigCrashed,
PkgConfigFailed,
@ -1504,27 +1520,6 @@ pub const LibExeObjStep = struct {
/// Permit read-only relocations in read-only segments. Disallowed by default.
link_z_notext: bool = false,
/// Uses system Wine installation to run cross compiled Windows build artifacts.
enable_wine: bool = false,
/// Uses system QEMU installation to run cross compiled foreign architecture build artifacts.
enable_qemu: bool = false,
/// Uses system Wasmtime installation to run cross compiled wasm/wasi build artifacts.
enable_wasmtime: bool = false,
/// Experimental. Uses system Darling installation to run cross compiled macOS build artifacts.
enable_darling: bool = false,
/// Darwin. Uses Rosetta to run x86_64 macOS build artifacts on arm64 macOS.
enable_rosetta: bool = false,
/// After following the steps in https://github.com/ziglang/zig/wiki/Updating-libc#glibc,
/// this will be the directory $glibc-build-dir/install/glibcs
/// Given the example of the aarch64 target, this is the directory
/// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`.
glibc_multi_install_dir: ?[]const u8 = null,
/// Position Independent Code
force_pic: ?bool = null,
@ -2533,13 +2528,13 @@ pub const LibExeObjStep = struct {
}
} else switch (self.target.getExternalExecutor()) {
.native, .unavailable => {},
.rosetta => if (self.enable_rosetta) {
.rosetta => if (builder.enable_rosetta) {
try zig_args.append("--test-cmd-bin");
},
.qemu => |bin_name| if (self.enable_qemu) qemu: {
.qemu => |bin_name| if (builder.enable_qemu) qemu: {
const need_cross_glibc = self.target.isGnuLibC() and self.is_linking_libc;
const glibc_dir_arg = if (need_cross_glibc)
self.glibc_multi_install_dir orelse break :qemu
builder.glibc_runtimes_dir orelse break :qemu
else
null;
try zig_args.append("--test-cmd");
@ -2567,19 +2562,19 @@ pub const LibExeObjStep = struct {
}
try zig_args.append("--test-cmd-bin");
},
.wine => |bin_name| if (self.enable_wine) {
.wine => |bin_name| if (builder.enable_wine) {
try zig_args.append("--test-cmd");
try zig_args.append(bin_name);
try zig_args.append("--test-cmd-bin");
},
.wasmtime => |bin_name| if (self.enable_wasmtime) {
.wasmtime => |bin_name| if (builder.enable_wasmtime) {
try zig_args.append("--test-cmd");
try zig_args.append(bin_name);
try zig_args.append("--test-cmd");
try zig_args.append("--dir=.");
try zig_args.append("--test-cmd-bin");
},
.darling => |bin_name| if (self.enable_darling) {
.darling => |bin_name| if (builder.enable_darling) {
try zig_args.append("--test-cmd");
try zig_args.append(bin_name);
try zig_args.append("--test-cmd-bin");

View File

@ -142,6 +142,11 @@ pub fn main() !void {
return usageAndErr(builder, false, stderr_stream);
};
try debug_log_scopes.append(next_arg);
} else if (mem.eql(u8, arg, "--glibc-runtimes")) {
builder.glibc_runtimes_dir = nextArg(args, &arg_idx) orelse {
std.debug.print("Expected argument after --glibc-runtimes\n\n", .{});
return usageAndErr(builder, false, stderr_stream);
};
} else if (mem.eql(u8, arg, "--verbose-tokenize")) {
builder.verbose_tokenize = true;
} else if (mem.eql(u8, arg, "--verbose-ast")) {
@ -160,6 +165,26 @@ pub fn main() !void {
builder.verbose_llvm_cpu_features = true;
} else if (mem.eql(u8, arg, "--prominent-compile-errors")) {
builder.prominent_compile_errors = true;
} else if (mem.eql(u8, arg, "-fwine")) {
builder.enable_wine = true;
} else if (mem.eql(u8, arg, "-fno-wine")) {
builder.enable_wine = false;
} else if (mem.eql(u8, arg, "-fqemu")) {
builder.enable_qemu = true;
} else if (mem.eql(u8, arg, "-fno-qemu")) {
builder.enable_qemu = false;
} else if (mem.eql(u8, arg, "-fwasmtime")) {
builder.enable_wasmtime = true;
} else if (mem.eql(u8, arg, "-fno-wasmtime")) {
builder.enable_wasmtime = false;
} else if (mem.eql(u8, arg, "-frosetta")) {
builder.enable_rosetta = true;
} else if (mem.eql(u8, arg, "-fno-rosetta")) {
builder.enable_rosetta = false;
} else if (mem.eql(u8, arg, "-fdarling")) {
builder.enable_darling = true;
} else if (mem.eql(u8, arg, "-fno-darling")) {
builder.enable_darling = false;
} else if (mem.eql(u8, arg, "--")) {
builder.args = argsRest(args, arg_idx);
break;
@ -233,6 +258,22 @@ fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void
\\ --search-prefix [path] Add a path to look for binaries, libraries, headers
\\ --libc [file] Provide a file which specifies libc paths
\\
\\ -fdarling, -fno-darling Integration with system-installed Darling to
\\ execute macOS programs on Linux hosts
\\ (default: no)
\\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute
\\ foreign-architecture programs on Linux hosts
\\ (default: no)
\\ --glibc-runtimes [path] Enhances QEMU integration by providing glibc built
\\ for multiple foreign architectures, allowing
\\ execution of non-native programs that link with glibc.
\\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on
\\ ARM64 macOS hosts. (default: no)
\\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to
\\ execute WASI binaries. (default: no)
\\ -fwine, -fno-wine Integration with system-installed Wine to execute
\\ Windows programs on Linux hosts. (default: no)
\\
\\ -h, --help Print this help and exit
\\ --verbose Print commands before executing them
\\ --color [auto|off|on] Enable or disable colored error messages

View File

@ -11,7 +11,7 @@ const enable_wine: bool = build_options.enable_wine;
const enable_wasmtime: bool = build_options.enable_wasmtime;
const enable_darling: bool = build_options.enable_darling;
const enable_rosetta: bool = build_options.enable_rosetta;
const glibc_multi_install_dir: ?[]const u8 = build_options.glibc_multi_install_dir;
const glibc_runtimes_dir: ?[]const u8 = build_options.glibc_runtimes_dir;
const skip_compile_errors = build_options.skip_compile_errors;
const ThreadPool = @import("ThreadPool.zig");
const CrossTarget = std.zig.CrossTarget;
@ -1143,7 +1143,7 @@ pub const TestContext = struct {
// TODO Ability for test cases to specify whether to link libc.
const need_cross_glibc = false; // target.isGnuLibC() and self.is_linking_libc;
const glibc_dir_arg = if (need_cross_glibc)
glibc_multi_install_dir orelse return // glibc dir not available; pass test
glibc_runtimes_dir orelse return // glibc dir not available; pass test
else
null;
try argv.append(qemu_bin_name);

View File

@ -513,12 +513,6 @@ pub fn addPkgTests(
skip_single_threaded: bool,
skip_non_native: bool,
skip_libc: bool,
is_wine_enabled: bool,
is_qemu_enabled: bool,
is_wasmtime_enabled: bool,
is_darling_enabled: bool,
is_rosetta_enabled: bool,
glibc_dir: ?[]const u8,
) *build.Step {
const step = b.step(b.fmt("test-{s}", .{name}), desc);
@ -575,12 +569,6 @@ pub fn addPkgTests(
these_tests.linkSystemLibrary("c");
}
these_tests.overrideZigLibDir("lib");
these_tests.enable_wine = is_wine_enabled;
these_tests.enable_qemu = is_qemu_enabled;
these_tests.enable_wasmtime = is_wasmtime_enabled;
these_tests.enable_darling = is_darling_enabled;
these_tests.enable_rosetta = is_rosetta_enabled;
these_tests.glibc_multi_install_dir = glibc_dir;
these_tests.addIncludeDir("test");
step.dependOn(&these_tests.step);