From b24cbecdb2abb4399d65553ebade318263cd57d3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Dec 2021 15:42:59 -0700 Subject: [PATCH] zig build: promote qemu, wine, wasmtime, darling, and rosetta from zig-specific options to generally recognized zig build options that any project can take advantage of. See the updated usage text for more details. --- build.zig | 43 ++-------- ci/azure/linux_script | 142 ------------------------------- ci/zinc/linux_test.sh | 30 +++---- lib/std/build.zig | 49 +++++------ lib/std/special/build_runner.zig | 41 +++++++++ src/test.zig | 4 +- test/tests.zig | 12 --- 7 files changed, 86 insertions(+), 235 deletions(-) delete mode 100755 ci/azure/linux_script diff --git a/build.zig b/build.zig index 28d1486ad0..e27f1eda95 100644 --- a/build.zig +++ b/build.zig @@ -296,13 +296,6 @@ pub fn build(b: *Builder) !void { const test_filter = b.option([]const u8, "test-filter", "Skip tests that do not match filter"); - const is_wine_enabled = b.option(bool, "enable-wine", "Use Wine to run cross compiled Windows tests") orelse false; - const is_qemu_enabled = b.option(bool, "enable-qemu", "Use QEMU to run cross compiled foreign architecture tests") orelse false; - const is_wasmtime_enabled = b.option(bool, "enable-wasmtime", "Use Wasmtime to enable and run WASI libstd tests") orelse false; - const is_darling_enabled = b.option(bool, "enable-darling", "[Experimental] Use Darling to run cross compiled macOS tests") orelse false; - const is_rosetta_enabled = b.option(bool, "enable-rosetta", "(Darwin) Use Rosetta to run x86_64 macOS tests on arm64 macOS") orelse false; - const glibc_multi_dir = b.option([]const u8, "enable-foreign-glibc", "Provide directory with glibc installations to run cross compiled tests that link glibc"); - const test_stage2_options = b.addOptions(); test_stage2.addOptions("build_options", test_stage2_options); @@ -317,13 +310,13 @@ pub fn build(b: *Builder) !void { test_stage2_options.addOption(bool, "llvm_has_csky", llvm_has_csky); test_stage2_options.addOption(bool, "llvm_has_ve", llvm_has_ve); test_stage2_options.addOption(bool, "llvm_has_arc", llvm_has_arc); - test_stage2_options.addOption(bool, "enable_qemu", is_qemu_enabled); - test_stage2_options.addOption(bool, "enable_wine", is_wine_enabled); - test_stage2_options.addOption(bool, "enable_wasmtime", is_wasmtime_enabled); - test_stage2_options.addOption(bool, "enable_rosetta", is_rosetta_enabled); + test_stage2_options.addOption(bool, "enable_qemu", b.enable_qemu); + test_stage2_options.addOption(bool, "enable_wine", b.enable_wine); + test_stage2_options.addOption(bool, "enable_wasmtime", b.enable_wasmtime); + test_stage2_options.addOption(bool, "enable_rosetta", b.enable_rosetta); + test_stage2_options.addOption(bool, "enable_darling", b.enable_darling); test_stage2_options.addOption(u32, "mem_leak_frames", mem_leak_frames * 2); - test_stage2_options.addOption(bool, "enable_darling", is_darling_enabled); - test_stage2_options.addOption(?[]const u8, "glibc_multi_install_dir", glibc_multi_dir); + test_stage2_options.addOption(?[]const u8, "glibc_runtimes_dir", b.glibc_runtimes_dir); test_stage2_options.addOption([:0]const u8, "version", try b.allocator.dupeZ(u8, version)); test_stage2_options.addOption(std.SemanticVersion, "semver", semver); @@ -368,12 +361,6 @@ pub fn build(b: *Builder) !void { false, // skip_single_threaded skip_non_native, skip_libc, - is_wine_enabled, - is_qemu_enabled, - is_wasmtime_enabled, - is_darling_enabled, - is_rosetta_enabled, - glibc_multi_dir, )); toolchain_step.dependOn(tests.addPkgTests( @@ -386,12 +373,6 @@ pub fn build(b: *Builder) !void { true, // skip_single_threaded skip_non_native, true, // skip_libc - is_wine_enabled, - is_qemu_enabled, - is_wasmtime_enabled, - is_darling_enabled, - is_rosetta_enabled, - glibc_multi_dir, )); toolchain_step.dependOn(tests.addPkgTests( @@ -404,12 +385,6 @@ pub fn build(b: *Builder) !void { true, // skip_single_threaded skip_non_native, true, // skip_libc - is_wine_enabled, - is_qemu_enabled, - is_wasmtime_enabled, - is_darling_enabled, - is_rosetta_enabled, - glibc_multi_dir, )); toolchain_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes)); @@ -435,12 +410,6 @@ pub fn build(b: *Builder) !void { false, skip_non_native, skip_libc, - is_wine_enabled, - is_qemu_enabled, - is_wasmtime_enabled, - is_darling_enabled, - is_rosetta_enabled, - glibc_multi_dir, ); const test_step = b.step("test", "Run all the tests"); diff --git a/ci/azure/linux_script b/ci/azure/linux_script deleted file mode 100755 index 857e87cab8..0000000000 --- a/ci/azure/linux_script +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/sh - -set -x -set -e - -sudo apt-get update -q -sudo apt-get install -y cmake s3cmd tidy - -ZIGDIR="$(pwd)" -ARCH="$(uname -m)" -TARGET="$ARCH-linux-musl" -CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.9.0-dev.1243+456d7e5f5" -PREFIX="$HOME/$CACHE_BASENAME" -MCPU="baseline" -JOBS="-j$(nproc)" - -rm -rf $PREFIX -cd $HOME - -wget -nv "https://ziglang.org/deps/$CACHE_BASENAME.tar.xz" -tar xf "$CACHE_BASENAME.tar.xz" - -QEMUBASE="qemu-linux-x86_64-6.1.0.1" -wget -nv "https://ziglang.org/deps/$QEMUBASE.tar.xz" -tar xf "$QEMUBASE.tar.xz" -export PATH="$(pwd)/$QEMUBASE/bin:$PATH" - -WASMTIME="wasmtime-v0.26.1-x86_64-linux" -wget -nv "https://github.com/bytecodealliance/wasmtime/releases/download/v0.26.1/$WASMTIME.tar.xz" -tar xf "$WASMTIME.tar.xz" -export PATH="$(pwd)/$WASMTIME:$PATH" - -ZIG="$PREFIX/bin/zig" -export CC="$ZIG cc -target $TARGET -mcpu=$MCPU" -export CXX="$ZIG c++ -target $TARGET -mcpu=$MCPU" - -cd $ZIGDIR - -# Make the `zig version` number consistent. -# This will affect the cmake command below. -git config core.abbrev 9 -git fetch --unshallow || true -git fetch --tags - -mkdir build -cd build -cmake .. \ - -DCMAKE_INSTALL_PREFIX="$(pwd)/release" \ - -DCMAKE_PREFIX_PATH="$PREFIX" \ - -DCMAKE_BUILD_TYPE=Release \ - -DZIG_TARGET_TRIPLE="$TARGET" \ - -DZIG_TARGET_MCPU="$MCPU" \ - -DZIG_STATIC=ON - -# Now cmake will use zig as the C/C++ compiler. We reset the environment variables -# so that installation and testing do not get affected by them. -unset CC -unset CXX - -make $JOBS install - -# Look for non-conforming code formatting. -# Formatting errors can be fixed by running `zig fmt` on the files printed here. -release/bin/zig fmt --check .. - -# Here we rebuild zig but this time using the Zig binary we just now produced to -# build zig1.o rather than relying on the one built with stage0. See -# https://github.com/ziglang/zig/issues/6830 for more details. -cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig" -make $JOBS install - -release/bin/zig test ../test/behavior.zig -fno-stage1 -fLLVM -I ../test - -release/bin/zig build test-behavior -Denable-qemu -Denable-wasmtime -release/bin/zig build test-compiler-rt -Denable-qemu -Denable-wasmtime -release/bin/zig build test-std -Denable-qemu -Denable-wasmtime -release/bin/zig build test-minilibc -Denable-qemu -Denable-wasmtime -release/bin/zig build test-compare-output -Denable-qemu -Denable-wasmtime -release/bin/zig build test-standalone -Denable-qemu -Denable-wasmtime -release/bin/zig build test-stack-traces -Denable-qemu -Denable-wasmtime -release/bin/zig build test-cli -Denable-qemu -Denable-wasmtime -release/bin/zig build test-asm-link -Denable-qemu -Denable-wasmtime -release/bin/zig build test-runtime-safety -Denable-qemu -Denable-wasmtime -release/bin/zig build test-translate-c -Denable-qemu -Denable-wasmtime -release/bin/zig build test-run-translated-c -Denable-qemu -Denable-wasmtime -release/bin/zig build docs -Denable-qemu -Denable-wasmtime -release/bin/zig build # test building self-hosted without LLVM -release/bin/zig build test-fmt -Denable-qemu -Denable-wasmtime -release/bin/zig build test-stage2 -Denable-qemu -Denable-wasmtime - -# Look for HTML errors. -tidy --drop-empty-elements no -qe ../zig-cache/langref.html - -if [ "${BUILD_REASON}" != "PullRequest" ]; then - # Produce the experimental std lib documentation. - mkdir -p release/docs/std - release/bin/zig test ../lib/std/std.zig \ - --zig-lib-dir ../lib \ - -femit-docs=release/docs/std \ - -fno-emit-bin - - mv ../LICENSE release/ - mv ../zig-cache/langref.html release/docs/ - - # Remove the unnecessary bin dir in $prefix/bin/zig - mv release/bin/zig release/ - rmdir release/bin - - # Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig - mv release/lib/zig release/lib2 - rmdir release/lib - mv release/lib2 release/lib - - VERSION=$(release/zig version) - DIRNAME="zig-linux-$ARCH-$VERSION" - TARBALL="$DIRNAME.tar.xz" - mv release "$DIRNAME" - tar cfJ "$TARBALL" "$DIRNAME" - - mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg" - s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "$TARBALL" s3://ziglang.org/builds/ - - SHASUM=$(sha256sum $TARBALL | cut '-d ' -f1) - BYTESIZE=$(wc -c < $TARBALL) - - JSONFILE="linux-$GITBRANCH.json" - touch $JSONFILE - echo "{\"tarball\": \"$TARBALL\"," >>$JSONFILE - echo "\"shasum\": \"$SHASUM\"," >>$JSONFILE - echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE - - s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" - s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$ARCH-linux-$VERSION.json" - - # `set -x` causes these variables to be mangled. - # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html - set +x - echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL" - echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM" - echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE" - echo "##vso[task.setvariable variable=version;isOutput=true]$VERSION" -fi diff --git a/ci/zinc/linux_test.sh b/ci/zinc/linux_test.sh index 9dc61fcd5e..fa30bd5cdf 100755 --- a/ci/zinc/linux_test.sh +++ b/ci/zinc/linux_test.sh @@ -7,23 +7,23 @@ ZIG=$DEBUG_STAGING/bin/zig $ZIG test test/behavior.zig -fno-stage1 -fLLVM -I test $ZIG test test/behavior.zig -fno-stage1 -ofmt=c -I test -$ZIG build test-behavior -Denable-qemu -Denable-wasmtime -$ZIG build test-compiler-rt -Denable-qemu -Denable-wasmtime -$ZIG build test-std -Denable-qemu -Denable-wasmtime -$ZIG build test-minilibc -Denable-qemu -Denable-wasmtime -$ZIG build test-compare-output -Denable-qemu -Denable-wasmtime -$ZIG build test-standalone -Denable-qemu -Denable-wasmtime -$ZIG build test-stack-traces -Denable-qemu -Denable-wasmtime -$ZIG build test-cli -Denable-qemu -Denable-wasmtime -$ZIG build test-asm-link -Denable-qemu -Denable-wasmtime -$ZIG build test-runtime-safety -Denable-qemu -Denable-wasmtime -$ZIG build test-translate-c -Denable-qemu -Denable-wasmtime -$ZIG build test-run-translated-c -Denable-qemu -Denable-wasmtime -$ZIG build docs -Denable-qemu -Denable-wasmtime +$ZIG build test-behavior -fqemu -fwasmtime +$ZIG build test-compiler-rt -fqemu -fwasmtime +$ZIG build test-std -fqemu -fwasmtime +$ZIG build test-minilibc -fqemu -fwasmtime +$ZIG build test-compare-output -fqemu -fwasmtime +$ZIG build test-standalone -fqemu -fwasmtime +$ZIG build test-stack-traces -fqemu -fwasmtime +$ZIG build test-cli -fqemu -fwasmtime +$ZIG build test-asm-link -fqemu -fwasmtime +$ZIG build test-runtime-safety -fqemu -fwasmtime +$ZIG build test-translate-c -fqemu -fwasmtime +$ZIG build test-run-translated-c -fqemu -fwasmtime +$ZIG build docs -fqemu -fwasmtime $ZIG build # test building self-hosted without LLVM $ZIG build -Dtarget=arm-linux-musleabihf # test building self-hosted for 32-bit arm -$ZIG build test-fmt -Denable-qemu -Denable-wasmtime -$ZIG build test-stage2 -Denable-qemu -Denable-wasmtime +$ZIG build test-fmt -fqemu -fwasmtime +$ZIG build test-stage2 -fqemu -fwasmtime # Produce the experimental std lib documentation. mkdir -p $RELEASE_STAGING/docs/std diff --git a/lib/std/build.zig b/lib/std/build.zig index 600fb691ae..041510bb02 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -70,6 +70,22 @@ pub const Builder = struct { args: ?[][]const u8 = null, debug_log_scopes: []const []const u8 = &.{}, + /// Experimental. Use system Darling installation to run cross compiled macOS build artifacts. + enable_darling: bool = false, + /// Use system QEMU installation to run cross compiled foreign architecture build artifacts. + enable_qemu: bool = false, + /// Darwin. Use Rosetta to run x86_64 macOS build artifacts on arm64 macOS. + enable_rosetta: bool = false, + /// Use system Wasmtime installation to run cross compiled wasm/wasi build artifacts. + enable_wasmtime: bool = false, + /// Use system Wine installation to run cross compiled Windows build artifacts. + enable_wine: bool = false, + /// After following the steps in https://github.com/ziglang/zig/wiki/Updating-libc#glibc, + /// this will be the directory $glibc-build-dir/install/glibcs + /// Given the example of the aarch64 target, this is the directory + /// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. + glibc_runtimes_dir: ?[]const u8 = null, + const PkgConfigError = error{ PkgConfigCrashed, PkgConfigFailed, @@ -1504,27 +1520,6 @@ pub const LibExeObjStep = struct { /// Permit read-only relocations in read-only segments. Disallowed by default. link_z_notext: bool = false, - /// Uses system Wine installation to run cross compiled Windows build artifacts. - enable_wine: bool = false, - - /// Uses system QEMU installation to run cross compiled foreign architecture build artifacts. - enable_qemu: bool = false, - - /// Uses system Wasmtime installation to run cross compiled wasm/wasi build artifacts. - enable_wasmtime: bool = false, - - /// Experimental. Uses system Darling installation to run cross compiled macOS build artifacts. - enable_darling: bool = false, - - /// Darwin. Uses Rosetta to run x86_64 macOS build artifacts on arm64 macOS. - enable_rosetta: bool = false, - - /// After following the steps in https://github.com/ziglang/zig/wiki/Updating-libc#glibc, - /// this will be the directory $glibc-build-dir/install/glibcs - /// Given the example of the aarch64 target, this is the directory - /// that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. - glibc_multi_install_dir: ?[]const u8 = null, - /// Position Independent Code force_pic: ?bool = null, @@ -2533,13 +2528,13 @@ pub const LibExeObjStep = struct { } } else switch (self.target.getExternalExecutor()) { .native, .unavailable => {}, - .rosetta => if (self.enable_rosetta) { + .rosetta => if (builder.enable_rosetta) { try zig_args.append("--test-cmd-bin"); }, - .qemu => |bin_name| if (self.enable_qemu) qemu: { + .qemu => |bin_name| if (builder.enable_qemu) qemu: { const need_cross_glibc = self.target.isGnuLibC() and self.is_linking_libc; const glibc_dir_arg = if (need_cross_glibc) - self.glibc_multi_install_dir orelse break :qemu + builder.glibc_runtimes_dir orelse break :qemu else null; try zig_args.append("--test-cmd"); @@ -2567,19 +2562,19 @@ pub const LibExeObjStep = struct { } try zig_args.append("--test-cmd-bin"); }, - .wine => |bin_name| if (self.enable_wine) { + .wine => |bin_name| if (builder.enable_wine) { try zig_args.append("--test-cmd"); try zig_args.append(bin_name); try zig_args.append("--test-cmd-bin"); }, - .wasmtime => |bin_name| if (self.enable_wasmtime) { + .wasmtime => |bin_name| if (builder.enable_wasmtime) { try zig_args.append("--test-cmd"); try zig_args.append(bin_name); try zig_args.append("--test-cmd"); try zig_args.append("--dir=."); try zig_args.append("--test-cmd-bin"); }, - .darling => |bin_name| if (self.enable_darling) { + .darling => |bin_name| if (builder.enable_darling) { try zig_args.append("--test-cmd"); try zig_args.append(bin_name); try zig_args.append("--test-cmd-bin"); diff --git a/lib/std/special/build_runner.zig b/lib/std/special/build_runner.zig index d798d2ab6f..2a64861cf9 100644 --- a/lib/std/special/build_runner.zig +++ b/lib/std/special/build_runner.zig @@ -142,6 +142,11 @@ pub fn main() !void { return usageAndErr(builder, false, stderr_stream); }; try debug_log_scopes.append(next_arg); + } else if (mem.eql(u8, arg, "--glibc-runtimes")) { + builder.glibc_runtimes_dir = nextArg(args, &arg_idx) orelse { + std.debug.print("Expected argument after --glibc-runtimes\n\n", .{}); + return usageAndErr(builder, false, stderr_stream); + }; } else if (mem.eql(u8, arg, "--verbose-tokenize")) { builder.verbose_tokenize = true; } else if (mem.eql(u8, arg, "--verbose-ast")) { @@ -160,6 +165,26 @@ pub fn main() !void { builder.verbose_llvm_cpu_features = true; } else if (mem.eql(u8, arg, "--prominent-compile-errors")) { builder.prominent_compile_errors = true; + } else if (mem.eql(u8, arg, "-fwine")) { + builder.enable_wine = true; + } else if (mem.eql(u8, arg, "-fno-wine")) { + builder.enable_wine = false; + } else if (mem.eql(u8, arg, "-fqemu")) { + builder.enable_qemu = true; + } else if (mem.eql(u8, arg, "-fno-qemu")) { + builder.enable_qemu = false; + } else if (mem.eql(u8, arg, "-fwasmtime")) { + builder.enable_wasmtime = true; + } else if (mem.eql(u8, arg, "-fno-wasmtime")) { + builder.enable_wasmtime = false; + } else if (mem.eql(u8, arg, "-frosetta")) { + builder.enable_rosetta = true; + } else if (mem.eql(u8, arg, "-fno-rosetta")) { + builder.enable_rosetta = false; + } else if (mem.eql(u8, arg, "-fdarling")) { + builder.enable_darling = true; + } else if (mem.eql(u8, arg, "-fno-darling")) { + builder.enable_darling = false; } else if (mem.eql(u8, arg, "--")) { builder.args = argsRest(args, arg_idx); break; @@ -233,6 +258,22 @@ fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void \\ --search-prefix [path] Add a path to look for binaries, libraries, headers \\ --libc [file] Provide a file which specifies libc paths \\ + \\ -fdarling, -fno-darling Integration with system-installed Darling to + \\ execute macOS programs on Linux hosts + \\ (default: no) + \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute + \\ foreign-architecture programs on Linux hosts + \\ (default: no) + \\ --glibc-runtimes [path] Enhances QEMU integration by providing glibc built + \\ for multiple foreign architectures, allowing + \\ execution of non-native programs that link with glibc. + \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on + \\ ARM64 macOS hosts. (default: no) + \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to + \\ execute WASI binaries. (default: no) + \\ -fwine, -fno-wine Integration with system-installed Wine to execute + \\ Windows programs on Linux hosts. (default: no) + \\ \\ -h, --help Print this help and exit \\ --verbose Print commands before executing them \\ --color [auto|off|on] Enable or disable colored error messages diff --git a/src/test.zig b/src/test.zig index 69d2d31a08..08c33b419d 100644 --- a/src/test.zig +++ b/src/test.zig @@ -11,7 +11,7 @@ const enable_wine: bool = build_options.enable_wine; const enable_wasmtime: bool = build_options.enable_wasmtime; const enable_darling: bool = build_options.enable_darling; const enable_rosetta: bool = build_options.enable_rosetta; -const glibc_multi_install_dir: ?[]const u8 = build_options.glibc_multi_install_dir; +const glibc_runtimes_dir: ?[]const u8 = build_options.glibc_runtimes_dir; const skip_compile_errors = build_options.skip_compile_errors; const ThreadPool = @import("ThreadPool.zig"); const CrossTarget = std.zig.CrossTarget; @@ -1143,7 +1143,7 @@ pub const TestContext = struct { // TODO Ability for test cases to specify whether to link libc. const need_cross_glibc = false; // target.isGnuLibC() and self.is_linking_libc; const glibc_dir_arg = if (need_cross_glibc) - glibc_multi_install_dir orelse return // glibc dir not available; pass test + glibc_runtimes_dir orelse return // glibc dir not available; pass test else null; try argv.append(qemu_bin_name); diff --git a/test/tests.zig b/test/tests.zig index 511d8bc6f2..99d84d14a7 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -513,12 +513,6 @@ pub fn addPkgTests( skip_single_threaded: bool, skip_non_native: bool, skip_libc: bool, - is_wine_enabled: bool, - is_qemu_enabled: bool, - is_wasmtime_enabled: bool, - is_darling_enabled: bool, - is_rosetta_enabled: bool, - glibc_dir: ?[]const u8, ) *build.Step { const step = b.step(b.fmt("test-{s}", .{name}), desc); @@ -575,12 +569,6 @@ pub fn addPkgTests( these_tests.linkSystemLibrary("c"); } these_tests.overrideZigLibDir("lib"); - these_tests.enable_wine = is_wine_enabled; - these_tests.enable_qemu = is_qemu_enabled; - these_tests.enable_wasmtime = is_wasmtime_enabled; - these_tests.enable_darling = is_darling_enabled; - these_tests.enable_rosetta = is_rosetta_enabled; - these_tests.glibc_multi_install_dir = glibc_dir; these_tests.addIncludeDir("test"); step.dependOn(&these_tests.step);