From bc592859696b9d6183f5b891d56a9f1032d757f5 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Thu, 11 Jun 2026 23:32:24 -0400 Subject: [PATCH 01/10] executor: fix RuntimeMountCache deinit --- src/action_executor.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/action_executor.zig b/src/action_executor.zig index 6c6c252..a2578da 100644 --- a/src/action_executor.zig +++ b/src/action_executor.zig @@ -50,7 +50,7 @@ pub const RuntimeMountCache = struct { self.glibc2_35.deinit(allocator); self.glibc2_39.deinit(allocator); self.bash.deinit(allocator); - self.* = .{}; + self.* = undefined; } fn forLibc(self: *const RuntimeMountCache, libc: []const u8) ?*const RuntimeMountSources { From 033fc193fb6c146b33114b56f9c4ea4d6fd0a496 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Thu, 11 Jun 2026 23:38:00 -0400 Subject: [PATCH 02/10] linux: run embedded QEMU from memfd --- .github/workflows/build_release.sh | 3 + .github/workflows/ci.yml | 29 +++ .github/workflows/release.yml | 1 + ARCHITECTURE.md | 23 +- MODULE.bazel | 15 ++ MODULE.bazel.lock | 5 +- README.md | 26 ++- cmd/linux-actiond/BUILD.bazel | 43 ++++ cmd/linux-actiond/main.zig | 40 ++++ platforms/BUILD.bazel | 9 + src/BUILD.bazel | 4 + src/darwin_vm_host.zig | 65 +----- src/grpc_vsock_bridge.zig | 14 +- src/linux_vm_host.zig | 91 ++++++++ src/qemu_vm.zig | 332 +++++++++++++++++++++++++++++ src/root.zig | 4 + src/vm_host.zig | 94 +++++++- tools/e2e.sh | 50 +++-- tools/zig_embedded_qemu.bzl | 68 ++++++ vm/linux_x86_64.config | 11 + 20 files changed, 823 insertions(+), 104 deletions(-) create mode 100644 cmd/linux-actiond/BUILD.bazel create mode 100644 cmd/linux-actiond/main.zig create mode 100644 src/linux_vm_host.zig create mode 100644 src/qemu_vm.zig create mode 100644 tools/zig_embedded_qemu.bzl diff --git a/.github/workflows/build_release.sh b/.github/workflows/build_release.sh index ae8b8f3..0575736 100755 --- a/.github/workflows/build_release.sh +++ b/.github/workflows/build_release.sh @@ -18,11 +18,13 @@ fi bazel build "${bazel_flags[@]}" -c opt \ //cmd/darwin-actiond:darwin-actiond_macos_arm64 \ + //cmd/linux-actiond:linux-actiond_linux_x86_64 \ //cmd/windows-actiond:windows-actiond_windows_arm64 \ //cmd/windows-actiond:windows-actiond_windows_x86_64 cp -f \ bazel-bin/cmd/darwin-actiond/darwin-actiond_macos_arm64/darwin-actiond_macos_arm64 \ + bazel-bin/cmd/linux-actiond/linux-actiond_linux_x86_64/linux-actiond_linux_x86_64 \ bazel-bin/cmd/windows-actiond/windows-actiond_windows_arm64/windows-actiond_windows_arm64.exe \ bazel-bin/cmd/windows-actiond/windows-actiond_windows_x86_64/windows-actiond_windows_x86_64.exe \ "${artifact_dir}/" @@ -30,6 +32,7 @@ cp -f \ cd "${artifact_dir}" shasum -a 256 \ darwin-actiond_macos_arm64 \ + linux-actiond_linux_x86_64 \ windows-actiond_windows_arm64.exe \ windows-actiond_windows_x86_64.exe \ > SHA256.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af19a72..c0f35d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,35 @@ jobs: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} run: .github/workflows/build_release.sh release + linux-qemu-e2e: + name: Linux QEMU e2e + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Enable vhost-vsock + run: | + if [[ ! -e /dev/vhost-vsock ]]; then + sudo modprobe vhost_vsock + fi + test -c /dev/vhost-vsock + sudo chmod a+rw /dev/vhost-vsock + + - name: Run QEMU e2e + env: + ACTIOND_E2E_BARE_COUNT: 16 + ACTIOND_E2E_NESTED_FILES_PER_GROUP: 8 + ACTIOND_E2E_SOURCE_DIRS: 2 + ACTIOND_E2E_SOURCE_FILES_PER_DIR: 8 + ACTIOND_REPO_BAZEL_FLAGS: --config=remote --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_API_KEY }} + ACTIOND_VM_CAS_IMAGE_SIZE_MIB: 1024 + ACTIOND_VM_CPUS: 2 + ACTIOND_VM_MEMORY_MIB: 1024 + run: tools/e2e.sh vm + windows-llvm-smoke: name: Windows ${{ matrix.mode }} (${{ matrix.architecture }}) strategy: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b227a77..bc77140 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,6 +39,7 @@ jobs: fi gh release upload "${TAG_NAME}" \ release/darwin-actiond_macos_arm64 \ + release/linux-actiond_linux_x86_64 \ release/windows-actiond_windows_arm64.exe \ release/windows-actiond_windows_x86_64.exe \ release/SHA256.txt \ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index df291f2..7971f51 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,9 +1,10 @@ # Architecture `actiond` is a local Remote Execution API worker and cache for Bazel. Its main -mode is `darwin-actiond serve-vm` or `windows-actiond serve-vm`: the host -process owns the public gRPC listener, starts a small Linux VM, and forwards -REAPI traffic into a Linux guest over virtio-vsock or Hyper-V sockets. +mode is `darwin-actiond serve-vm`, `windows-actiond serve-vm`, or +`linux-actiond serve-vm`: the host process owns the public gRPC listener, +starts a small Linux VM, and forwards REAPI traffic into a Linux guest over +virtio-vsock or Hyper-V sockets. The design centers on three ideas: @@ -18,7 +19,7 @@ Bazel | | gRPC / REAPI v -darwin-actiond / windows-actiond +darwin-actiond / windows-actiond / linux-actiond | | TCP-to-virtio-vsock / TCP-to-AF_HYPERV bridge v @@ -32,7 +33,9 @@ guest ext4 disk mounted at /cas On Windows, `windows-actiond` uses Host Compute System `LinuxKernelDirect`, Hyper-V synthetic SCSI, and `AF_HYPERV`. Guest AF_VSOCK port 5001 maps to the standard Hyper-V socket service GUID template. The Windows guest matches the -ARM64 or x86_64 host architecture; the macOS guest is ARM64. +ARM64 or x86_64 host architecture; the macOS guest is ARM64. On Linux x86_64, +`linux-actiond` uses QEMU microvm, virtio-mmio block devices, and +`vhost-vsock-device`. In VM mode, the host does not keep a second CAS mirror. Uploads, downloads, ActionCache requests, and Execute requests are forwarded to the guest. The @@ -52,6 +55,14 @@ the matching Linux kernel, initramfs, and runtime SquashFS. At startup it materializes those bytes under `--root`, wraps the runtime and CAS as fixed VHD files, and starts the VM with Host Compute System. +`linux-actiond` is released for x86_64. Zig `@embedFile` includes the Linux +kernel, initramfs, runtime SquashFS, `qemu-system-x86_64`, `bios-256k.bin`, +and `linuxboot_dma.bin` selected by the `rules_qemu` target toolchain. +`linux-actiond` materializes the two firmware files under `--root`, writes +`qemu-system-x86_64` to a sealed memfd, and executes it with `execveat`; +`qemu-system-x86_64` is not extracted to disk. The current implementation uses +TCG. KVM and `io_uring` remain follow-up work. + `linux-actiond-guest` lives in the initramfs. It runs as guest init, mounts the minimal guest filesystems, mounts `/cas` and `/runtimes`, then execs itself as the guest REAPI worker. @@ -65,7 +76,7 @@ The VM is intentionally small: - writable virtio or Hyper-V synthetic SCSI block device for `/cas` - read-only virtio or Hyper-V synthetic SCSI block device for `/runtimes` - virtio-vsock or `AF_HYPERV` for gRPC -- serial stderr for logs on macOS +- serial host logs on macOS and Linux - no guest network device, SSH, systemd, package manager, graphics, or login The VM is long-lived. Each action gets its own Linux process sandbox inside the diff --git a/MODULE.bazel b/MODULE.bazel index 90bd905..e5e4086 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,6 +15,12 @@ bazel_dep(name = "squashfs-tools", version = "4.7.5") bazel_dep(name = "zstd", version = "1.5.7.bcr.1") bazel_dep(name = "codesign.bzl", version = "0.0.13") bazel_dep(name = "gawk", version = "5.3.2.bcr.3") +bazel_dep(name = "rules_qemu", version = "0.0.0") +git_override( + module_name = "rules_qemu", + commit = "cc3915b919c2e06e2ac8d07f1205a790ddc7423a", + remote = "https://github.com/hermeticbuild/rules_qemu.git", +) http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") @@ -56,6 +62,15 @@ register_toolchains("@zig_toolchains//:all") register_toolchains("@llvm//toolchain:all") register_toolchains("@codesign.bzl//toolchain:all") +qemu = use_extension("@rules_qemu//qemu/extension:qemu.bzl", "qemu") +qemu.system_toolchain( + system_target = "x86_64-softmmu", + target_settings = ["//platforms:qemu_system_x86_64"], +) +use_repo(qemu, "qemu_system_toolchains", "qemu_user_toolchains") + +register_toolchains("@qemu_system_toolchains//:all") + osx = use_extension("@llvm//extensions:osx.bzl", "osx") osx.frameworks(names = [ "CoreFoundation", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index d6e6353..2b4929c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -94,6 +94,8 @@ "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", + "https://bcr.bazel.build/modules/hermetic_launcher/0.0.5/MODULE.bazel": "0e00b51788823b75b4273aedbc6ba21f64dad453f7567f9359a2e96eb6ec101c", + "https://bcr.bazel.build/modules/hermetic_launcher/0.0.5/source.json": "eb5cf0f29fb36c10c2fde8f660cf5edfd377d8ea167f277fff9a811fd0a26028", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", @@ -111,7 +113,8 @@ "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", - "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", + "https://bcr.bazel.build/modules/package_metadata/0.0.6/MODULE.bazel": "341dab6f417197494517d54c8e557c0baee1de7aec83543a4fbefe57900acb7e", + "https://bcr.bazel.build/modules/package_metadata/0.0.6/source.json": "9581d8b22db43550ac75ecc314ee4fa0a33400bfdc77d1317d8af6b18dca7756", "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", diff --git a/README.md b/README.md index 42551f4..a157fb6 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # actiond -`actiond` is a local Remote Execution API worker and cache for Bazel. On macOS -and Windows it starts a small Linux VM and runs Bazel actions inside that VM, -so the host can act like a local Linux remote-execution worker. +`actiond` is a local Remote Execution API worker and cache for Bazel. On macOS, +Windows, and Linux it starts a small Linux VM and runs Bazel actions inside +that VM, so the host can act like a local Linux remote-execution worker. -The macOS ARM64 and Windows ARM64/x86_64 releases include the matching VM -kernel, initramfs, and Linux runtime image. +The macOS ARM64, Windows ARM64/x86_64, and Linux x86_64 releases include the +matching VM kernel, initramfs, and Linux runtime image. ## Why Use It? @@ -61,6 +61,19 @@ Windows requires Hyper-V. `windows-actiond` wraps the runtime SquashFS and guest-owned ext4 CAS in fixed VHD files. The default VHD paths are under `--root`; `--cas-image` can select another CAS VHD path. +The Linux x86_64 release embeds `qemu-system-x86_64`, `bios-256k.bin`, and +`linuxboot_dma.bin` from `rules_qemu`. `linux-actiond` materializes the two +firmware files under `--root` and executes QEMU from a sealed memfd. QEMU is +not extracted to disk. The current implementation uses TCG and requires +`/dev/vhost-vsock`: + +```bash +sudo modprobe vhost_vsock +./linux-actiond_linux_x86_64 serve-vm \ + --listen=127.0.0.1:8980 \ + --root="$HOME/.cache/actiond/vm" +``` + ## Point Bazel At actiond Add a config like this to your workspace `.bazelrc`: @@ -121,6 +134,7 @@ Most users should use releases. Source builds are mainly for development: ```bash bazel build --config=remote -c opt //cmd/darwin-actiond +bazel build --config=remote -c opt //cmd/linux-actiond:linux-actiond_linux_x86_64 ``` Normal contributor checks: @@ -130,7 +144,7 @@ bazel build --config=remote //... bazel test --config=remote //... ``` -The macOS VM e2e harness is: +The macOS and Linux VM e2e harness is: ```bash tools/e2e.sh vm diff --git a/cmd/linux-actiond/BUILD.bazel b/cmd/linux-actiond/BUILD.bazel new file mode 100644 index 0000000..34de09e --- /dev/null +++ b/cmd/linux-actiond/BUILD.bazel @@ -0,0 +1,43 @@ +load("@bazel_lib//lib:transitions.bzl", "platform_transition_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load("//tools:zig_embedded_assets.bzl", "zig_embedded_assets") +load("//tools:zig_embedded_qemu.bzl", "zig_embedded_qemu") + +zig_binary( + name = "linux-actiond", + main = "main.zig", + strip_debug_symbols = True, + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + visibility = ["//visibility:public"], + deps = [ + ":embedded_assets", + ":embedded_qemu", + "//src:actiond", + ], +) + +zig_embedded_assets( + name = "embedded_assets", + initramfs = "//vm:initramfs_x86_64", + kernel = "//vm:linux_kernel.image", + runtime_image = "//runtimes:runtimes_squashfs_x86_64", +) + +zig_embedded_qemu( + name = "embedded_qemu", + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], +) + +platform_transition_binary( + name = "linux-actiond_linux_x86_64", + basename = "linux-actiond_linux_x86_64", + binary = ":linux-actiond", + target_platform = "//platforms:linux_x86_64_musl", + visibility = ["//visibility:public"], +) diff --git a/cmd/linux-actiond/main.zig b/cmd/linux-actiond/main.zig new file mode 100644 index 0000000..73e3fa5 --- /dev/null +++ b/cmd/linux-actiond/main.zig @@ -0,0 +1,40 @@ +const std = @import("std"); +const Io = std.Io; +const actiond = @import("actiond"); +const embedded_assets = @import("actiond_embedded_assets"); +const embedded_qemu = @import("actiond_embedded_qemu"); + +const fexec_qemu_argument = "--actiond-internal-fexec-qemu"; + +pub fn main(init: std.process.Init) !void { + const io = init.io; + const arena = init.arena.allocator(); + const args = try init.minimal.args.toSlice(arena); + + if (args.len > 1 and std.mem.eql(u8, args[1], fexec_qemu_argument)) { + if (args.len < 3) return error.MissingQemuArguments; + try actiond.qemu_vm.fexecEmbedded( + io, + arena, + init.minimal.environ, + embedded_qemu.qemu_system, + args[2..], + ); + } + + if (args.len > 1 and std.mem.eql(u8, args[1], "serve-vm")) { + const options = try actiond.vm_host.parseServeVmArgs(args[2..]); + return actiond.linux_vm_host.serve(io, std.heap.smp_allocator, options, embedded_assets, embedded_qemu); + } + + var stdout_buffer: [256]u8 = undefined; + var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer); + const stdout = &stdout_writer.interface; + try stdout.print( + \\linux-actiond zig={s} bazel={s} + \\usage: + \\ linux-actiond serve-vm [--kernel=/path/bzImage] [--initramfs=/path/initramfs.cpio[.zst]] [--runtime-image=/path/runtimes.sqfs] [--cas-image=/path/cas.ext4] [--listen=127.0.0.1:8980] [--root=/tmp/actiond-vm] [--actiondfs-stats-path=/path/stats.txt] + \\ + , .{ actiond.version.zig, actiond.version.bazel }); + try stdout.flush(); +} diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 0c79e04..9949106 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -21,3 +21,12 @@ platform( ], visibility = ["//visibility:public"], ) + +config_setting( + name = "qemu_system_x86_64", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + visibility = ["//visibility:public"], +) diff --git a/src/BUILD.bazel b/src/BUILD.bazel index 7008a5f..b1ba192 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -41,7 +41,9 @@ zig_library( "http2_frame.zig", "http2_header.zig", "http2_hpack.zig", + "linux_vm_host.zig", "protobuf_wire.zig", + "qemu_vm.zig", "reapi.zig", "reapi_dispatch.zig", "staged_cas_index.zig", @@ -89,7 +91,9 @@ zig_test( "http2_frame.zig", "http2_header.zig", "http2_hpack.zig", + "linux_vm_host.zig", "protobuf_wire.zig", + "qemu_vm.zig", "reapi.zig", "reapi_dispatch.zig", "staged_cas_index.zig", diff --git a/src/darwin_vm_host.zig b/src/darwin_vm_host.zig index 1fd91be..d255788 100644 --- a/src/darwin_vm_host.zig +++ b/src/darwin_vm_host.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const control_protocol = @import("control_protocol.zig"); const control_transport_fd = @import("control_transport_fd.zig"); const darwin_vm = @import("darwin_vm.zig"); const grpc_vsock_bridge = @import("grpc_vsock_bridge.zig"); @@ -20,7 +19,7 @@ pub fn serve( ""; defer if (options.cas_image == null) allocator.free(owned_cas_image_path); const cas_image_path = options.cas_image orelse owned_cas_image_path; - const format_cas_image = try ensureCasImageFile(io, cas_image_path, options.cas_image_size_mib); + const format_cas_image = try vm_host.ensureCasImageFile(io, cas_image_path, options.cas_image_size_mib); var assets = try vm_host.resolveAssets(io, allocator, root_dir, options, embedded_assets); defer assets.deinit(allocator); @@ -68,67 +67,7 @@ pub fn serve( if (options.actiondfs_stats_path) |path| { const stats_path = try allocator.dupe(u8, path); errdefer allocator.free(stats_path); - try background_tasks.concurrent(io, actiondfsStatsTask, .{ io, allocator, &fd_client, stats_path }); + try background_tasks.concurrent(io, vm_host.actiondfsStatsTask, .{ io, allocator, &fd_client, stats_path }); } return grpc_vsock_bridge.serve(io, options.listen, &vm); } - -fn actiondfsStatsTask( - io: std.Io, - allocator: std.mem.Allocator, - client: *control_transport_fd.Client, - path: []const u8, -) !void { - defer allocator.free(path); - while (true) { - writeActiondfsStatsSnapshot(io, allocator, client, path) catch |err| { - std.log.warn("actiondfs stats snapshot failed: {s}", .{@errorName(err)}); - }; - try io.sleep(.fromMilliseconds(1_000), .awake); - } -} - -fn writeActiondfsStatsSnapshot( - io: std.Io, - allocator: std.mem.Allocator, - client: *control_transport_fd.Client, - path: []const u8, -) !void { - var response = try client.call(io, allocator, .{ - .kind = .unary, - .method = control_protocol.actiondfs_stats_method, - .body = "", - }); - defer response.deinit(allocator); - if (response.status != .ok) return error.GuestApplicationError; - - try createParentDirs(io, path); - try std.Io.Dir.cwd().writeFile(io, .{ - .sub_path = path, - .data = response.body, - .flags = .{ .read = true, .permissions = .default_file }, - }); -} - -fn createParentDirs(io: std.Io, path: []const u8) !void { - const slash = std.mem.lastIndexOfScalar(u8, path, '/') orelse return; - if (slash == 0) return; - try std.Io.Dir.cwd().createDirPath(io, path[0..slash]); -} - -fn ensureCasImageFile(io: std.Io, path: []const u8, size_mib: u64) !bool { - if (std.Io.Dir.cwd().statFile(io, path, .{})) |stat| { - if (stat.kind != .file) return error.InvalidCasImage; - return false; - } else |err| switch (err) { - error.FileNotFound => {}, - else => return err, - } - - try createParentDirs(io, path); - var file = try std.Io.Dir.cwd().createFile(io, path, .{}); - defer file.close(io); - const size_bytes = try std.math.mul(u64, size_mib, 1024 * 1024); - try file.setLength(io, size_bytes); - return true; -} diff --git a/src/grpc_vsock_bridge.zig b/src/grpc_vsock_bridge.zig index 2c1fa6e..2a8e983 100644 --- a/src/grpc_vsock_bridge.zig +++ b/src/grpc_vsock_bridge.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const darwin_vm = @import("darwin_vm.zig"); const vsock = @import("vsock.zig"); pub const Error = error{ @@ -22,9 +21,9 @@ const PumpStats = struct { pub fn serve( io: std.Io, listen: []const u8, - machine: *darwin_vm.Machine, + machine: anytype, ) !void { - if (comptime @import("builtin").os.tag != .macos) return error.UnsupportedHost; + if (comptime @import("builtin").os.tag != .macos and @import("builtin").os.tag != .linux) return error.UnsupportedHost; const address = try parseListenAddress(listen); var listener = try address.listen(io, .{ .reuse_address = true }); @@ -32,6 +31,11 @@ pub fn serve( var connections: std.Io.Group = .init; defer connections.cancel(io); + const ConnectionTask = struct { + fn run(task_io: std.Io, task_machine: @TypeOf(machine), client_fd: std.posix.fd_t) void { + connectionTask(task_io, task_machine, client_fd); + } + }; std.log.info("actiond VM raw gRPC bridge listening on {s} -> vsock:{d}", .{ listen, vsock.grpc_port }); while (true) { @@ -45,7 +49,7 @@ pub fn serve( setTcpNoDelay(client_fd) catch |err| { std.log.debug("raw gRPC bridge TCP_NODELAY failed: {s}", .{@errorName(err)}); }; - connections.concurrent(io, connectionTask, .{ io, machine, client_fd }) catch |err| { + connections.concurrent(io, ConnectionTask.run, .{ io, machine, client_fd }) catch |err| { accepted.close(io); std.log.err("raw gRPC bridge connection task failed: {s}", .{@errorName(err)}); sleepMilliseconds(io, 10); @@ -54,7 +58,7 @@ pub fn serve( } } -fn connectionTask(io: std.Io, machine: *darwin_vm.Machine, client_fd: std.posix.fd_t) void { +fn connectionTask(io: std.Io, machine: anytype, client_fd: std.posix.fd_t) void { defer closeFd(client_fd); const started = std.Io.Clock.awake.now(io); diff --git a/src/linux_vm_host.zig b/src/linux_vm_host.zig new file mode 100644 index 0000000..8fd5adc --- /dev/null +++ b/src/linux_vm_host.zig @@ -0,0 +1,91 @@ +const builtin = @import("builtin"); +const std = @import("std"); +const control_transport_fd = @import("control_transport_fd.zig"); +const grpc_vsock_bridge = @import("grpc_vsock_bridge.zig"); +const qemu_vm = @import("qemu_vm.zig"); +const vm_host = @import("vm_host.zig"); + +pub fn serve( + io: std.Io, + allocator: std.mem.Allocator, + options: vm_host.ServeVmOptions, + comptime embedded_assets: type, + comptime embedded_qemu: type, +) !void { + if (comptime builtin.os.tag != .linux or builtin.cpu.arch != .x86_64) { + return error.UnsupportedHost; + } + + var root_dir = try std.Io.Dir.cwd().createDirPathOpen(io, options.root, .{}); + defer root_dir.close(io); + + const owned_cas_image_path = if (options.cas_image == null) + try std.fs.path.join(allocator, &.{ options.root, "cas.ext4" }) + else + ""; + defer if (options.cas_image == null) allocator.free(owned_cas_image_path); + const cas_image_path = options.cas_image orelse owned_cas_image_path; + const format_cas_image = try vm_host.ensureCasImageFile(io, cas_image_path, options.cas_image_size_mib); + + var assets = try vm_host.resolveAssets(io, allocator, root_dir, options, embedded_assets); + defer assets.deinit(allocator); + + const raw_kernel = try vm_host.prepareBootKernel(io, allocator, root_dir, assets.kernel); + defer if (raw_kernel) |path| allocator.free(path); + const boot_kernel_path = raw_kernel orelse assets.kernel; + + const raw_initramfs = try vm_host.prepareBootInitramfs(io, allocator, root_dir, assets.initramfs); + defer if (raw_initramfs) |path| allocator.free(path); + const boot_initramfs_path = raw_initramfs orelse assets.initramfs; + + const bios_256k_path = try vm_host.materializeEmbeddedFile( + io, + allocator, + root_dir, + "qemu/bios-256k.bin", + embedded_qemu.bios_256k, + ); + defer allocator.free(bios_256k_path); + const linuxboot_dma_path = try vm_host.materializeEmbeddedFile( + io, + allocator, + root_dir, + "qemu/linuxboot_dma.bin", + embedded_qemu.linuxboot_dma, + ); + defer allocator.free(linuxboot_dma_path); + const qemu_data_path = try vm_host.absoluteSubPath(io, allocator, root_dir, "qemu"); + defer allocator.free(qemu_data_path); + + std.log.info("starting QEMU VM kernel={s} initramfs={s} runtimes={s} cas={s}", .{ + boot_kernel_path, + boot_initramfs_path, + assets.runtime_image, + cas_image_path, + }); + var machine = try qemu_vm.Machine.start(io, allocator, .{ + .kernel_path = boot_kernel_path, + .initramfs_path = boot_initramfs_path, + .runtime_image_path = assets.runtime_image, + .cas_image_path = cas_image_path, + .qemu_data_path = qemu_data_path, + .format_cas_image = format_cas_image, + .memory_mib = options.memory_mib, + .cpu_count = options.cpus, + .start_timeout_ms = options.start_timeout_ms, + .connect_timeout_ms = options.connect_timeout_ms, + }); + defer machine.deinit(); + + std.log.info("actiond QEMU VM started; proxying gRPC to linux-actiond-guest", .{}); + var fd_client = control_transport_fd.Client{ .opener = machine.opener() }; + defer fd_client.deinit(io); + var background_tasks: std.Io.Group = .init; + defer background_tasks.cancel(io); + if (options.actiondfs_stats_path) |path| { + const stats_path = try allocator.dupe(u8, path); + errdefer allocator.free(stats_path); + try background_tasks.concurrent(io, vm_host.actiondfsStatsTask, .{ io, allocator, &fd_client, stats_path }); + } + return grpc_vsock_bridge.serve(io, options.listen, &machine); +} diff --git a/src/qemu_vm.zig b/src/qemu_vm.zig new file mode 100644 index 0000000..974e7ee --- /dev/null +++ b/src/qemu_vm.zig @@ -0,0 +1,332 @@ +const builtin = @import("builtin"); +const std = @import("std"); +const control_transport_fd = @import("control_transport_fd.zig"); +const vsock = @import("vsock.zig"); + +const linux = std.os.linux; +const fexec_qemu_argument = "--actiond-internal-fexec-qemu"; +const mfd_exec = 0x0010; + +pub const Error = error{ + ConnectFailed, + ConnectTimedOut, + FexecFailed, + StartFailed, + UnsupportedHost, +}; + +pub const Options = struct { + kernel_path: []const u8, + initramfs_path: []const u8, + runtime_image_path: []const u8, + cas_image_path: []const u8, + qemu_data_path: []const u8, + format_cas_image: bool = false, + memory_mib: u64 = 512, + cpu_count: u32 = 2, + start_timeout_ms: u32 = 30_000, + connect_timeout_ms: u32 = 60_000, + connect_attempt_timeout_ms: u32 = 1_000, + guest_cid: ?u32 = null, +}; + +pub const Machine = struct { + child: std.process.Child, + io: std.Io, + guest_cid: u32, + connect_timeout_ms: u32, + connect_attempt_timeout_ms: u32, + + pub fn start(io: std.Io, allocator: std.mem.Allocator, options: Options) !Machine { + if (comptime builtin.os.tag != .linux or builtin.cpu.arch != .x86_64) { + return error.UnsupportedHost; + } + + const guest_cid = options.guest_cid orelse try randomGuestCid(io); + const memory = try std.fmt.allocPrint(allocator, "{d}M", .{options.memory_mib}); + defer allocator.free(memory); + const cpus = try std.fmt.allocPrint(allocator, "{d}", .{options.cpu_count}); + defer allocator.free(cpus); + const vsock_device = try std.fmt.allocPrint(allocator, "vhost-vsock-pci,id=vsock0,guest-cid={d}", .{guest_cid}); + defer allocator.free(vsock_device); + const cas_drive = try driveArg(allocator, "cas", options.cas_image_path, false); + defer allocator.free(cas_drive); + const runtime_drive = try driveArg(allocator, "runtimes", options.runtime_image_path, true); + defer allocator.free(runtime_drive); + const kernel_append = try kernelAppendArg(allocator, options.format_cas_image); + defer allocator.free(kernel_append); + + // TODO: Use KVM after the rules_qemu QEMU prebuilt is verified with + // /dev/kvm on the Linux runner. + const argv = [_][]const u8{ + "/proc/self/exe", + fexec_qemu_argument, + "qemu-system-x86_64", + "-machine", + "q35,accel=tcg", + "-cpu", + "max", + "-L", + options.qemu_data_path, + "-smp", + cpus, + "-m", + memory, + "-no-user-config", + "-nodefaults", + "-display", + "none", + "-monitor", + "none", + "-no-reboot", + "-serial", + "stdio", + "-kernel", + options.kernel_path, + "-initrd", + options.initramfs_path, + "-append", + kernel_append, + "-device", + vsock_device, + "-drive", + cas_drive, + "-device", + "virtio-blk-pci,drive=cas", + "-drive", + runtime_drive, + "-device", + "virtio-blk-pci,drive=runtimes", + }; + + std.log.info("starting embedded qemu-system-x86_64 guest_cid={d}", .{guest_cid}); + var child = try std.process.spawn(io, .{ + .argv = &argv, + .stdin = .ignore, + .stdout = .inherit, + .stderr = .inherit, + }); + errdefer child.kill(io); + + var machine: Machine = .{ + .child = child, + .io = io, + .guest_cid = guest_cid, + .connect_timeout_ms = options.connect_timeout_ms, + .connect_attempt_timeout_ms = options.connect_attempt_timeout_ms, + }; + machine.waitForControlPort(options.start_timeout_ms) catch |err| { + machine.deinit(); + return err; + }; + return machine; + } + + pub fn deinit(self: *Machine) void { + self.child.kill(self.io); + self.* = undefined; + } + + pub fn opener(self: *Machine) control_transport_fd.Opener { + return .{ .ctx = self, .open = open }; + } + + fn open(ctx: *anyopaque) !std.posix.fd_t { + const self: *Machine = @ptrCast(@alignCast(ctx)); + return self.connectControlPort(vsock.control_port); + } + + pub fn connectControlPort(self: *Machine, port: u32) !std.posix.fd_t { + if (comptime builtin.os.tag != .linux) return error.UnsupportedHost; + + var remaining_ms = if (self.connect_timeout_ms == 0) + self.connect_attempt_timeout_ms + else + self.connect_timeout_ms; + while (true) { + if (connectVsock(self.guest_cid, port)) |fd| return fd else |err| { + if (try self.reapExitedChild()) |status| { + std.log.err("QEMU exited before guest vsock became ready status=0x{x}", .{status}); + return error.StartFailed; + } + if (remaining_ms <= self.connect_attempt_timeout_ms) { + std.log.err("timed out connecting to guest cid={d} vsock:{d}: {s}", .{ self.guest_cid, port, @errorName(err) }); + return error.ConnectTimedOut; + } + } + const sleep_ms = @min(@as(u32, 100), remaining_ms); + try self.io.sleep(.fromMilliseconds(sleep_ms), .awake); + remaining_ms -= sleep_ms; + } + } + + fn waitForControlPort(self: *Machine, timeout_ms: u32) !void { + const saved_timeout = self.connect_timeout_ms; + self.connect_timeout_ms = timeout_ms; + defer self.connect_timeout_ms = saved_timeout; + + const fd = try self.connectControlPort(vsock.control_port); + closeFd(fd); + } + + fn reapExitedChild(self: *Machine) !?u32 { + const pid = self.child.id orelse return 0; + while (true) { + var status: u32 = undefined; + const result = linux.waitpid(pid, &status, linux.W.NOHANG); + switch (linux.errno(result)) { + .SUCCESS => { + if (result == 0) return null; + self.child.id = null; + return status; + }, + .INTR => continue, + .CHILD => { + self.child.id = null; + return 0; + }, + else => return error.StartFailed, + } + } + } +}; + +pub fn fexecEmbedded( + io: std.Io, + allocator: std.mem.Allocator, + environ: std.process.Environ, + executable: []const u8, + args: []const [:0]const u8, +) !noreturn { + if (comptime builtin.os.tag != .linux) return error.UnsupportedHost; + + const parent_pid = linux.getppid(); + const parent_death_result = linux.prctl( + @intFromEnum(linux.PR.SET_PDEATHSIG), + @intFromEnum(linux.SIG.KILL), + 0, + 0, + 0, + ); + if (linux.errno(parent_death_result) != .SUCCESS) return error.FexecFailed; + if (linux.getppid() != parent_pid) return error.ParentExited; + + const memfd_flags = linux.MFD.CLOEXEC | linux.MFD.ALLOW_SEALING; + const fd = std.posix.memfd_create("actiond-qemu", memfd_flags | mfd_exec) catch |err| switch (err) { + // Linux added MFD_EXEC in 6.3. The fixed name is shorter than NAME_MAX, + // so NameTooLong here means the kernel rejected MFD_EXEC. + error.NameTooLong => try std.posix.memfd_create("actiond-qemu", memfd_flags), + else => return err, + }; + const file: std.Io.File = .{ .handle = fd, .flags = .{ .nonblocking = false } }; + defer file.close(io); + try file.writePositionalAll(io, executable, 0); + try file.setPermissions(io, .executable_file); + + const seals = linux.F.SEAL_SEAL | linux.F.SEAL_SHRINK | linux.F.SEAL_GROW | linux.F.SEAL_WRITE; + const seal_result = linux.fcntl(fd, linux.F.ADD_SEALS, seals); + if (linux.errno(seal_result) != .SUCCESS) return error.FexecFailed; + + const argv = try allocator.allocSentinel(?[*:0]const u8, args.len, null); + for (args, argv) |arg, *entry| entry.* = arg.ptr; + + const environment = environ.block.view().slice; + const envp = try allocator.allocSentinel(?[*:0]const u8, environment.len, null); + for (environment, envp) |entry, *output| output.* = entry; + + const result = linux.execveat(fd, "", argv.ptr, envp.ptr, .{ .SYMLINK_NOFOLLOW = false, .EMPTY_PATH = true }); + std.log.err("execveat for embedded qemu-system-x86_64 failed: {s}", .{@tagName(linux.errno(result))}); + return error.FexecFailed; +} + +fn driveArg(allocator: std.mem.Allocator, id: []const u8, path: []const u8, readonly: bool) ![]u8 { + const readonly_arg = if (readonly) ",readonly=on" else ""; + const escaped_path = try escapeDriveValue(allocator, path); + defer allocator.free(escaped_path); + // TODO: Add aio=io_uring after the rules_qemu QEMU prebuilt and Linux + // runner are verified with io_uring. + return std.fmt.allocPrint(allocator, "if=none,id={s},file={s},format=raw{s},cache=none", .{ + id, + escaped_path, + readonly_arg, + }); +} + +fn escapeDriveValue(allocator: std.mem.Allocator, value: []const u8) ![]u8 { + const escaped = try allocator.alloc(u8, value.len + std.mem.count(u8, value, ",")); + var output_index: usize = 0; + for (value) |byte| { + escaped[output_index] = byte; + output_index += 1; + if (byte == ',') { + escaped[output_index] = ','; + output_index += 1; + } + } + return escaped; +} + +fn randomGuestCid(io: std.Io) !u32 { + var random: u32 = undefined; + try io.randomSecure(std.mem.asBytes(&random)); + return 3 + random % (std.math.maxInt(u32) - 3); +} + +fn kernelAppendArg(allocator: std.mem.Allocator, format_cas_image: bool) ![]u8 { + return allocator.dupe(u8, if (format_cas_image) + "init=/init console=ttyS0 panic=-1 actiond.cas_device=/dev/vda actiond.format_cas=1" + else + "init=/init console=ttyS0 panic=-1 actiond.cas_device=/dev/vda"); +} + +fn connectVsock(cid: u32, port: u32) !std.posix.fd_t { + const socket_result = linux.socket(linux.AF.VSOCK, linux.SOCK.STREAM | linux.SOCK.CLOEXEC, 0); + if (linux.errno(socket_result) != .SUCCESS) return error.ConnectFailed; + const fd: std.posix.fd_t = @intCast(socket_result); + errdefer closeFd(fd); + + var address: linux.sockaddr.vm = .{ + .family = linux.AF.VSOCK, + .reserved1 = 0, + .port = port, + .cid = cid, + .flags = 0, + .zero = [_]u8{0} ** 3, + }; + const connect_result = linux.connect( + fd, + @ptrCast(&address), + @sizeOf(linux.sockaddr.vm), + ); + if (linux.errno(connect_result) != .SUCCESS) return error.ConnectFailed; + return fd; +} + +fn closeFd(fd: std.posix.fd_t) void { + while (true) switch (linux.errno(linux.close(fd))) { + .SUCCESS => return, + .INTR => continue, + else => return, + }; +} + +test "driveArg leaves io_uring disabled" { + const drive = try driveArg(std.testing.allocator, "cas", "/tmp/cas.ext4", false); + defer std.testing.allocator.free(drive); + try std.testing.expectEqualStrings("if=none,id=cas,file=/tmp/cas.ext4,format=raw,cache=none", drive); +} + +test "driveArg escapes commas in paths" { + const drive = try driveArg(std.testing.allocator, "cas", "/tmp/actiond,vm/cas.ext4", false); + defer std.testing.allocator.free(drive); + try std.testing.expectEqualStrings("if=none,id=cas,file=/tmp/actiond,,vm/cas.ext4,format=raw,cache=none", drive); +} + +test "kernelAppendArg identifies the CAS block device" { + const argument = try kernelAppendArg(std.testing.allocator, true); + defer std.testing.allocator.free(argument); + try std.testing.expectEqualStrings( + "init=/init console=ttyS0 panic=-1 actiond.cas_device=/dev/vda actiond.format_cas=1", + argument, + ); +} diff --git a/src/root.zig b/src/root.zig index ada4625..c88baeb 100644 --- a/src/root.zig +++ b/src/root.zig @@ -21,7 +21,9 @@ pub const grpc_vsock_bridge = @import("grpc_vsock_bridge.zig"); pub const http2_frame = @import("http2_frame.zig"); pub const http2_header = @import("http2_header.zig"); pub const http2_hpack = @import("http2_hpack.zig"); +pub const linux_vm_host = @import("linux_vm_host.zig"); pub const protobuf_wire = @import("protobuf_wire.zig"); +pub const qemu_vm = @import("qemu_vm.zig"); pub const reapi = @import("reapi.zig"); pub const reapi_dispatch = @import("reapi_dispatch.zig"); pub const staged_cas_index = @import("staged_cas_index.zig"); @@ -55,7 +57,9 @@ test { _ = http2_frame; _ = http2_header; _ = http2_hpack; + _ = linux_vm_host; _ = protobuf_wire; + _ = qemu_vm; _ = reapi; _ = reapi_dispatch; _ = staged_cas_index; diff --git a/src/vm_host.zig b/src/vm_host.zig index 80a8861..4902abb 100644 --- a/src/vm_host.zig +++ b/src/vm_host.zig @@ -1,5 +1,7 @@ const builtin = @import("builtin"); const std = @import("std"); +const control_protocol = @import("control_protocol.zig"); +const control_transport_fd = @import("control_transport_fd.zig"); const zstd_test = if (builtin.is_test) @import("c") else struct {}; const max_compressed_initramfs_bytes = 128 * 1024 * 1024; @@ -63,13 +65,27 @@ fn materializeEmbeddedAsset( name: []const u8, bytes: []const u8, ) ![]u8 { - try root_dir.createDirPath(io, "embedded"); - var hash: [32]u8 = undefined; std.crypto.hash.sha2.Sha256.hash(bytes, &hash, .{}); const output_rel = try std.fmt.allocPrint(allocator, "embedded/{s}-{s}", .{ name, std.fmt.bytesToHex(hash, .lower) }); defer allocator.free(output_rel); + return materializeEmbeddedFile(io, allocator, root_dir, output_rel, bytes); +} + +pub fn materializeEmbeddedFile( + io: std.Io, + allocator: std.mem.Allocator, + root_dir: std.Io.Dir, + output_rel: []const u8, + bytes: []const u8, +) ![]u8 { + const parent = std.fs.path.dirname(output_rel) orelse "."; + if (!std.mem.eql(u8, parent, ".")) try root_dir.createDirPath(io, parent); + + var hash: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(bytes, &hash, .{}); + if (root_dir.statFile(io, output_rel, .{})) |stat| { if (stat.kind == .file and stat.size == bytes.len and try embeddedAssetMatches(io, root_dir, output_rel, &hash)) { return absoluteSubPath(io, allocator, root_dir, output_rel); @@ -86,17 +102,23 @@ fn materializeEmbeddedAsset( return absoluteSubPath(io, allocator, root_dir, output_rel); } - var temp_path_buffer: [64]u8 = undefined; var output: std.Io.File = undefined; const temp_path = while (true) { const id = next_embedded_asset_temp_id.fetchAdd(1, .monotonic); - const path = try std.fmt.bufPrint(&temp_path_buffer, "embedded/.asset-{d}", .{id}); + const path = try std.fmt.allocPrint(allocator, "{s}/.asset-{d}", .{ parent, id }); output = root_dir.createFile(io, path, .{ .exclusive = true }) catch |err| switch (err) { - error.PathAlreadyExists => continue, - else => |e| return e, + error.PathAlreadyExists => { + allocator.free(path); + continue; + }, + else => |e| { + allocator.free(path); + return e; + }, }; break path; }; + defer allocator.free(temp_path); errdefer root_dir.deleteFile(io, temp_path) catch {}; var output_open = true; defer if (output_open) output.close(io); @@ -357,6 +379,66 @@ pub fn absoluteSubPath( return std.fs.path.join(allocator, &.{ root_buffer[0..root_len], sub_path }); } +pub fn ensureCasImageFile(io: std.Io, path: []const u8, size_mib: u64) !bool { + if (std.Io.Dir.cwd().statFile(io, path, .{})) |stat| { + if (stat.kind != .file) return error.InvalidCasImage; + return false; + } else |err| switch (err) { + error.FileNotFound => {}, + else => return err, + } + + try createParentDirs(io, path); + var file = try std.Io.Dir.cwd().createFile(io, path, .{}); + defer file.close(io); + const size_bytes = try std.math.mul(u64, size_mib, 1024 * 1024); + try file.setLength(io, size_bytes); + return true; +} + +pub fn actiondfsStatsTask( + io: std.Io, + allocator: std.mem.Allocator, + client: *control_transport_fd.Client, + path: []const u8, +) !void { + defer allocator.free(path); + while (true) { + writeActiondfsStatsSnapshot(io, allocator, client, path) catch |err| { + std.log.warn("actiondfs stats snapshot failed: {s}", .{@errorName(err)}); + }; + try io.sleep(.fromMilliseconds(1_000), .awake); + } +} + +fn writeActiondfsStatsSnapshot( + io: std.Io, + allocator: std.mem.Allocator, + client: *control_transport_fd.Client, + path: []const u8, +) !void { + var response = try client.call(io, allocator, .{ + .kind = .unary, + .method = control_protocol.actiondfs_stats_method, + .body = "", + }); + defer response.deinit(allocator); + if (response.status != .ok) return error.GuestApplicationError; + + try createParentDirs(io, path); + try std.Io.Dir.cwd().writeFile(io, .{ + .sub_path = path, + .data = response.body, + .flags = .{ .read = true, .permissions = .default_file }, + }); +} + +fn createParentDirs(io: std.Io, path: []const u8) !void { + const slash = std.mem.lastIndexOfScalar(u8, path, '/') orelse return; + if (slash == 0) return; + try std.Io.Dir.cwd().createDirPath(io, path[0..slash]); +} + test "parseServeVmArgs accepts VM flags" { const options = try parseServeVmArgs(&.{ "--listen=127.0.0.1:9999", diff --git a/tools/e2e.sh b/tools/e2e.sh index af9eecc..df0f558 100755 --- a/tools/e2e.sh +++ b/tools/e2e.sh @@ -10,7 +10,7 @@ usage: tools/e2e.sh Modes: build Run repository build/test checks and build the stress action tools. - vm Start darwin-actiond serve-vm and run test/ via Bazel remote execution. + vm Start the host actiond VM and run test/ via Bazel remote execution. all Run build plus the host-appropriate e2e mode when configured. Environment: @@ -118,6 +118,10 @@ wait_for_port() { if (echo >"/dev/tcp/${host}/${port}") >/dev/null 2>&1; then return 0 fi + if [[ -n "${e2e_server_pid}" ]] && ! kill -0 "${e2e_server_pid}" >/dev/null 2>&1; then + echo "${e2e_log_label} exited before ${host}:${port} became ready" >&2 + return 1 + fi if (( "$(date +%s)" - start >= timeout )); then echo "timed out waiting for ${host}:${port}" >&2 return 1 @@ -164,18 +168,35 @@ run_build_checks() { } run_vm_e2e() { - if [[ "$(uname -s)" != "Darwin" ]]; then - echo "vm e2e must run on macOS with Virtualization.framework" >&2 - return 1 - fi + local architecture server_label server_name + case "$(uname -s)" in + Darwin) + architecture="aarch64" + server_label="//cmd/darwin-actiond" + server_name="darwin-actiond" + ;; + Linux) + if [[ "$(uname -m)" != "x86_64" ]]; then + echo "QEMU VM e2e currently requires Linux x86_64" >&2 + return 1 + fi + architecture="x86_64" + server_label="//cmd/linux-actiond:linux-actiond_linux_x86_64" + server_name="linux-actiond" + ;; + *) + echo "VM e2e requires macOS or Linux x86_64" >&2 + return 1 + ;; + esac - prepare_stress_workspace aarch64 + prepare_stress_workspace "${architecture}" local server root log local cas_image cas_image_size_mib local -a server_args root="$(mktemp -d "${TMPDIR:-/tmp}/actiond-vm-e2e.XXXXXX")" - log="${root}/darwin-actiond-vm.log" + log="${root}/${server_name}-vm.log" cas_image="${ACTIOND_VM_CAS_IMAGE:-${root}/server/cas.ext4}" cas_image_size_mib="${ACTIOND_VM_CAS_IMAGE_SIZE_MIB:-8192}" server_args=( @@ -187,8 +208,8 @@ run_vm_e2e() { --cpus="${ACTIOND_VM_CPUS:-4}" ) - run_bazel build //cmd/darwin-actiond - server="$(bazel_output //cmd/darwin-actiond)" + run_bazel build "${server_label}" + server="$(bazel_output "${server_label}")" if [[ -n "${ACTIOND_E2E_ACTIONDFS_STATS_PATH:-}" ]]; then server_args+=( --actiondfs-stats-path="${ACTIOND_E2E_ACTIONDFS_STATS_PATH}" @@ -199,10 +220,10 @@ run_vm_e2e() { e2e_server_pid="$!" e2e_root="${root}" e2e_log="${log}" - e2e_log_label="darwin-actiond VM log" + e2e_log_label="${server_name} VM log" trap 'cleanup_e2e_server $?' EXIT - wait_for_port "${e2e_host}" "${e2e_port}" 90 + wait_for_port "${e2e_host}" "${e2e_port}" 180 run_stress_workspace cleanup_e2e_server 0 trap - EXIT @@ -217,12 +238,7 @@ case "${1:-}" in ;; all) run_build_checks - if [[ "$(uname -s)" == "Darwin" ]]; then - run_vm_e2e - else - echo "tools/e2e.sh VM e2e requires macOS; use the PowerShell smoke on Windows" >&2 - exit 1 - fi + run_vm_e2e ;; *) usage diff --git a/tools/zig_embedded_qemu.bzl b/tools/zig_embedded_qemu.bzl new file mode 100644 index 0000000..04310c4 --- /dev/null +++ b/tools/zig_embedded_qemu.bzl @@ -0,0 +1,68 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +_QEMU_TOOLCHAIN_TYPE = "@rules_qemu//qemu:target_toolchain_type" + +def _zig_embedded_qemu_source_impl(ctx): + qemu = ctx.toolchains[_QEMU_TOOLCHAIN_TYPE] + qemu_system = ctx.actions.declare_file(ctx.label.name + ".qemu-system") + bios_256k = ctx.actions.declare_file(ctx.label.name + ".bios-256k.bin") + linuxboot_dma = ctx.actions.declare_file(ctx.label.name + ".linuxboot_dma.bin") + source = ctx.actions.declare_file(ctx.label.name + ".zig") + ctx.actions.symlink(output = qemu_system, target_file = qemu.qemu_system) + ctx.actions.run_shell( + arguments = [ + qemu.system_data_anchor.path, + bios_256k.path, + linuxboot_dma.path, + ], + command = """ +set -eu +cp "$1/bios-256k.bin" "$2" +cp "$1/linuxboot_dma.bin" "$3" +""", + inputs = qemu.system_data_files, + outputs = [bios_256k, linuxboot_dma], + ) + ctx.actions.write( + source, + """\ +pub const qemu_system = @embedFile("{qemu_system}"); +pub const bios_256k = @embedFile("{bios_256k}"); +pub const linuxboot_dma = @embedFile("{linuxboot_dma}"); +""".format( + bios_256k = bios_256k.basename, + linuxboot_dma = linuxboot_dma.basename, + qemu_system = qemu_system.basename, + ), + ) + return [ + DefaultInfo(files = depset([source])), + OutputGroupInfo(embedded = depset([qemu_system, bios_256k, linuxboot_dma])), + ] + +_zig_embedded_qemu_source = rule( + implementation = _zig_embedded_qemu_source_impl, + toolchains = [_QEMU_TOOLCHAIN_TYPE], +) + +def zig_embedded_qemu(name, target_compatible_with = [], visibility = None): + source_name = name + "_source" + qemu_name = name + "_file" + _zig_embedded_qemu_source( + name = source_name, + target_compatible_with = target_compatible_with, + ) + native.filegroup( + name = qemu_name, + srcs = [":" + source_name], + output_group = "embedded", + target_compatible_with = target_compatible_with, + ) + zig_library( + name = name, + extra_srcs = [":" + qemu_name], + import_name = "actiond_embedded_qemu", + main = ":" + source_name, + target_compatible_with = target_compatible_with, + visibility = visibility, + ) diff --git a/vm/linux_x86_64.config b/vm/linux_x86_64.config index 9f4d071..f697abd 100644 --- a/vm/linux_x86_64.config +++ b/vm/linux_x86_64.config @@ -1,6 +1,8 @@ CONFIG_64BIT=y CONFIG_ACTIONDFS_FS=y CONFIG_ACPI=y +CONFIG_HPET_TIMER=y +CONFIG_X86_PM_TIMER=y CONFIG_X86_64=y CONFIG_SMP=y CONFIG_NR_CPUS=64 @@ -28,6 +30,8 @@ CONFIG_FS_MBCACHE=y CONFIG_FUTEX=y CONFIG_INET=y CONFIG_JBD2=y +CONFIG_KERNEL_LZ4=y +CONFIG_KVM_GUEST=y CONFIG_MEMCG=y CONFIG_MISC_FILESYSTEMS=y CONFIG_MULTIUSER=y @@ -41,6 +45,8 @@ CONFIG_POSIX_TIMERS=y CONFIG_PROC_FS=y CONFIG_PRINTK=y CONFIG_SECCOMP=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SIGNALFD=y CONFIG_SYSFS=y CONFIG_SQUASHFS=y @@ -50,6 +56,11 @@ CONFIG_TIMERFD=y CONFIG_UNIX=y CONFIG_USER_NS=y CONFIG_UTS_NS=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_VIRTIO_MENU=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_VSOCKETS=y CONFIG_VSOCKETS=y CONFIG_TTY=y # CONFIG_BT is not set From db1db864750588eda2c08f8f06ead1a1cfb846a5 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 12 Jun 2026 00:38:48 -0400 Subject: [PATCH 03/10] guest: decode raw Linux syscall errors --- src/guest_init.zig | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/guest_init.zig b/src/guest_init.zig index 20f613f..b0d1d3e 100644 --- a/src/guest_init.zig +++ b/src/guest_init.zig @@ -168,7 +168,7 @@ fn isCandidateBlockDevice(name: []const u8) bool { fn tryMountRuntimeDevice(stderr: *std.Io.Writer, device: [:0]const u8) !bool { const linux = std.os.linux; const fd_rc = linux.open(device.ptr, .{ .CLOEXEC = true }, 0); - switch (std.posix.errno(fd_rc)) { + switch (linux.errno(fd_rc)) { .SUCCESS => _ = linux.close(@intCast(fd_rc)), .NOENT, .NXIO, .NODEV, .NOTBLK => return false, else => return false, @@ -181,7 +181,7 @@ fn tryMountRuntimeDevice(stderr: *std.Io.Writer, device: [:0]const u8) !bool { linux.MS.RDONLY | linux.MS.NOSUID | linux.MS.NODEV, 0, ); - switch (std.posix.errno(rc)) { + switch (linux.errno(rc)) { .SUCCESS => { stderr.print("mounted runtime image from {s}\n", .{device}) catch {}; stderr.flush() catch {}; @@ -203,7 +203,7 @@ fn tryMountRuntimeDevice(stderr: *std.Io.Writer, device: [:0]const u8) !bool { fn tryMountCasDevice(stderr: *std.Io.Writer, device: [:0]const u8) !CasMountAttempt { const linux = std.os.linux; const fd_rc = linux.open(device.ptr, .{ .CLOEXEC = true }, 0); - switch (std.posix.errno(fd_rc)) { + switch (linux.errno(fd_rc)) { .SUCCESS => _ = linux.close(@intCast(fd_rc)), .NOENT, .NXIO, .NODEV, .NOTBLK => return .unavailable, else => return .unavailable, @@ -217,7 +217,7 @@ fn tryMountCasDevice(stderr: *std.Io.Writer, device: [:0]const u8) !CasMountAtte cas_mount_flags, @intFromPtr(data.ptr), ); - switch (std.posix.errno(rc)) { + switch (linux.errno(rc)) { .SUCCESS => { stderr.print("mounted guest CAS from {s}\n", .{device}) catch {}; stderr.flush() catch {}; @@ -315,7 +315,7 @@ fn sleepRuntimeDevicePollInterval() void { .sec = 0, .nsec = runtime_device_wait_ns, }; - while (std.posix.errno(std.os.linux.nanosleep(&request, &request)) == .INTR) {} + while (std.os.linux.errno(std.os.linux.nanosleep(&request, &request)) == .INTR) {} } fn mount(stderr: *std.Io.Writer, mount_spec: Mount) !void { @@ -328,7 +328,7 @@ fn mount(stderr: *std.Io.Writer, mount_spec: Mount) !void { mount_spec.flags, if (mount_spec.data) |data| @intFromPtr(data.ptr) else 0, ); - switch (std.posix.errno(rc)) { + switch (std.os.linux.errno(rc)) { .SUCCESS => {}, else => |errno| { stderr.print("mount {s} on {s} type {s} failed: {s}\n", .{ From 35eff78d235243fcf827c436304fb91107637e8b Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 12 Jun 2026 07:24:32 -0400 Subject: [PATCH 04/10] ci: compare Linux QEMU LLVM smoke with host --- .github/workflows/ci.yml | 31 ++-- e2e/README.md | 18 ++ e2e/run_llvm_linux_vm_smoke.sh | 326 +++++++++++++++++++++++++++++++++ 3 files changed, 362 insertions(+), 13 deletions(-) create mode 100755 e2e/run_llvm_linux_vm_smoke.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0f35d2..70b7c59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,10 +24,10 @@ jobs: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} run: .github/workflows/build_release.sh release - linux-qemu-e2e: - name: Linux QEMU e2e + linux-qemu-llvm-smoke: + name: Linux QEMU LLVM smoke runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 240 steps: - name: Checkout @@ -41,17 +41,22 @@ jobs: test -c /dev/vhost-vsock sudo chmod a+rw /dev/vhost-vsock - - name: Run QEMU e2e + - name: Compare actiond and Linux host env: - ACTIOND_E2E_BARE_COUNT: 16 - ACTIOND_E2E_NESTED_FILES_PER_GROUP: 8 - ACTIOND_E2E_SOURCE_DIRS: 2 - ACTIOND_E2E_SOURCE_FILES_PER_DIR: 8 - ACTIOND_REPO_BAZEL_FLAGS: --config=remote --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_API_KEY }} - ACTIOND_VM_CAS_IMAGE_SIZE_MIB: 1024 - ACTIOND_VM_CPUS: 2 - ACTIOND_VM_MEMORY_MIB: 1024 - run: tools/e2e.sh vm + BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + ACTIOND_VM_CAS_IMAGE_SIZE_MIB: 8192 + ACTIOND_VM_MEMORY_MIB: 4096 + run: e2e/run_llvm_linux_vm_smoke.sh "$RUNNER_TEMP/actiond-linux-llvm" + + - name: Upload Linux LLVM smoke results + if: always() + uses: actions/upload-artifact@v4 + with: + name: linux-qemu-llvm-smoke + if-no-files-found: warn + path: | + ${{ runner.temp }}/actiond-linux-llvm/*.md + ${{ runner.temp }}/actiond-linux-llvm/*.log windows-llvm-smoke: name: Windows ${{ matrix.mode }} (${{ matrix.architecture }}) diff --git a/e2e/README.md b/e2e/README.md index d8c5d9c..d50ea9a 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -100,3 +100,21 @@ GitHub's `windows-11-arm` runner does not provide Hyper-V, so CI passes `-BuildOnly` there to build the standalone ARM64 `windows-actiond` executable in one Bazel command and run it. The x86_64 matrix entry runs the full Hyper-V LLVM comparison. + +## Linux QEMU LLVM Smoke Runner + +`run_llvm_linux_vm_smoke.sh` builds `@llvm-project//llvm:llvm-tblgen` once +through `linux-actiond` and once directly on the Linux x86_64 host. Both +measurements use fresh Bazel output bases, the Linux x86_64 musl target and +host platforms, and `//e2e:llvm_exec_warmup` before the measured build. The +runner stops QEMU before the Linux-host measurement, passes no `--jobs` value +by default, and requires equal total and executed process counts. It writes +`linux-llvm-smoke-timings.md` with Bazel elapsed time, wall time, process +counts, and the actiond-to-host ratio: + +```bash +e2e/run_llvm_linux_vm_smoke.sh +``` + +The actiond result includes QEMU TCG overhead and is an end-to-end comparison, +not an executor-only measurement. diff --git a/e2e/run_llvm_linux_vm_smoke.sh b/e2e/run_llvm_linux_vm_smoke.sh new file mode 100755 index 0000000..8458439 --- /dev/null +++ b/e2e/run_llvm_linux_vm_smoke.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +set -euo pipefail + +export LC_ALL=C + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then + echo "Linux LLVM VM smoke requires a Linux x86_64 host" >&2 + exit 1 +fi +if [[ "$#" -gt 1 ]]; then + echo "usage: e2e/run_llvm_linux_vm_smoke.sh [output-directory]" >&2 + exit 1 +fi +output_root="${1:-${ACTIOND_LLVM_LINUX_SMOKE_ROOT:-$(mktemp -d "${TMPDIR:-/tmp}/actiond-linux-llvm.XXXXXX")}}" +target="${ACTIOND_LLVM_SMOKE_TARGET:-@llvm-project//llvm:llvm-tblgen}" +warmup_target="${ACTIOND_LLVM_SMOKE_WARMUP_TARGET-//e2e:llvm_exec_warmup}" +llvm_platform="${ACTIOND_LLVM_SMOKE_PLATFORM:-@llvm//platforms:linux_x86_64_musl}" +execution_platform="${ACTIOND_LLVM_SMOKE_EXEC_PLATFORM:-//e2e:actiond_linux_x86_64_musl_exec}" +endpoint="${ACTIOND_LLVM_SMOKE_ENDPOINT:-127.0.0.1:8998}" +server_target="//cmd/linux-actiond:linux-actiond_linux_x86_64" +vm_cpus="${ACTIOND_VM_CPUS:-$(nproc)}" +vm_memory_mib="${ACTIOND_VM_MEMORY_MIB:-4096}" +cas_image_size_mib="${ACTIOND_VM_CAS_IMAGE_SIZE_MIB:-8192}" +jobs="${ACTIOND_LLVM_SMOKE_JOBS-}" +jobs_label="${jobs:-Bazel default}" +jobs_flags=() +if [[ -n "${jobs}" ]]; then + jobs_flags=(--jobs="${jobs}") +fi + +mkdir -p "${output_root}" +output_root="$(cd "${output_root}" && pwd)" +actiond_output_base="${output_root}/actiond-bazel-output-base" +host_output_base="${output_root}/linux-host-bazel-output-base" +server_root="${output_root}/vm" +server_log="${output_root}/linux-actiond.log" +summary_path="${output_root}/linux-llvm-smoke-timings.md" +rm -rf "${actiond_output_base}" "${host_output_base}" "${server_root}" + +server_pid="" + +stop_server() { + if [[ -n "${server_pid}" ]]; then + kill "${server_pid}" >/dev/null 2>&1 || true + wait "${server_pid}" >/dev/null 2>&1 || true + server_pid="" + fi +} + +cleanup() { + local status="${1:-$?}" + stop_server + if [[ "${status}" -ne 0 && -f "${server_log}" ]]; then + echo "----- linux-actiond log (${server_log}) -----" >&2 + tail -200 "${server_log}" >&2 || true + fi +} + +trap 'cleanup $?' EXIT + +wait_for_port() { + local host="${endpoint%:*}" + local port="${endpoint##*:}" + local start + start="$(date +%s)" + while true; do + if [[ -n "${server_pid}" ]] && ! kill -0 "${server_pid}" >/dev/null 2>&1; then + echo "linux-actiond exited before ${endpoint} became ready" >&2 + return 1 + fi + if (echo >"/dev/tcp/${host}/${port}") >/dev/null 2>&1; then + return 0 + fi + if (( "$(date +%s)" - start >= 180 )); then + echo "timed out waiting for ${endpoint}" >&2 + return 1 + fi + sleep 0.2 + done +} + +setup_server() { + local build_log="${output_root}/linux-actiond-build.log" + local cquery_log="${output_root}/linux-actiond-cquery.log" + local -a headers=() + if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then + headers+=( + --bes_header="x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}" + --remote_header="x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}" + ) + fi + + echo "Building ${server_target}" >&2 + if ! ( + cd "${repo_root}" + bazel build --config=remote --remote_timeout=900 "${headers[@]}" "${server_target}" + ) >"${build_log}" 2>&1; then + echo "linux-actiond build failed; log: ${build_log}" >&2 + tail -200 "${build_log}" >&2 || true + return 1 + fi + + local server + if ! server="$({ + cd "${repo_root}" + bazel cquery --config=remote --output=files --bes_backend= --noshow_progress "${headers[@]}" "${server_target}" + } 2>"${cquery_log}" | tail -n 1)"; then + echo "linux-actiond cquery failed; log: ${cquery_log}" >&2 + tail -200 "${cquery_log}" >&2 || true + return 1 + fi + if [[ "${server}" != /* ]]; then + server="${repo_root}/${server}" + fi + if [[ ! -x "${server}" ]]; then + echo "linux-actiond executable does not exist: ${server}" >&2 + return 1 + fi + + ( + cd "${repo_root}" + bazel shutdown + ) >>"${build_log}" 2>&1 + + "${server}" serve-vm \ + --listen="${endpoint}" \ + --root="${server_root}" \ + --cas-image-size-mib="${cas_image_size_mib}" \ + --memory-mib="${vm_memory_mib}" \ + --cpus="${vm_cpus}" \ + --connect-timeout-ms=900000 \ + >"${server_log}" 2>&1 & + server_pid="$!" + wait_for_port +} + +process_total() { + sed -E 's/^([0-9]+) processes:.*/\1/' <<<"$1" +} + +process_executed() { + local summary="$1" + local total="$2" + awk -v total="${total}" -F',' ' + { + skipped = 0 + for (i = 1; i <= NF; i++) { + part = $i + sub(/^[^:]*: /, "", part) + gsub(/^[[:space:]]+|[[:space:].]+$/, "", part) + if (part ~ /^[0-9]+ (action cache hit|disk cache hit|remote cache hit|internal)$/) { + split(part, fields, " ") + skipped += fields[1] + } + } + print total - skipped + } + ' <<<"${summary}" +} + +measurement_common_flags=( + -c opt + --strip=always + --stripopt=--strip-all + --color=no + --curses=no + --bes_backend= + --platforms="${llvm_platform}" + --host_platform="${llvm_platform}" + --extra_execution_platforms="${execution_platform}" + --shell_executable=/bin/bash + --experimental_remote_downloader= + --experimental_remote_downloader_local_fallback=true + --noremote_cache_compression + --remote_upload_local_results=false + --disk_cache= +) + +measure() { + local mode="$1" + local output_base="$2" + local warmup_log="${output_root}/llvm-tblgen-${mode}-warmup.log" + local build_log="${output_root}/llvm-tblgen-${mode}.log" + local shutdown_log="${output_root}/llvm-tblgen-${mode}-shutdown.log" + local -a mode_flags=() + local -a warmup_cache_flags=(--noremote_accept_cached) + local -a measured_cache_flags=() + + rm -rf "${output_base}" + if [[ "${mode}" == "actiond" ]]; then + mode_flags=( + --remote_executor="grpc://${endpoint}" + --remote_cache="grpc://${endpoint}" + --remote_local_fallback=false + --remote_download_outputs=toplevel + --remote_timeout=900 + --spawn_strategy=remote + --genrule_strategy=remote + ) + measured_cache_flags=(--remote_accept_cached) + else + mode_flags=( + --remote_executor= + --remote_cache= + --spawn_strategy=local + --genrule_strategy=local + ) + measured_cache_flags=(--noremote_accept_cached) + fi + + echo "Starting ${mode} LLVM warmup" >&2 + if ! ( + cd "${repo_root}" + bazel --output_base="${output_base}" build "${warmup_target}" \ + "${measurement_common_flags[@]}" \ + "${mode_flags[@]}" \ + "${warmup_cache_flags[@]}" \ + "${jobs_flags[@]}" + ) >"${warmup_log}" 2>&1; then + echo "${mode} LLVM warmup failed; log: ${warmup_log}" >&2 + tail -200 "${warmup_log}" >&2 || true + return 1 + fi + + local start_ns end_ns wall_elapsed bazel_elapsed process_summary total executed + start_ns="$(date +%s%N)" + echo "Starting ${mode} LLVM measurement" >&2 + if ! ( + cd "${repo_root}" + bazel --output_base="${output_base}" build "${target}" \ + "${measurement_common_flags[@]}" \ + "${mode_flags[@]}" \ + "${measured_cache_flags[@]}" \ + "${jobs_flags[@]}" + ) >"${build_log}" 2>&1; then + echo "${mode} LLVM measurement failed; log: ${build_log}" >&2 + tail -200 "${build_log}" >&2 || true + return 1 + fi + end_ns="$(date +%s%N)" + + ( + cd "${repo_root}" + bazel --output_base="${output_base}" shutdown + ) >"${shutdown_log}" 2>&1 + + wall_elapsed="$(awk -v start="${start_ns}" -v end="${end_ns}" 'BEGIN { printf "%.3f", (end - start) / 1000000000 }')" + bazel_elapsed="$(sed -n 's/.*Elapsed time: \([0-9.]*\)s.*/\1/p' "${build_log}" | tail -n 1)" + process_summary="$(sed -n 's/^INFO: \(.*processes:.*\)$/\1/p' "${build_log}" | tail -n 1)" + if [[ -z "${bazel_elapsed}" || -z "${process_summary}" ]]; then + echo "could not parse ${mode} LLVM result; log: ${build_log}" >&2 + return 1 + fi + total="$(process_total "${process_summary}")" + executed="$(process_executed "${process_summary}" "${total}")" + if [[ ! "${total}" =~ ^[0-9]+$ || ! "${executed}" =~ ^[0-9]+$ || "${executed}" -eq 0 ]]; then + echo "could not parse ${mode} process counts: ${process_summary}" >&2 + return 1 + fi + + if [[ "${mode}" == "actiond" ]]; then + actiond_bazel_elapsed="${bazel_elapsed}" + actiond_wall_elapsed="${wall_elapsed}" + actiond_process_summary="${process_summary}" + actiond_total="${total}" + actiond_executed="${executed}" + else + host_bazel_elapsed="${bazel_elapsed}" + host_wall_elapsed="${wall_elapsed}" + host_process_summary="${process_summary}" + host_total="${total}" + host_executed="${executed}" + fi + echo "${mode} LLVM measurement: Bazel ${bazel_elapsed}s, wall ${wall_elapsed}s, ${process_summary}" >&2 +} + +if [[ -z "${warmup_target}" ]]; then + echo "Linux LLVM VM smoke requires ACTIOND_LLVM_SMOKE_WARMUP_TARGET" >&2 + exit 1 +fi + +echo "Linux LLVM smoke output: ${output_root}" >&2 +setup_server +measure actiond "${actiond_output_base}" +stop_server +measure linux-host "${host_output_base}" + +if [[ "${actiond_total}" -ne "${host_total}" || "${actiond_executed}" -ne "${host_executed}" ]]; then + echo "LLVM process-count mismatch:" >&2 + echo " actiond total=${actiond_total} executed=${actiond_executed}: ${actiond_process_summary}" >&2 + echo " Linux host total=${host_total} executed=${host_executed}: ${host_process_summary}" >&2 + exit 1 +fi + +ratio="$(awk -v actiond="${actiond_bazel_elapsed}" -v host="${host_bazel_elapsed}" 'BEGIN { printf "%.3f", actiond / host }')" +revision="$(git -C "${repo_root}" rev-parse HEAD)" +cat >"${summary_path}" <>"${GITHUB_STEP_SUMMARY}" +fi + +trap - EXIT From f9f694ef712343b1c68c51314d32c61c8d9456f6 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 12 Jun 2026 09:33:24 -0400 Subject: [PATCH 05/10] ci: parse LLVM execution process counts --- e2e/run_llvm_linux_vm_smoke.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/e2e/run_llvm_linux_vm_smoke.sh b/e2e/run_llvm_linux_vm_smoke.sh index 8458439..c64f1f1 100755 --- a/e2e/run_llvm_linux_vm_smoke.sh +++ b/e2e/run_llvm_linux_vm_smoke.sh @@ -139,22 +139,22 @@ process_total() { sed -E 's/^([0-9]+) processes:.*/\1/' <<<"$1" } -process_executed() { +process_kind_count() { local summary="$1" - local total="$2" - awk -v total="${total}" -F',' ' + local kind="$2" + awk -v kind="${kind}" -F',' ' { - skipped = 0 + count = 0 for (i = 1; i <= NF; i++) { part = $i sub(/^[^:]*: /, "", part) gsub(/^[[:space:]]+|[[:space:].]+$/, "", part) - if (part ~ /^[0-9]+ (action cache hit|disk cache hit|remote cache hit|internal)$/) { + if (part ~ ("^[0-9]+ " kind "$")) { split(part, fields, " ") - skipped += fields[1] + count += fields[1] } } - print total - skipped + print count } ' <<<"${summary}" } @@ -186,6 +186,7 @@ measure() { local -a mode_flags=() local -a warmup_cache_flags=(--noremote_accept_cached) local -a measured_cache_flags=() + local execution_kind rm -rf "${output_base}" if [[ "${mode}" == "actiond" ]]; then @@ -199,6 +200,7 @@ measure() { --genrule_strategy=remote ) measured_cache_flags=(--remote_accept_cached) + execution_kind="remote" else mode_flags=( --remote_executor= @@ -207,6 +209,7 @@ measure() { --genrule_strategy=local ) measured_cache_flags=(--noremote_accept_cached) + execution_kind="local" fi echo "Starting ${mode} LLVM warmup" >&2 @@ -253,7 +256,7 @@ measure() { return 1 fi total="$(process_total "${process_summary}")" - executed="$(process_executed "${process_summary}" "${total}")" + executed="$(process_kind_count "${process_summary}" "${execution_kind}")" if [[ ! "${total}" =~ ^[0-9]+$ || ! "${executed}" =~ ^[0-9]+$ || "${executed}" -eq 0 ]]; then echo "could not parse ${mode} process counts: ${process_summary}" >&2 return 1 From fba591a3bd7e57dde131c4e79a5b605ded0dc185 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 12 Jun 2026 14:51:14 -0400 Subject: [PATCH 06/10] linux: add ARM64 QEMU release and smoke test --- .github/workflows/build_release.sh | 3 + .github/workflows/ci.yml | 25 ++++-- .github/workflows/release.yml | 1 + ARCHITECTURE.md | 20 ++--- MODULE.bazel | 4 + README.md | 14 ++-- cmd/linux-actiond/BUILD.bazel | 30 +++++--- cmd/linux-actiond/main.zig | 6 +- e2e/README.md | 6 +- e2e/run_llvm_linux_vm_smoke.sh | 54 ++++++++----- platforms/BUILD.bazel | 9 +++ src/darwin_vm_host.zig | 56 ++++---------- src/linux_vm_host.zig | 97 +++++++++-------------- src/qemu_vm.zig | 120 ++++++++++++++++++++--------- src/vm_host.zig | 93 ++++++++++++++++++++-- tools/e2e.sh | 22 ++++-- tools/zig_embedded_qemu.bzl | 69 ++++++++++------- vm/linux.config | 4 + 18 files changed, 394 insertions(+), 239 deletions(-) diff --git a/.github/workflows/build_release.sh b/.github/workflows/build_release.sh index 0575736..5c65553 100755 --- a/.github/workflows/build_release.sh +++ b/.github/workflows/build_release.sh @@ -18,12 +18,14 @@ fi bazel build "${bazel_flags[@]}" -c opt \ //cmd/darwin-actiond:darwin-actiond_macos_arm64 \ + //cmd/linux-actiond:linux-actiond_linux_arm64 \ //cmd/linux-actiond:linux-actiond_linux_x86_64 \ //cmd/windows-actiond:windows-actiond_windows_arm64 \ //cmd/windows-actiond:windows-actiond_windows_x86_64 cp -f \ bazel-bin/cmd/darwin-actiond/darwin-actiond_macos_arm64/darwin-actiond_macos_arm64 \ + bazel-bin/cmd/linux-actiond/linux-actiond_linux_arm64/linux-actiond_linux_arm64 \ bazel-bin/cmd/linux-actiond/linux-actiond_linux_x86_64/linux-actiond_linux_x86_64 \ bazel-bin/cmd/windows-actiond/windows-actiond_windows_arm64/windows-actiond_windows_arm64.exe \ bazel-bin/cmd/windows-actiond/windows-actiond_windows_x86_64/windows-actiond_windows_x86_64.exe \ @@ -32,6 +34,7 @@ cp -f \ cd "${artifact_dir}" shasum -a 256 \ darwin-actiond_macos_arm64 \ + linux-actiond_linux_arm64 \ linux-actiond_linux_x86_64 \ windows-actiond_windows_arm64.exe \ windows-actiond_windows_x86_64.exe \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70b7c59..dccac0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,10 @@ on: - main workflow_dispatch: +concurrency: + group: build-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: release-binaries: name: Release binaries @@ -25,8 +29,16 @@ jobs: run: .github/workflows/build_release.sh release linux-qemu-llvm-smoke: - name: Linux QEMU LLVM smoke - runs-on: ubuntu-latest + name: Linux QEMU LLVM smoke (${{ matrix.architecture }}) + strategy: + fail-fast: false + matrix: + include: + - architecture: arm64 + runner: ubuntu-24.04-arm + - architecture: x86_64 + runner: ubuntu-24.04 + runs-on: ${{ matrix.runner }} timeout-minutes: 240 steps: @@ -44,19 +56,20 @@ jobs: - name: Compare actiond and Linux host env: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + ACTIOND_REVISION: ${{ github.event.pull_request.head.sha || github.sha }} ACTIOND_VM_CAS_IMAGE_SIZE_MIB: 8192 ACTIOND_VM_MEMORY_MIB: 4096 - run: e2e/run_llvm_linux_vm_smoke.sh "$RUNNER_TEMP/actiond-linux-llvm" + run: e2e/run_llvm_linux_vm_smoke.sh "$RUNNER_TEMP/actiond-linux-llvm-${{ matrix.architecture }}" - name: Upload Linux LLVM smoke results if: always() uses: actions/upload-artifact@v4 with: - name: linux-qemu-llvm-smoke + name: linux-qemu-llvm-smoke-${{ matrix.architecture }} if-no-files-found: warn path: | - ${{ runner.temp }}/actiond-linux-llvm/*.md - ${{ runner.temp }}/actiond-linux-llvm/*.log + ${{ runner.temp }}/actiond-linux-llvm-${{ matrix.architecture }}/*.md + ${{ runner.temp }}/actiond-linux-llvm-${{ matrix.architecture }}/*.log windows-llvm-smoke: name: Windows ${{ matrix.mode }} (${{ matrix.architecture }}) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bc77140..6ff2469 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,6 +39,7 @@ jobs: fi gh release upload "${TAG_NAME}" \ release/darwin-actiond_macos_arm64 \ + release/linux-actiond_linux_arm64 \ release/linux-actiond_linux_x86_64 \ release/windows-actiond_windows_arm64.exe \ release/windows-actiond_windows_x86_64.exe \ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7971f51..a6debe0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -33,9 +33,9 @@ guest ext4 disk mounted at /cas On Windows, `windows-actiond` uses Host Compute System `LinuxKernelDirect`, Hyper-V synthetic SCSI, and `AF_HYPERV`. Guest AF_VSOCK port 5001 maps to the standard Hyper-V socket service GUID template. The Windows guest matches the -ARM64 or x86_64 host architecture; the macOS guest is ARM64. On Linux x86_64, -`linux-actiond` uses QEMU microvm, virtio-mmio block devices, and -`vhost-vsock-device`. +ARM64 or x86_64 host architecture; the macOS guest is ARM64. On Linux, +`linux-actiond` uses QEMU `virt` on ARM64 and QEMU `q35` on x86_64. Both QEMU +machines use virtio PCI block devices and `vhost-vsock-pci`. In VM mode, the host does not keep a second CAS mirror. Uploads, downloads, ActionCache requests, and Execute requests are forwarded to the guest. The @@ -55,13 +55,13 @@ the matching Linux kernel, initramfs, and runtime SquashFS. At startup it materializes those bytes under `--root`, wraps the runtime and CAS as fixed VHD files, and starts the VM with Host Compute System. -`linux-actiond` is released for x86_64. Zig `@embedFile` includes the Linux -kernel, initramfs, runtime SquashFS, `qemu-system-x86_64`, `bios-256k.bin`, -and `linuxboot_dma.bin` selected by the `rules_qemu` target toolchain. -`linux-actiond` materializes the two firmware files under `--root`, writes -`qemu-system-x86_64` to a sealed memfd, and executes it with `execveat`; -`qemu-system-x86_64` is not extracted to disk. The current implementation uses -TCG. KVM and `io_uring` remain follow-up work. +`linux-actiond` is released for ARM64 and x86_64. Zig `@embedFile` includes the +matching Linux kernel, initramfs, runtime SquashFS, and QEMU executable selected +by the `rules_qemu` target toolchain. The x86_64 release also includes +`bios-256k.bin` and `linuxboot_dma.bin`; QEMU `virt` direct kernel boot does not +require those PC firmware files. `linux-actiond` writes QEMU to a sealed memfd +and executes it with `execveat`; QEMU is not extracted to disk. The current +implementation uses TCG. KVM and `io_uring` remain follow-up work. `linux-actiond-guest` lives in the initramfs. It runs as guest init, mounts the minimal guest filesystems, mounts `/cas` and `/runtimes`, then execs itself as diff --git a/MODULE.bazel b/MODULE.bazel index e5e4086..7c810b5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -63,6 +63,10 @@ register_toolchains("@llvm//toolchain:all") register_toolchains("@codesign.bzl//toolchain:all") qemu = use_extension("@rules_qemu//qemu/extension:qemu.bzl", "qemu") +qemu.system_toolchain( + system_target = "aarch64-softmmu", + target_settings = ["//platforms:qemu_system_aarch64"], +) qemu.system_toolchain( system_target = "x86_64-softmmu", target_settings = ["//platforms:qemu_system_x86_64"], diff --git a/README.md b/README.md index a157fb6..fb2ce6a 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ Windows, and Linux it starts a small Linux VM and runs Bazel actions inside that VM, so the host can act like a local Linux remote-execution worker. -The macOS ARM64, Windows ARM64/x86_64, and Linux x86_64 releases include the -matching VM kernel, initramfs, and Linux runtime image. +The macOS ARM64, Windows ARM64/x86_64, and Linux ARM64/x86_64 releases include +the matching VM kernel, initramfs, and Linux runtime image. ## Why Use It? @@ -61,11 +61,10 @@ Windows requires Hyper-V. `windows-actiond` wraps the runtime SquashFS and guest-owned ext4 CAS in fixed VHD files. The default VHD paths are under `--root`; `--cas-image` can select another CAS VHD path. -The Linux x86_64 release embeds `qemu-system-x86_64`, `bios-256k.bin`, and -`linuxboot_dma.bin` from `rules_qemu`. `linux-actiond` materializes the two -firmware files under `--root` and executes QEMU from a sealed memfd. QEMU is -not extracted to disk. The current implementation uses TCG and requires -`/dev/vhost-vsock`: +The Linux releases embed the matching QEMU executable from `rules_qemu`. The +x86_64 release also embeds `bios-256k.bin` and `linuxboot_dma.bin`. +`linux-actiond` executes QEMU from a sealed memfd, so QEMU is not extracted to +disk. The current implementation uses TCG and requires `/dev/vhost-vsock`: ```bash sudo modprobe vhost_vsock @@ -134,6 +133,7 @@ Most users should use releases. Source builds are mainly for development: ```bash bazel build --config=remote -c opt //cmd/darwin-actiond +bazel build --config=remote -c opt //cmd/linux-actiond:linux-actiond_linux_arm64 bazel build --config=remote -c opt //cmd/linux-actiond:linux-actiond_linux_x86_64 ``` diff --git a/cmd/linux-actiond/BUILD.bazel b/cmd/linux-actiond/BUILD.bazel index 34de09e..24478e5 100644 --- a/cmd/linux-actiond/BUILD.bazel +++ b/cmd/linux-actiond/BUILD.bazel @@ -7,10 +7,7 @@ zig_binary( name = "linux-actiond", main = "main.zig", strip_debug_symbols = True, - target_compatible_with = [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], + target_compatible_with = ["@platforms//os:linux"], visibility = ["//visibility:public"], deps = [ ":embedded_assets", @@ -19,19 +16,32 @@ zig_binary( ], ) +alias( + name = "embedded_initramfs", + actual = select({ + "@platforms//cpu:aarch64": "//vm:initramfs_aarch64", + "@platforms//cpu:x86_64": "//vm:initramfs_x86_64", + }), +) + zig_embedded_assets( name = "embedded_assets", - initramfs = "//vm:initramfs_x86_64", + initramfs = ":embedded_initramfs", kernel = "//vm:linux_kernel.image", - runtime_image = "//runtimes:runtimes_squashfs_x86_64", + runtime_image = "//runtimes:runtimes_squashfs", ) zig_embedded_qemu( name = "embedded_qemu", - target_compatible_with = [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], + target_compatible_with = ["@platforms//os:linux"], +) + +platform_transition_binary( + name = "linux-actiond_linux_arm64", + basename = "linux-actiond_linux_arm64", + binary = ":linux-actiond", + target_platform = "//platforms:linux_aarch64_musl", + visibility = ["//visibility:public"], ) platform_transition_binary( diff --git a/cmd/linux-actiond/main.zig b/cmd/linux-actiond/main.zig index 73e3fa5..b3d8b96 100644 --- a/cmd/linux-actiond/main.zig +++ b/cmd/linux-actiond/main.zig @@ -4,14 +4,12 @@ const actiond = @import("actiond"); const embedded_assets = @import("actiond_embedded_assets"); const embedded_qemu = @import("actiond_embedded_qemu"); -const fexec_qemu_argument = "--actiond-internal-fexec-qemu"; - pub fn main(init: std.process.Init) !void { const io = init.io; const arena = init.arena.allocator(); const args = try init.minimal.args.toSlice(arena); - if (args.len > 1 and std.mem.eql(u8, args[1], fexec_qemu_argument)) { + if (args.len > 1 and std.mem.eql(u8, args[1], actiond.qemu_vm.fexec_argument)) { if (args.len < 3) return error.MissingQemuArguments; try actiond.qemu_vm.fexecEmbedded( io, @@ -33,7 +31,7 @@ pub fn main(init: std.process.Init) !void { try stdout.print( \\linux-actiond zig={s} bazel={s} \\usage: - \\ linux-actiond serve-vm [--kernel=/path/bzImage] [--initramfs=/path/initramfs.cpio[.zst]] [--runtime-image=/path/runtimes.sqfs] [--cas-image=/path/cas.ext4] [--listen=127.0.0.1:8980] [--root=/tmp/actiond-vm] [--actiondfs-stats-path=/path/stats.txt] + \\ linux-actiond serve-vm [--kernel=/path/kernel] [--initramfs=/path/initramfs.cpio[.zst]] [--runtime-image=/path/runtimes.sqfs] [--cas-image=/path/cas.ext4] [--listen=127.0.0.1:8980] [--root=/tmp/actiond-vm] [--actiondfs-stats-path=/path/stats.txt] \\ , .{ actiond.version.zig, actiond.version.bazel }); try stdout.flush(); diff --git a/e2e/README.md b/e2e/README.md index d50ea9a..3b420d8 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -104,9 +104,9 @@ LLVM comparison. ## Linux QEMU LLVM Smoke Runner `run_llvm_linux_vm_smoke.sh` builds `@llvm-project//llvm:llvm-tblgen` once -through `linux-actiond` and once directly on the Linux x86_64 host. Both -measurements use fresh Bazel output bases, the Linux x86_64 musl target and -host platforms, and `//e2e:llvm_exec_warmup` before the measured build. The +through `linux-actiond` and once directly on the Linux ARM64 or x86_64 host. +Both measurements use fresh Bazel output bases, the matching Linux musl target +and host platforms, and `//e2e:llvm_exec_warmup` before the measured build. The runner stops QEMU before the Linux-host measurement, passes no `--jobs` value by default, and requires equal total and executed process counts. It writes `linux-llvm-smoke-timings.md` with Bazel elapsed time, wall time, process diff --git a/e2e/run_llvm_linux_vm_smoke.sh b/e2e/run_llvm_linux_vm_smoke.sh index c64f1f1..ab88775 100755 --- a/e2e/run_llvm_linux_vm_smoke.sh +++ b/e2e/run_llvm_linux_vm_smoke.sh @@ -4,10 +4,30 @@ set -euo pipefail export LC_ALL=C repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then - echo "Linux LLVM VM smoke requires a Linux x86_64 host" >&2 +if [[ "$(uname -s)" != "Linux" ]]; then + echo "Linux LLVM VM smoke requires Linux" >&2 exit 1 fi +case "$(uname -m)" in + aarch64|arm64) + architecture="arm64" + qemu_machine="virt" + default_llvm_platform="@llvm//platforms:linux_arm64_musl" + default_execution_platform="//e2e:actiond_linux_arm64_musl_exec" + server_target="//cmd/linux-actiond:linux-actiond_linux_arm64" + ;; + x86_64) + architecture="x86_64" + qemu_machine="q35" + default_llvm_platform="@llvm//platforms:linux_x86_64_musl" + default_execution_platform="//e2e:actiond_linux_x86_64_musl_exec" + server_target="//cmd/linux-actiond:linux-actiond_linux_x86_64" + ;; + *) + echo "Linux LLVM VM smoke does not support host architecture $(uname -m)" >&2 + exit 1 + ;; +esac if [[ "$#" -gt 1 ]]; then echo "usage: e2e/run_llvm_linux_vm_smoke.sh [output-directory]" >&2 exit 1 @@ -15,10 +35,9 @@ fi output_root="${1:-${ACTIOND_LLVM_LINUX_SMOKE_ROOT:-$(mktemp -d "${TMPDIR:-/tmp}/actiond-linux-llvm.XXXXXX")}}" target="${ACTIOND_LLVM_SMOKE_TARGET:-@llvm-project//llvm:llvm-tblgen}" warmup_target="${ACTIOND_LLVM_SMOKE_WARMUP_TARGET-//e2e:llvm_exec_warmup}" -llvm_platform="${ACTIOND_LLVM_SMOKE_PLATFORM:-@llvm//platforms:linux_x86_64_musl}" -execution_platform="${ACTIOND_LLVM_SMOKE_EXEC_PLATFORM:-//e2e:actiond_linux_x86_64_musl_exec}" +llvm_platform="${ACTIOND_LLVM_SMOKE_TARGET_PLATFORM:-${default_llvm_platform}}" +execution_platform="${ACTIOND_LLVM_SMOKE_EXEC_PLATFORM:-${default_execution_platform}}" endpoint="${ACTIOND_LLVM_SMOKE_ENDPOINT:-127.0.0.1:8998}" -server_target="//cmd/linux-actiond:linux-actiond_linux_x86_64" vm_cpus="${ACTIOND_VM_CPUS:-$(nproc)}" vm_memory_mib="${ACTIOND_VM_MEMORY_MIB:-4096}" cas_image_size_mib="${ACTIOND_VM_CAS_IMAGE_SIZE_MIB:-8192}" @@ -42,7 +61,7 @@ server_pid="" stop_server() { if [[ -n "${server_pid}" ]]; then - kill "${server_pid}" >/dev/null 2>&1 || true + kill -- "-${server_pid}" >/dev/null 2>&1 || true wait "${server_pid}" >/dev/null 2>&1 || true server_pid="" fi @@ -59,9 +78,7 @@ cleanup() { trap 'cleanup $?' EXIT -wait_for_port() { - local host="${endpoint%:*}" - local port="${endpoint##*:}" +wait_for_server() { local start start="$(date +%s)" while true; do @@ -69,10 +86,10 @@ wait_for_port() { echo "linux-actiond exited before ${endpoint} became ready" >&2 return 1 fi - if (echo >"/dev/tcp/${host}/${port}") >/dev/null 2>&1; then + if grep -Fq "actiond VM raw gRPC bridge listening on ${endpoint}" "${server_log}"; then return 0 fi - if (( "$(date +%s)" - start >= 180 )); then + if (( "$(date +%s)" - start >= 240 )); then echo "timed out waiting for ${endpoint}" >&2 return 1 fi @@ -123,16 +140,16 @@ setup_server() { bazel shutdown ) >>"${build_log}" 2>&1 - "${server}" serve-vm \ + setsid "${server}" serve-vm \ --listen="${endpoint}" \ --root="${server_root}" \ --cas-image-size-mib="${cas_image_size_mib}" \ --memory-mib="${vm_memory_mib}" \ --cpus="${vm_cpus}" \ - --connect-timeout-ms=900000 \ + --start-timeout-ms=180000 \ >"${server_log}" 2>&1 & server_pid="$!" - wait_for_port + wait_for_server } process_total() { @@ -297,19 +314,20 @@ if [[ "${actiond_total}" -ne "${host_total}" || "${actiond_executed}" -ne "${hos fi ratio="$(awk -v actiond="${actiond_bazel_elapsed}" -v host="${host_bazel_elapsed}" 'BEGIN { printf "%.3f", actiond / host }')" -revision="$(git -C "${repo_root}" rev-parse HEAD)" +revision="${ACTIOND_REVISION:-$(git -C "${repo_root}" rev-parse HEAD)}" cat >"${summary_path}" <= @as(i64, timeout_ms)) return error.ConnectTimedOut; + const remaining_ms: u32 = timeout_ms - @as(u32, @intCast(elapsed_ms)); + const attempt_timeout_ms = @min(self.connect_attempt_timeout_ms, remaining_ms); + if (connectVsock(self.guest_cid, port, attempt_timeout_ms)) |fd| return fd else |err| { if (try self.reapExitedChild()) |status| { std.log.err("QEMU exited before guest vsock became ready status=0x{x}", .{status}); return error.StartFailed; } - if (remaining_ms <= self.connect_attempt_timeout_ms) { + const updated_elapsed_ms = started.durationTo(std.Io.Clock.awake.now(self.io)).toMilliseconds(); + if (updated_elapsed_ms >= @as(i64, timeout_ms)) { std.log.err("timed out connecting to guest cid={d} vsock:{d}: {s}", .{ self.guest_cid, port, @errorName(err) }); return error.ConnectTimedOut; } } - const sleep_ms = @min(@as(u32, 100), remaining_ms); + const before_sleep_ms = started.durationTo(std.Io.Clock.awake.now(self.io)).toMilliseconds(); + if (before_sleep_ms >= @as(i64, timeout_ms)) return error.ConnectTimedOut; + const sleep_ms = @min(@as(u32, 100), timeout_ms - @as(u32, @intCast(before_sleep_ms))); try self.io.sleep(.fromMilliseconds(sleep_ms), .awake); - remaining_ms -= sleep_ms; } } @@ -235,7 +247,7 @@ pub fn fexecEmbedded( for (environment, envp) |entry, *output| output.* = entry; const result = linux.execveat(fd, "", argv.ptr, envp.ptr, .{ .SYMLINK_NOFOLLOW = false, .EMPTY_PATH = true }); - std.log.err("execveat for embedded qemu-system-x86_64 failed: {s}", .{@tagName(linux.errno(result))}); + std.log.err("execveat for embedded QEMU failed: {s}", .{@tagName(linux.errno(result))}); return error.FexecFailed; } @@ -272,15 +284,21 @@ fn randomGuestCid(io: std.Io) !u32 { return 3 + random % (std.math.maxInt(u32) - 3); } -fn kernelAppendArg(allocator: std.mem.Allocator, format_cas_image: bool) ![]u8 { - return allocator.dupe(u8, if (format_cas_image) - "init=/init console=ttyS0 panic=-1 actiond.cas_device=/dev/vda actiond.format_cas=1" +fn kernelAppendArg(allocator: std.mem.Allocator, target_arch: []const u8, format_cas_image: bool) ![]u8 { + const console = if (std.mem.eql(u8, target_arch, "aarch64")) + "ttyAMA0" + else if (std.mem.eql(u8, target_arch, "x86_64")) + "ttyS0" else - "init=/init console=ttyS0 panic=-1 actiond.cas_device=/dev/vda"); + return error.UnsupportedHost; + return std.fmt.allocPrint(allocator, "init=/init console={s} panic=-1 actiond.cas_device=/dev/vda{s}", .{ + console, + if (format_cas_image) " actiond.format_cas=1" else "", + }); } -fn connectVsock(cid: u32, port: u32) !std.posix.fd_t { - const socket_result = linux.socket(linux.AF.VSOCK, linux.SOCK.STREAM | linux.SOCK.CLOEXEC, 0); +fn connectVsock(cid: u32, port: u32, timeout_ms: u32) !std.posix.fd_t { + const socket_result = linux.socket(linux.AF.VSOCK, linux.SOCK.STREAM | linux.SOCK.CLOEXEC | linux.SOCK.NONBLOCK, 0); if (linux.errno(socket_result) != .SUCCESS) return error.ConnectFailed; const fd: std.posix.fd_t = @intCast(socket_result); errdefer closeFd(fd); @@ -298,7 +316,30 @@ fn connectVsock(cid: u32, port: u32) !std.posix.fd_t { @ptrCast(&address), @sizeOf(linux.sockaddr.vm), ); - if (linux.errno(connect_result) != .SUCCESS) return error.ConnectFailed; + switch (linux.errno(connect_result)) { + .SUCCESS => {}, + .AGAIN, .INPROGRESS => { + var poll_fds = [_]linux.pollfd{.{ .fd = fd, .events = linux.POLL.OUT, .revents = 0 }}; + const poll_result = linux.poll(poll_fds[0..].ptr, poll_fds.len, @intCast(timeout_ms)); + if (linux.errno(poll_result) != .SUCCESS or poll_result == 0) return error.ConnectFailed; + + var socket_error: i32 = 0; + var socket_error_len: linux.socklen_t = @sizeOf(@TypeOf(socket_error)); + const socket_error_result = linux.getsockopt( + fd, + linux.SOL.SOCKET, + linux.SO.ERROR, + std.mem.asBytes(&socket_error).ptr, + &socket_error_len, + ); + if (linux.errno(socket_error_result) != .SUCCESS or socket_error != 0) return error.ConnectFailed; + }, + else => return error.ConnectFailed, + } + const flags_result = linux.fcntl(fd, linux.F.GETFL, 0); + if (linux.errno(flags_result) != .SUCCESS) return error.ConnectFailed; + const blocking_result = linux.fcntl(fd, linux.F.SETFL, flags_result & ~@as(usize, linux.SOCK.NONBLOCK)); + if (linux.errno(blocking_result) != .SUCCESS) return error.ConnectFailed; return fd; } @@ -322,11 +363,20 @@ test "driveArg escapes commas in paths" { try std.testing.expectEqualStrings("if=none,id=cas,file=/tmp/actiond,,vm/cas.ext4,format=raw,cache=none", drive); } -test "kernelAppendArg identifies the CAS block device" { - const argument = try kernelAppendArg(std.testing.allocator, true); +test "kernelAppendArg selects the x86 console and CAS block device" { + const argument = try kernelAppendArg(std.testing.allocator, "x86_64", true); defer std.testing.allocator.free(argument); try std.testing.expectEqualStrings( "init=/init console=ttyS0 panic=-1 actiond.cas_device=/dev/vda actiond.format_cas=1", argument, ); } + +test "kernelAppendArg selects the aarch64 console" { + const argument = try kernelAppendArg(std.testing.allocator, "aarch64", false); + defer std.testing.allocator.free(argument); + try std.testing.expectEqualStrings( + "init=/init console=ttyAMA0 panic=-1 actiond.cas_device=/dev/vda", + argument, + ); +} diff --git a/src/vm_host.zig b/src/vm_host.zig index 4902abb..ebc68de 100644 --- a/src/vm_host.zig +++ b/src/vm_host.zig @@ -2,6 +2,7 @@ const builtin = @import("builtin"); const std = @import("std"); const control_protocol = @import("control_protocol.zig"); const control_transport_fd = @import("control_transport_fd.zig"); +const grpc_vsock_bridge = @import("grpc_vsock_bridge.zig"); const zstd_test = if (builtin.is_test) @import("c") else struct {}; const max_compressed_initramfs_bytes = 128 * 1024 * 1024; @@ -27,6 +28,54 @@ pub const ResolvedAssets = struct { } }; +pub const PreparedVm = struct { + root_dir: std.Io.Dir, + assets: ResolvedAssets = .{}, + cas_image_path: []const u8 = "", + owned_cas_image_path: ?[]u8 = null, + boot_kernel_path: []const u8 = "", + owned_boot_kernel_path: ?[]u8 = null, + boot_initramfs_path: []const u8 = "", + owned_boot_initramfs_path: ?[]u8 = null, + format_cas_image: bool = false, + + pub fn deinit(self: *PreparedVm, io: std.Io, allocator: std.mem.Allocator) void { + if (self.owned_boot_initramfs_path) |path| allocator.free(path); + if (self.owned_boot_kernel_path) |path| allocator.free(path); + self.assets.deinit(allocator); + if (self.owned_cas_image_path) |path| allocator.free(path); + self.root_dir.close(io); + self.* = undefined; + } +}; + +pub fn prepareVm( + io: std.Io, + allocator: std.mem.Allocator, + options: ServeVmOptions, + comptime embedded_assets: type, +) !PreparedVm { + var prepared: PreparedVm = .{ + .root_dir = try std.Io.Dir.cwd().createDirPathOpen(io, options.root, .{}), + }; + errdefer prepared.deinit(io, allocator); + + if (options.cas_image) |path| { + prepared.cas_image_path = path; + } else { + prepared.owned_cas_image_path = try std.fs.path.join(allocator, &.{ options.root, "cas.ext4" }); + prepared.cas_image_path = prepared.owned_cas_image_path.?; + } + prepared.format_cas_image = try ensureCasImageFile(io, prepared.cas_image_path, options.cas_image_size_mib); + + prepared.assets = try resolveAssets(io, allocator, prepared.root_dir, options, embedded_assets); + prepared.owned_boot_kernel_path = try prepareBootKernel(io, allocator, prepared.root_dir, prepared.assets.kernel); + prepared.boot_kernel_path = prepared.owned_boot_kernel_path orelse prepared.assets.kernel; + prepared.owned_boot_initramfs_path = try prepareBootInitramfs(io, allocator, prepared.root_dir, prepared.assets.initramfs); + prepared.boot_initramfs_path = prepared.owned_boot_initramfs_path orelse prepared.assets.initramfs; + return prepared; +} + pub fn resolveAssets( io: std.Io, allocator: std.mem.Allocator, @@ -70,7 +119,7 @@ fn materializeEmbeddedAsset( const output_rel = try std.fmt.allocPrint(allocator, "embedded/{s}-{s}", .{ name, std.fmt.bytesToHex(hash, .lower) }); defer allocator.free(output_rel); - return materializeEmbeddedFile(io, allocator, root_dir, output_rel, bytes); + return materializeEmbeddedFileWithHash(io, allocator, root_dir, output_rel, bytes, &hash); } pub fn materializeEmbeddedFile( @@ -80,14 +129,24 @@ pub fn materializeEmbeddedFile( output_rel: []const u8, bytes: []const u8, ) ![]u8 { - const parent = std.fs.path.dirname(output_rel) orelse "."; - if (!std.mem.eql(u8, parent, ".")) try root_dir.createDirPath(io, parent); - var hash: [32]u8 = undefined; std.crypto.hash.sha2.Sha256.hash(bytes, &hash, .{}); + return materializeEmbeddedFileWithHash(io, allocator, root_dir, output_rel, bytes, &hash); +} + +fn materializeEmbeddedFileWithHash( + io: std.Io, + allocator: std.mem.Allocator, + root_dir: std.Io.Dir, + output_rel: []const u8, + bytes: []const u8, + hash: *const [32]u8, +) ![]u8 { + const parent = std.fs.path.dirname(output_rel) orelse "."; + if (!std.mem.eql(u8, parent, ".")) try root_dir.createDirPath(io, parent); if (root_dir.statFile(io, output_rel, .{})) |stat| { - if (stat.kind == .file and stat.size == bytes.len and try embeddedAssetMatches(io, root_dir, output_rel, &hash)) { + if (stat.kind == .file and stat.size == bytes.len and try embeddedAssetMatches(io, root_dir, output_rel, hash)) { return absoluteSubPath(io, allocator, root_dir, output_rel); } } else |err| switch (err) { @@ -128,7 +187,7 @@ pub fn materializeEmbeddedFile( output_open = false; root_dir.renamePreserve(temp_path, root_dir, output_rel, io) catch |err| switch (err) { error.PathAlreadyExists => { - if (try embeddedAssetMatches(io, root_dir, output_rel, &hash)) { + if (try embeddedAssetMatches(io, root_dir, output_rel, hash)) { try root_dir.deleteFile(io, temp_path); } else { try root_dir.rename(temp_path, root_dir, output_rel, io); @@ -379,7 +438,7 @@ pub fn absoluteSubPath( return std.fs.path.join(allocator, &.{ root_buffer[0..root_len], sub_path }); } -pub fn ensureCasImageFile(io: std.Io, path: []const u8, size_mib: u64) !bool { +fn ensureCasImageFile(io: std.Io, path: []const u8, size_mib: u64) !bool { if (std.Io.Dir.cwd().statFile(io, path, .{})) |stat| { if (stat.kind != .file) return error.InvalidCasImage; return false; @@ -396,7 +455,25 @@ pub fn ensureCasImageFile(io: std.Io, path: []const u8, size_mib: u64) !bool { return true; } -pub fn actiondfsStatsTask( +pub fn serveGrpcBridge( + io: std.Io, + allocator: std.mem.Allocator, + options: ServeVmOptions, + machine: anytype, +) !void { + var fd_client = control_transport_fd.Client{ .opener = machine.opener() }; + defer fd_client.deinit(io); + var background_tasks: std.Io.Group = .init; + defer background_tasks.cancel(io); + if (options.actiondfs_stats_path) |path| { + const stats_path = try allocator.dupe(u8, path); + errdefer allocator.free(stats_path); + try background_tasks.concurrent(io, actiondfsStatsTask, .{ io, allocator, &fd_client, stats_path }); + } + return grpc_vsock_bridge.serve(io, options.listen, machine); +} + +fn actiondfsStatsTask( io: std.Io, allocator: std.mem.Allocator, client: *control_transport_fd.Client, diff --git a/tools/e2e.sh b/tools/e2e.sh index df0f558..85c3981 100755 --- a/tools/e2e.sh +++ b/tools/e2e.sh @@ -176,16 +176,24 @@ run_vm_e2e() { server_name="darwin-actiond" ;; Linux) - if [[ "$(uname -m)" != "x86_64" ]]; then - echo "QEMU VM e2e currently requires Linux x86_64" >&2 - return 1 - fi - architecture="x86_64" - server_label="//cmd/linux-actiond:linux-actiond_linux_x86_64" + case "$(uname -m)" in + aarch64|arm64) + architecture="aarch64" + server_label="//cmd/linux-actiond:linux-actiond_linux_arm64" + ;; + x86_64) + architecture="x86_64" + server_label="//cmd/linux-actiond:linux-actiond_linux_x86_64" + ;; + *) + echo "QEMU VM e2e does not support Linux architecture $(uname -m)" >&2 + return 1 + ;; + esac server_name="linux-actiond" ;; *) - echo "VM e2e requires macOS or Linux x86_64" >&2 + echo "VM e2e requires macOS or Linux" >&2 return 1 ;; esac diff --git a/tools/zig_embedded_qemu.bzl b/tools/zig_embedded_qemu.bzl index 04310c4..23d79a1 100644 --- a/tools/zig_embedded_qemu.bzl +++ b/tools/zig_embedded_qemu.bzl @@ -5,39 +5,52 @@ _QEMU_TOOLCHAIN_TYPE = "@rules_qemu//qemu:target_toolchain_type" def _zig_embedded_qemu_source_impl(ctx): qemu = ctx.toolchains[_QEMU_TOOLCHAIN_TYPE] qemu_system = ctx.actions.declare_file(ctx.label.name + ".qemu-system") - bios_256k = ctx.actions.declare_file(ctx.label.name + ".bios-256k.bin") - linuxboot_dma = ctx.actions.declare_file(ctx.label.name + ".linuxboot_dma.bin") source = ctx.actions.declare_file(ctx.label.name + ".zig") ctx.actions.symlink(output = qemu_system, target_file = qemu.qemu_system) - ctx.actions.run_shell( - arguments = [ - qemu.system_data_anchor.path, - bios_256k.path, - linuxboot_dma.path, - ], - command = """ + embedded = [qemu_system] + lines = [ + 'pub const qemu_system = @embedFile("{}");'.format(qemu_system.basename), + 'pub const qemu_system_name = "{}";'.format(qemu.qemu_system.basename), + 'pub const accel = "{}";'.format(qemu.accel), + 'pub const machine = "{}";'.format(qemu.machine), + 'pub const system_target = "{}";'.format(qemu.system_target), + 'pub const target_arch = "{}";'.format(qemu.target_arch), + ] + + if qemu.system_target == "x86_64-softmmu": + bios_256k = ctx.actions.declare_file(ctx.label.name + ".bios-256k.bin") + linuxboot_dma = ctx.actions.declare_file(ctx.label.name + ".linuxboot_dma.bin") + ctx.actions.run_shell( + arguments = [ + qemu.system_data_anchor.path, + bios_256k.path, + linuxboot_dma.path, + ], + command = """ set -eu cp "$1/bios-256k.bin" "$2" cp "$1/linuxboot_dma.bin" "$3" """, - inputs = qemu.system_data_files, - outputs = [bios_256k, linuxboot_dma], - ) - ctx.actions.write( - source, - """\ -pub const qemu_system = @embedFile("{qemu_system}"); -pub const bios_256k = @embedFile("{bios_256k}"); -pub const linuxboot_dma = @embedFile("{linuxboot_dma}"); -""".format( - bios_256k = bios_256k.basename, - linuxboot_dma = linuxboot_dma.basename, - qemu_system = qemu_system.basename, - ), - ) + inputs = qemu.system_data_files, + outputs = [bios_256k, linuxboot_dma], + ) + embedded.extend([bios_256k, linuxboot_dma]) + lines.extend([ + 'pub const bios_256k: ?[]const u8 = @embedFile("{}");'.format(bios_256k.basename), + 'pub const linuxboot_dma: ?[]const u8 = @embedFile("{}");'.format(linuxboot_dma.basename), + ]) + elif qemu.system_target == "aarch64-softmmu": + lines.extend([ + "pub const bios_256k: ?[]const u8 = null;", + "pub const linuxboot_dma: ?[]const u8 = null;", + ]) + else: + fail("unsupported embedded QEMU system target: {}".format(qemu.system_target)) + + ctx.actions.write(source, "\n".join(lines) + "\n") return [ DefaultInfo(files = depset([source])), - OutputGroupInfo(embedded = depset([qemu_system, bios_256k, linuxboot_dma])), + OutputGroupInfo(embedded = depset(embedded)), ] _zig_embedded_qemu_source = rule( @@ -47,20 +60,20 @@ _zig_embedded_qemu_source = rule( def zig_embedded_qemu(name, target_compatible_with = [], visibility = None): source_name = name + "_source" - qemu_name = name + "_file" + files_name = name + "_files" _zig_embedded_qemu_source( name = source_name, target_compatible_with = target_compatible_with, ) native.filegroup( - name = qemu_name, + name = files_name, srcs = [":" + source_name], output_group = "embedded", target_compatible_with = target_compatible_with, ) zig_library( name = name, - extra_srcs = [":" + qemu_name], + extra_srcs = [":" + files_name], import_name = "actiond_embedded_qemu", main = ":" + source_name, target_compatible_with = target_compatible_with, diff --git a/vm/linux.config b/vm/linux.config index 492d5c8..90a7c3a 100644 --- a/vm/linux.config +++ b/vm/linux.config @@ -2,6 +2,8 @@ CONFIG_64BIT=y CONFIG_ACTIONDFS_FS=y CONFIG_ACPI=y CONFIG_ARM64=y +CONFIG_SMP=y +CONFIG_NR_CPUS=64 CONFIG_BINFMT_ELF=y CONFIG_BINFMT_SCRIPT=y CONFIG_BLOCK=y @@ -48,6 +50,8 @@ CONFIG_POSIX_TIMERS=y CONFIG_PROC_FS=y CONFIG_PRINTK=y CONFIG_SECCOMP=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y CONFIG_SCSI=y CONFIG_SCSI_COMMON=y CONFIG_SCSI_LOWLEVEL=y From 30cefb4a001fcdcd3289199093650b35bfdcb183 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sat, 13 Jun 2026 11:07:09 -0400 Subject: [PATCH 07/10] linux: boot QEMU from embedded memfds --- ARCHITECTURE.md | 13 +-- MODULE.bazel | 1 + README.md | 7 +- cmd/linux-actiond/main.zig | 5 +- e2e/run_llvm_linux_vm_smoke.sh | 14 ++- src/linux_vm_host.zig | 39 ++++---- src/qemu_vm.zig | 159 ++++++++++++++++++++++++++++----- src/vm_host.zig | 34 ++++--- tools/zig_embedded_qemu.bzl | 24 ++--- vm/linux.config | 1 + vm/linux_x86_64.config | 4 + 11 files changed, 211 insertions(+), 90 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a6debe0..1870932 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -34,8 +34,8 @@ On Windows, `windows-actiond` uses Host Compute System `LinuxKernelDirect`, Hyper-V synthetic SCSI, and `AF_HYPERV`. Guest AF_VSOCK port 5001 maps to the standard Hyper-V socket service GUID template. The Windows guest matches the ARM64 or x86_64 host architecture; the macOS guest is ARM64. On Linux, -`linux-actiond` uses QEMU `virt` on ARM64 and QEMU `q35` on x86_64. Both QEMU -machines use virtio PCI block devices and `vhost-vsock-pci`. +`linux-actiond` uses QEMU `virt` on ARM64 and QEMU `microvm` on x86_64. Both +QEMU machines use virtio-mmio block devices and `vhost-vsock-device`. In VM mode, the host does not keep a second CAS mirror. Uploads, downloads, ActionCache requests, and Execute requests are forwarded to the guest. The @@ -58,10 +58,11 @@ files, and starts the VM with Host Compute System. `linux-actiond` is released for ARM64 and x86_64. Zig `@embedFile` includes the matching Linux kernel, initramfs, runtime SquashFS, and QEMU executable selected by the `rules_qemu` target toolchain. The x86_64 release also includes -`bios-256k.bin` and `linuxboot_dma.bin`; QEMU `virt` direct kernel boot does not -require those PC firmware files. `linux-actiond` writes QEMU to a sealed memfd -and executes it with `execveat`; QEMU is not extracted to disk. The current -implementation uses TCG. KVM and `io_uring` remain follow-up work. +`qboot.rom`; QEMU `virt` direct kernel boot on ARM64 does not require firmware. +`linux-actiond` creates sealed memfds for QEMU and every immutable embedded VM +artifact, then executes QEMU with `execveat`. Only the persistent guest-owned +CAS image is stored under `--root`. The current implementation uses TCG. KVM +and `io_uring` remain follow-up work. `linux-actiond-guest` lives in the initramfs. It runs as guest init, mounts the minimal guest filesystems, mounts `/cas` and `/runtimes`, then execs itself as diff --git a/MODULE.bazel b/MODULE.bazel index 7c810b5..9cb1bf9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -68,6 +68,7 @@ qemu.system_toolchain( target_settings = ["//platforms:qemu_system_aarch64"], ) qemu.system_toolchain( + machine = "microvm,acpi=off,pcie=off,x-option-roms=off", system_target = "x86_64-softmmu", target_settings = ["//platforms:qemu_system_x86_64"], ) diff --git a/README.md b/README.md index fb2ce6a..82d8f8c 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,10 @@ guest-owned ext4 CAS in fixed VHD files. The default VHD paths are under `--root`; `--cas-image` can select another CAS VHD path. The Linux releases embed the matching QEMU executable from `rules_qemu`. The -x86_64 release also embeds `bios-256k.bin` and `linuxboot_dma.bin`. -`linux-actiond` executes QEMU from a sealed memfd, so QEMU is not extracted to -disk. The current implementation uses TCG and requires `/dev/vhost-vsock`: +x86_64 release also embeds `qboot.rom`. `linux-actiond` passes QEMU, the Linux +kernel, the initramfs, the runtime SquashFS, and `qboot.rom` through sealed +memfds. The guest-owned CAS remains a persistent ext4 file under `--root`. The +current implementation uses TCG and requires `/dev/vhost-vsock`: ```bash sudo modprobe vhost_vsock diff --git a/cmd/linux-actiond/main.zig b/cmd/linux-actiond/main.zig index b3d8b96..618296e 100644 --- a/cmd/linux-actiond/main.zig +++ b/cmd/linux-actiond/main.zig @@ -15,14 +15,15 @@ pub fn main(init: std.process.Init) !void { io, arena, init.minimal.environ, - embedded_qemu.qemu_system, + embedded_assets, + embedded_qemu, args[2..], ); } if (args.len > 1 and std.mem.eql(u8, args[1], "serve-vm")) { const options = try actiond.vm_host.parseServeVmArgs(args[2..]); - return actiond.linux_vm_host.serve(io, std.heap.smp_allocator, options, embedded_assets, embedded_qemu); + return actiond.linux_vm_host.serve(io, std.heap.smp_allocator, options, embedded_qemu); } var stdout_buffer: [256]u8 = undefined; diff --git a/e2e/run_llvm_linux_vm_smoke.sh b/e2e/run_llvm_linux_vm_smoke.sh index ab88775..e2dcc03 100755 --- a/e2e/run_llvm_linux_vm_smoke.sh +++ b/e2e/run_llvm_linux_vm_smoke.sh @@ -18,7 +18,7 @@ case "$(uname -m)" in ;; x86_64) architecture="x86_64" - qemu_machine="q35" + qemu_machine="microvm" default_llvm_platform="@llvm//platforms:linux_x86_64_musl" default_execution_platform="//e2e:actiond_linux_x86_64_musl_exec" server_target="//cmd/linux-actiond:linux-actiond_linux_x86_64" @@ -58,6 +58,7 @@ summary_path="${output_root}/linux-llvm-smoke-timings.md" rm -rf "${actiond_output_base}" "${host_output_base}" "${server_root}" server_pid="" +server_startup_elapsed="" stop_server() { if [[ -n "${server_pid}" ]]; then @@ -100,6 +101,7 @@ wait_for_server() { setup_server() { local build_log="${output_root}/linux-actiond-build.log" local cquery_log="${output_root}/linux-actiond-cquery.log" + local server_start_ns server_ready_ns local -a headers=() if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then headers+=( @@ -140,6 +142,7 @@ setup_server() { bazel shutdown ) >>"${build_log}" 2>&1 + server_start_ns="$(date +%s%N)" setsid "${server}" serve-vm \ --listen="${endpoint}" \ --root="${server_root}" \ @@ -150,6 +153,14 @@ setup_server() { >"${server_log}" 2>&1 & server_pid="$!" wait_for_server + server_ready_ns="$(date +%s%N)" + server_startup_elapsed="$(awk -v start="${server_start_ns}" -v end="${server_ready_ns}" 'BEGIN { printf "%.3f", (end - start) / 1000000000 }')" + + mapfile -t server_files < <(find "${server_root}" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort) + if [[ "${server_files[*]}" != "cas.ext4" ]]; then + echo "unexpected files under ${server_root}: ${server_files[*]}" >&2 + exit 1 + fi } process_total() { @@ -327,6 +338,7 @@ cat >"${summary_path}" < Date: Sat, 13 Jun 2026 21:42:10 +0200 Subject: [PATCH 08/10] build: use rules_qemu 0.3.0 --- MODULE.bazel | 7 +------ MODULE.bazel.lock | 2 ++ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 9cb1bf9..3b42cce 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,12 +15,7 @@ bazel_dep(name = "squashfs-tools", version = "4.7.5") bazel_dep(name = "zstd", version = "1.5.7.bcr.1") bazel_dep(name = "codesign.bzl", version = "0.0.13") bazel_dep(name = "gawk", version = "5.3.2.bcr.3") -bazel_dep(name = "rules_qemu", version = "0.0.0") -git_override( - module_name = "rules_qemu", - commit = "cc3915b919c2e06e2ac8d07f1205a790ddc7423a", - remote = "https://github.com/hermeticbuild/rules_qemu.git", -) +bazel_dep(name = "rules_qemu", version = "0.3.0") http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 2b4929c..3b6063e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -242,6 +242,8 @@ "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", "https://bcr.bazel.build/modules/rules_python/1.7.0/source.json": "028a084b65dcf8f4dc4f82f8778dbe65df133f234b316828a82e060d81bdce32", + "https://bcr.bazel.build/modules/rules_qemu/0.3.0/MODULE.bazel": "2ddd5524b5a8eeee5fb4049018e99111232206c9c37e2cd063a78190cc18f94f", + "https://bcr.bazel.build/modules/rules_qemu/0.3.0/source.json": "40443bb588205be9bb19455d576579a916f72d3faf5704c9934808581a21da18", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", From ba79fc73878db5f0788d9c2b3f3283e66c051df8 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sat, 13 Jun 2026 15:46:16 -0400 Subject: [PATCH 09/10] linux: use KVM and io_uring --- .github/workflows/ci.yml | 24 +++++++++++--- ARCHITECTURE.md | 5 +-- MODULE.bazel | 2 +- README.md | 5 ++- e2e/README.md | 5 +-- e2e/run_llvm_linux_vm_smoke.sh | 10 +++++- src/linux_vm_host.zig | 1 - src/qemu_vm.zig | 59 ++++++++++++++++++++-------------- tools/e2e.sh | 8 +++++ tools/zig_embedded_qemu.bzl | 1 - 10 files changed, 83 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dccac0f..defd1f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,14 +29,16 @@ jobs: run: .github/workflows/build_release.sh release linux-qemu-llvm-smoke: - name: Linux QEMU LLVM smoke (${{ matrix.architecture }}) + name: Linux QEMU ${{ matrix.mode }} (${{ matrix.architecture }}) strategy: fail-fast: false matrix: include: - architecture: arm64 + mode: build-only runner: ubuntu-24.04-arm - architecture: x86_64 + mode: llvm-smoke runner: ubuntu-24.04 runs-on: ${{ matrix.runner }} timeout-minutes: 240 @@ -45,15 +47,29 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Enable vhost-vsock + - name: Enable KVM and vhost-vsock + if: matrix.mode == 'llvm-smoke' run: | + test -c /dev/kvm + sudo chmod a+rw /dev/kvm if [[ ! -e /dev/vhost-vsock ]]; then sudo modprobe vhost_vsock fi test -c /dev/vhost-vsock sudo chmod a+rw /dev/vhost-vsock + - name: Build Linux ARM64 release + if: matrix.mode == 'build-only' + env: + BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + run: | + bazel build --config=remote \ + --remote_header="x-buildbuddy-api-key=$BUILDBUDDY_API_KEY" \ + -c opt \ + //cmd/linux-actiond:linux-actiond_linux_arm64 + - name: Compare actiond and Linux host + if: matrix.mode == 'llvm-smoke' env: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} ACTIOND_REVISION: ${{ github.event.pull_request.head.sha || github.sha }} @@ -62,10 +78,10 @@ jobs: run: e2e/run_llvm_linux_vm_smoke.sh "$RUNNER_TEMP/actiond-linux-llvm-${{ matrix.architecture }}" - name: Upload Linux LLVM smoke results - if: always() + if: always() && matrix.mode == 'llvm-smoke' uses: actions/upload-artifact@v4 with: - name: linux-qemu-llvm-smoke-${{ matrix.architecture }} + name: linux-qemu-${{ matrix.mode }}-${{ matrix.architecture }} if-no-files-found: warn path: | ${{ runner.temp }}/actiond-linux-llvm-${{ matrix.architecture }}/*.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1870932..429b860 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,8 +61,9 @@ by the `rules_qemu` target toolchain. The x86_64 release also includes `qboot.rom`; QEMU `virt` direct kernel boot on ARM64 does not require firmware. `linux-actiond` creates sealed memfds for QEMU and every immutable embedded VM artifact, then executes QEMU with `execveat`. Only the persistent guest-owned -CAS image is stored under `--root`. The current implementation uses TCG. KVM -and `io_uring` remain follow-up work. +CAS image is stored under `--root`. QEMU uses KVM and host CPU passthrough. The +persistent CAS image uses `cache=none`, `aio=io_uring`, one IOThread, and one +virtio-blk queue per vCPU up to four queues. `linux-actiond-guest` lives in the initramfs. It runs as guest init, mounts the minimal guest filesystems, mounts `/cas` and `/runtimes`, then execs itself as diff --git a/MODULE.bazel b/MODULE.bazel index 3b42cce..3e99752 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -63,7 +63,7 @@ qemu.system_toolchain( target_settings = ["//platforms:qemu_system_aarch64"], ) qemu.system_toolchain( - machine = "microvm,acpi=off,pcie=off,x-option-roms=off", + machine = "microvm,acpi=off,pcie=off,x-option-roms=off,rtc=off", system_target = "x86_64-softmmu", target_settings = ["//platforms:qemu_system_x86_64"], ) diff --git a/README.md b/README.md index 82d8f8c..6fbb86c 100644 --- a/README.md +++ b/README.md @@ -65,9 +65,12 @@ The Linux releases embed the matching QEMU executable from `rules_qemu`. The x86_64 release also embeds `qboot.rom`. `linux-actiond` passes QEMU, the Linux kernel, the initramfs, the runtime SquashFS, and `qboot.rom` through sealed memfds. The guest-owned CAS remains a persistent ext4 file under `--root`. The -current implementation uses TCG and requires `/dev/vhost-vsock`: +QEMU process uses KVM, host CPU passthrough, and `io_uring` for the CAS file. +The host kernel must support `io_uring`. `linux-actiond` requires read/write +access to `/dev/kvm` and `/dev/vhost-vsock`: ```bash +test -r /dev/kvm -a -w /dev/kvm sudo modprobe vhost_vsock ./linux-actiond_linux_x86_64 serve-vm \ --listen=127.0.0.1:8980 \ diff --git a/e2e/README.md b/e2e/README.md index 3b420d8..a0b292f 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -116,5 +116,6 @@ counts, and the actiond-to-host ratio: e2e/run_llvm_linux_vm_smoke.sh ``` -The actiond result includes QEMU TCG overhead and is an end-to-end comparison, -not an executor-only measurement. +The actiond result includes QEMU KVM overhead and is an end-to-end comparison, +not an executor-only measurement. GitHub's hosted ARM64 runner does not expose +KVM, so CI builds the Linux ARM64 release and runs this comparison on x86_64. diff --git a/e2e/run_llvm_linux_vm_smoke.sh b/e2e/run_llvm_linux_vm_smoke.sh index e2dcc03..869735e 100755 --- a/e2e/run_llvm_linux_vm_smoke.sh +++ b/e2e/run_llvm_linux_vm_smoke.sh @@ -32,6 +32,14 @@ if [[ "$#" -gt 1 ]]; then echo "usage: e2e/run_llvm_linux_vm_smoke.sh [output-directory]" >&2 exit 1 fi +if [[ ! -c /dev/kvm || ! -r /dev/kvm || ! -w /dev/kvm ]]; then + echo "Linux LLVM VM smoke requires read/write access to /dev/kvm" >&2 + exit 1 +fi +if [[ ! -c /dev/vhost-vsock || ! -r /dev/vhost-vsock || ! -w /dev/vhost-vsock ]]; then + echo "Linux LLVM VM smoke requires read/write access to /dev/vhost-vsock" >&2 + exit 1 +fi output_root="${1:-${ACTIOND_LLVM_LINUX_SMOKE_ROOT:-$(mktemp -d "${TMPDIR:-/tmp}/actiond-linux-llvm.XXXXXX")}}" target="${ACTIOND_LLVM_SMOKE_TARGET:-@llvm-project//llvm:llvm-tblgen}" warmup_target="${ACTIOND_LLVM_SMOKE_WARMUP_TARGET-//e2e:llvm_exec_warmup}" @@ -337,7 +345,7 @@ cat >"${summary_path}" < max_vcpus or options.memory_mib == 0) return error.InvalidVmConfiguration; + const accelerator = if (std.mem.eql(u8, options.target_arch, "x86_64")) "kvm,kernel-irqchip=on" else "kvm"; + const cas_queue_count = @min(options.cpu_count, max_cas_queues); const guest_cid = options.guest_cid orelse try randomGuestCid(io); const memory = try std.fmt.allocPrint(allocator, "{d}M", .{options.memory_mib}); defer allocator.free(memory); - const cpus = try std.fmt.allocPrint(allocator, "{d}", .{options.cpu_count}); - defer allocator.free(cpus); + const smp = try std.fmt.allocPrint(allocator, "cpus={d},sockets=1,cores={d},threads=1", .{ options.cpu_count, options.cpu_count }); + defer allocator.free(smp); const vsock_device = try std.fmt.allocPrint(allocator, "vhost-vsock-device,id=vsock0,guest-cid={d}", .{guest_cid}); defer allocator.free(vsock_device); - const cas_drive = try driveArg(allocator, "cas", options.cas_image_path, false, .none); + const cas_drive = try driveArg(allocator, "cas", options.cas_image_path, false, .none, .io_uring); defer allocator.free(cas_drive); - const runtime_drive = try driveArg(allocator, "runtimes", options.runtime_image_path, true, .writeback); + const cas_device = try std.fmt.allocPrint(allocator, "virtio-blk-device,drive=cas,iothread=cas-io,num-queues={d}", .{cas_queue_count}); + defer allocator.free(cas_device); + const runtime_drive = try driveArg(allocator, "runtimes", options.runtime_image_path, true, .writeback, .threads); defer allocator.free(runtime_drive); const kernel_append = try kernelAppendArg(allocator, options.target_arch, options.format_cas_image); defer allocator.free(kernel_append); - const machine_option = try std.fmt.allocPrint(allocator, "{s},accel={s}", .{ options.machine, options.accel }); - defer allocator.free(machine_option); - // TODO: Use KVM after the rules_qemu QEMU prebuilt is verified with - // /dev/kvm on the Linux runner. var argv: std.ArrayListUnmanaged([]const u8) = .empty; defer argv.deinit(allocator); try argv.appendSlice(allocator, &.{ @@ -69,16 +71,18 @@ pub const Machine = struct { fexec_argument, options.qemu_system_name, "-machine", - machine_option, + options.machine, + "-accel", + accelerator, "-cpu", - "max", + "host", }); if (options.firmware_path) |path| { try argv.appendSlice(allocator, &.{ "-bios", path }); } try argv.appendSlice(allocator, &.{ "-smp", - cpus, + smp, "-m", memory, "-no-user-config", @@ -98,17 +102,23 @@ pub const Machine = struct { kernel_append, "-device", vsock_device, + "-object", + "iothread,id=cas-io", "-drive", cas_drive, "-device", - "virtio-blk-device,drive=cas", + cas_device, "-drive", runtime_drive, "-device", "virtio-blk-device,drive=runtimes", }); - std.log.info("starting embedded {s} guest_cid={d}", .{ options.qemu_system_name, guest_cid }); + std.log.info("starting embedded {s} with KVM and io_uring guest_cid={d} vcpus={d}", .{ + options.qemu_system_name, + guest_cid, + options.cpu_count, + }); var child = try std.process.spawn(io, .{ .argv = argv.items, .stdin = .ignore, @@ -322,6 +332,7 @@ fn createSealedMemfd(io: std.Io, name: []const u8, bytes: []const u8) !std.posix } const DriveCache = enum { none, writeback }; +const DriveAio = enum { io_uring, threads }; fn driveArg( allocator: std.mem.Allocator, @@ -329,17 +340,17 @@ fn driveArg( path: []const u8, readonly: bool, cache: DriveCache, + aio: DriveAio, ) ![]u8 { const readonly_arg = if (readonly) ",readonly=on" else ""; const escaped_path = try escapeDriveValue(allocator, path); defer allocator.free(escaped_path); - // TODO: Add aio=io_uring after the rules_qemu QEMU prebuilt and Linux - // runner are verified with io_uring. - return std.fmt.allocPrint(allocator, "if=none,id={s},file={s},format=raw{s},cache={s}", .{ + return std.fmt.allocPrint(allocator, "if=none,id={s},file={s},format=raw{s},cache={s},aio={s}", .{ id, escaped_path, readonly_arg, @tagName(cache), + @tagName(aio), }); } @@ -431,10 +442,10 @@ fn closeFd(fd: std.posix.fd_t) void { }; } -test "driveArg leaves io_uring disabled" { - const drive = try driveArg(std.testing.allocator, "cas", "/tmp/cas.ext4", false, .none); +test "driveArg uses io_uring for the CAS image" { + const drive = try driveArg(std.testing.allocator, "cas", "/tmp/cas.ext4", false, .none, .io_uring); defer std.testing.allocator.free(drive); - try std.testing.expectEqualStrings("if=none,id=cas,file=/tmp/cas.ext4,format=raw,cache=none", drive); + try std.testing.expectEqualStrings("if=none,id=cas,file=/tmp/cas.ext4,format=raw,cache=none,aio=io_uring", drive); } test "embedded asset arguments select embedded bytes" { @@ -468,16 +479,16 @@ test "embedded asset arguments select embedded bytes" { } test "driveArg escapes commas in paths" { - const drive = try driveArg(std.testing.allocator, "cas", "/tmp/actiond,vm/cas.ext4", false, .none); + const drive = try driveArg(std.testing.allocator, "cas", "/tmp/actiond,vm/cas.ext4", false, .none, .io_uring); defer std.testing.allocator.free(drive); - try std.testing.expectEqualStrings("if=none,id=cas,file=/tmp/actiond,,vm/cas.ext4,format=raw,cache=none", drive); + try std.testing.expectEqualStrings("if=none,id=cas,file=/tmp/actiond,,vm/cas.ext4,format=raw,cache=none,aio=io_uring", drive); } test "driveArg uses buffered I/O for a read-only memfd" { - const drive = try driveArg(std.testing.allocator, "runtimes", "/proc/self/fd/7", true, .writeback); + const drive = try driveArg(std.testing.allocator, "runtimes", "/proc/self/fd/7", true, .writeback, .threads); defer std.testing.allocator.free(drive); try std.testing.expectEqualStrings( - "if=none,id=runtimes,file=/proc/self/fd/7,format=raw,readonly=on,cache=writeback", + "if=none,id=runtimes,file=/proc/self/fd/7,format=raw,readonly=on,cache=writeback,aio=threads", drive, ); } diff --git a/tools/e2e.sh b/tools/e2e.sh index 85c3981..1c74391 100755 --- a/tools/e2e.sh +++ b/tools/e2e.sh @@ -176,6 +176,14 @@ run_vm_e2e() { server_name="darwin-actiond" ;; Linux) + if [[ ! -c /dev/kvm || ! -r /dev/kvm || ! -w /dev/kvm ]]; then + echo "Linux VM e2e requires read/write access to /dev/kvm" >&2 + return 1 + fi + if [[ ! -c /dev/vhost-vsock || ! -r /dev/vhost-vsock || ! -w /dev/vhost-vsock ]]; then + echo "Linux VM e2e requires read/write access to /dev/vhost-vsock" >&2 + return 1 + fi case "$(uname -m)" in aarch64|arm64) architecture="aarch64" diff --git a/tools/zig_embedded_qemu.bzl b/tools/zig_embedded_qemu.bzl index b1b8963..e30435b 100644 --- a/tools/zig_embedded_qemu.bzl +++ b/tools/zig_embedded_qemu.bzl @@ -11,7 +11,6 @@ def _zig_embedded_qemu_source_impl(ctx): lines = [ 'pub const qemu_system = @embedFile("{}");'.format(qemu_system.basename), 'pub const qemu_system_name = "{}";'.format(qemu.qemu_system.basename), - 'pub const accel = "{}";'.format(qemu.accel), 'pub const machine = "{}";'.format(qemu.machine), 'pub const target_arch = "{}";'.format(qemu.target_arch), ] From d37b232f53981aab042bc594a6479d434bf1e1f3 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sat, 13 Jun 2026 15:46:16 -0400 Subject: [PATCH 10/10] vm: enable CONFIG_KALLSYMS --- vm/linux.config | 1 + vm/linux_x86_64.config | 1 + 2 files changed, 2 insertions(+) diff --git a/vm/linux.config b/vm/linux.config index 65b631f..480dcfa 100644 --- a/vm/linux.config +++ b/vm/linux.config @@ -35,6 +35,7 @@ CONFIG_HYPERV_VMBUS=y CONFIG_HYPERV_VSOCKETS=y CONFIG_INET=y CONFIG_JBD2=y +CONFIG_KALLSYMS=y CONFIG_MEMCG=y CONFIG_MISC_FILESYSTEMS=y CONFIG_MULTIUSER=y diff --git a/vm/linux_x86_64.config b/vm/linux_x86_64.config index 278b73a..4985593 100644 --- a/vm/linux_x86_64.config +++ b/vm/linux_x86_64.config @@ -32,6 +32,7 @@ CONFIG_FS_MBCACHE=y CONFIG_FUTEX=y CONFIG_INET=y CONFIG_JBD2=y +CONFIG_KALLSYMS=y CONFIG_KERNEL_LZ4=y CONFIG_KVM_GUEST=y CONFIG_MEMCG=y