diff --git a/.github/workflows/actiond-production.yaml b/.github/workflows/actiond-production.yaml new file mode 100644 index 0000000..b94e57a --- /dev/null +++ b/.github/workflows/actiond-production.yaml @@ -0,0 +1,78 @@ +name: actiond production VRT + +on: + pull_request: + paths: + - 'experiments/actiond/**' + - 'runtime/**' + - 'internal/**' + - 'playwright/**' + - '.github/workflows/actiond-production.yaml' + +permissions: + contents: read + +jobs: + vrt: + runs-on: ubuntu-24.04 + timeout-minutes: 40 + env: + USE_BAZEL_VERSION: '9.2.0' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: '24' + - uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # 0.19.0 + with: + bazelisk-cache: true + repository-cache: true + disk-cache: actiond-linux-amd64 + - name: Prepare production suites and caller runtime + run: | + corepack enable pnpm + pnpm install --frozen-lockfile + docker pull --platform linux/amd64 mcr.microsoft.com/playwright:v1.63.0-noble@sha256:bc6ab0d6d44ff4826e4cb8c1e6d801e185bfc42bb0753f8e2a30efc70db054c7 + work="$RUNNER_TEMP/actiond-production" + bash experiments/actiond/prepare.sh "$work" + node experiments/actiond/prepare-public.mjs "$work" + - name: Build patched actiond + run: | + work="$RUNNER_TEMP/actiond-production" + git -C "$work/actiond" apply "$GITHUB_WORKSPACE/experiments/actiond/actiond-advice.patch" + git -C "$work/actiond" apply "$GITHUB_WORKSPACE/experiments/actiond/actiond-input-rootfs.patch" + cd "$work/actiond" + bazelisk --output_base="$work/worker-output" build --bes_backend= --remote_executor= --remote_cache= --spawn_strategy=local --jobs=2 //cmd/linux-actiond:linux-actiond_linux_x86_64 > "$work/worker-build.log" 2>&1 + worker=$(bazelisk --output_base="$work/worker-output" cquery --bes_backend= //cmd/linux-actiond:linux-actiond_linux_x86_64 --output=starlark '--starlark:expr=providers(target)["DefaultInfo"].files_to_run.executable.path') + cp "$worker" "$work/actiond-worker" + - name: Enable KVM and vhost-vsock + run: | + test -c /dev/kvm + sudo chmod a+rw /dev/kvm + if [[ ! -e /dev/vhost-vsock ]]; then sudo modprobe vhost_vsock; fi + test -c /dev/vhost-vsock + sudo chmod a+rw /dev/vhost-vsock + - name: Capture and compare in the VM + env: + ACTIOND_BAZEL: bazelisk + run: | + work="$RUNNER_TEMP/actiond-production" + "$work/actiond-worker" serve-vm --root="$work/vm" --listen=127.0.0.1:8980 --memory-mib=6144 --cpus=2 --cas-image-size-mib=4096 > "$work/vm.log" 2>&1 & + worker_pid=$! + trap 'kill "$worker_pid" 2>/dev/null || true' EXIT + ready=false + for attempt in $(seq 1 90); do + kill -0 "$worker_pid" + if (echo > /dev/tcp/127.0.0.1/8980) 2>/dev/null; then ready=true; break; fi + sleep 1 + done + "$ready" + bash experiments/actiond/run-public-actiond.sh "$work" grpc://127.0.0.1:8980 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: actiond-production-vrt + path: | + ${{ runner.temp }}/actiond-production/results/ + ${{ runner.temp }}/actiond-production/vm.log + ${{ runner.temp }}/actiond-production/worker-build.log diff --git a/MODULE.bazel b/MODULE.bazel index 931dd61..88e1a00 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -5,6 +5,8 @@ module( ) bazel_dep(name = "rules_shell", version = "0.8.0") +bazel_dep(name = "platforms", version = "1.1.0") +bazel_dep(name = "rules_python", version = "1.7.0") bazel_dep(name = "aspect_rules_js", version = "3.4.1") bazel_dep(name = "bazel_lib", version = "3.7.2") bazel_dep(name = "aspect_rules_ts", version = "3.10.1") diff --git a/docs/actiond-migration.md b/docs/actiond-migration.md new file mode 100644 index 0000000..6e15930 --- /dev/null +++ b/docs/actiond-migration.md @@ -0,0 +1,93 @@ +# Replace Testcontainers with actiond + +Active implementation plan. The existing Chromium prototype in PR #27 passes +through actiond's Linux amd64 VM after applying `actiond-advice.patch` (upstream +actiond PR #48). Production VRT still uses Testcontainers. + +## Execution contract + +The fixture server, Playwright test runner, Chromium, and screenshot comparison +execute together as a Linux Bazel action. Browser files, libraries, fonts, Node, +test code, assets, and baselines are declared inputs. The executor provides +isolation; no Docker daemon, image pulls, or reaper run inside the action. + +Callers may construct their own OCI image. Packaging must convert its pinned +contents into declared runtime files before execution, including an explicit +browser executable and architecture. Building Chromium from source is optional. +Keep the rule compatible with REAPI rather than depending on actiond's CLI inside +the test runner. + +Baseline capture produces declared downloadable outputs. A local `.update` +wrapper applies successful captures to the source tree using the existing +destination validation. Failed or empty captures must not replace baselines. + +## Implementation sequence + +1. Add declared browser runtime metadata and direct browser launch support. + Exercise the existing gallery and native screenshot runners, not just the + standalone HTML prototype. +2. Package a caller-owned pinned runtime through Bazel, removing the prototype's + manual Docker extraction prerequisite from the supported execution path. +3. Add Linux execution actions for comparison and baseline capture, plus the + local baseline application wrapper. Select amd64 explicitly; an ARM64 worker + must not silently produce shared amd64 baselines. +4. Validate on a patched actiond VM with local fallback disabled. Check artifacts, + baseline updates, failures, timeouts, cancellation, and network isolation. +5. Provide concrete AGI and FormatJS callsite migrations, including built assets, + caller fixture servers, custom Playwright configuration, and native specs. + External services must become declared local fixtures or have an explicitly + documented unsupported migration case; the VM has no external network. +6. Remove Testcontainers, Ryuk, their patches, and Docker-specific VRT plumbing + once the replacement passes these checks. Update examples, CI, and docs. + +## Completion evidence + +- Both `visual_test` and `component_visual_test` compare and update screenshots + through actiond using the production runner. +- Browser runtime acquisition is pinned and separate from offline execution. +- Screenshot outputs and reports survive remote execution and failed tests. +- Baseline application is local, explicit, and refuses failed or empty captures. +- Host E2E/component browser tests continue to work independently. +- AGI and FormatJS migration examples describe the actual supported interface. +- Any necessary actiond changes remain isolated patches with reproductions and + upstream status. The current required patch enables memory-advice syscalls; + no browser-driven relaxation of the action sandbox has been established as + necessary. + +Keep dependent PRs stacked with ordinary Git and descriptions concise. Do not +mark the goal complete based solely on the standalone prototype passing. + +## Current progress + +- Both production VRT modes capture, apply baselines, and compare through the + public rules in actiond's Linux VM (CI run 34779979065). +- Capture jobs generated by the public rules also pass process-isolation tests + with the archive-built runtime, without hand-editing their descriptors. +- Public rules with a declared browser now create remote comparison/capture + actions and local result consumers. Bazel analysis verifies Linux amd64 + constraints and declared runtime paths. Explicit remote mnemonic strategies + and disabled fallback keep these actions on the worker. +- Failed and empty captures preserve source baselines; downloaded failure + reports survive through the local test wrapper. Filesystem and subprocess + regression tests pass. +- Public-rule isolation and deliberate post-capture failure jobs pass under the + actiond process runner: Node gets `ENETUNREACH`, the browser cannot access the + external address, and failed captures retain diagnostic PNGs without exposing + eligible baseline outputs. CI also checks empty captures and mismatched + reference PNGs through the real local commands; VM results remain pending. +- The separate actiond `input-rootfs`/`input-rootfs-env` patch passes actiond's + full build, unit tests, and the public-rule production VM workflow. + The hand-staged VM diagnostic hit an undeclared npm file beneath a nested + Bazel package; CI now runs the public rules and their declared runfiles. +- `browser_runtime_archive` unpacks a caller-produced flattened runtime tar + through a declared Python toolchain. It normalizes image-root links, preserves + executables, and rejects dangling links and unflattened OCI whiteouts. +- `browser_runtime_oci` verifies and applies declared OCI layers offline, then + materializes the selected runtime subtree. Layer/whiteout/hardlink tests pass, + and the real OCI-built runtime matches the archive-built browser, Node, and + fixture font. The CI example now constructs and passes an OCI image target. + +Next: verify the full public-rule workflow in the VM, migrate concrete +AGI/FormatJS callsites, and remove the legacy backend. Baseline application, +failures, cancellation, and isolation still need end-to-end VM coverage through +the public rules. OCI support itself does not complete the migration. diff --git a/docs/actiond-validation.md b/docs/actiond-validation.md new file mode 100644 index 0000000..63f217b --- /dev/null +++ b/docs/actiond-validation.md @@ -0,0 +1,14 @@ +# Actiond runtime validation + +[VM run 34781995883](https://github.com/perplexityai/rules_web_e2e/actions/runs/34781995883) +passes native/component capture and comparison through the public Bazel rules, +network isolation, failed/empty captures, screenshot diffs, and execution deadlines. +The worker uses the two isolated patches in `experiments/actiond`. + +[CI run 34782772531](https://github.com/perplexityai/rules_web_e2e/actions/runs/34782772531) +passes Bazel 8.6 and 9.2 on Linux and macOS, including OCI/archive extraction and +host browser tests. Extraction canonicalizes temporary roots before checking +image links, covering macOS's symlinked temporary directories. + +Native macOS VM execution and cross-architecture screenshot equivalence are not +covered. The backend-removal PR adds cancellation and caller Bazel server coverage. diff --git a/docs/browser-runtime.md b/docs/browser-runtime.md new file mode 100644 index 0000000..76de676 --- /dev/null +++ b/docs/browser-runtime.md @@ -0,0 +1,87 @@ +# Declared browser runtimes + +The actiond migration accepts caller-owned Linux runtime files through +`browser_runtime`. The runtime must contain Chromium, Node, their ELF loader and +shared libraries, and the fonts/fontconfig used for screenshots. Native +Playwright `webServer` commands also need `/bin/sh`. + +A caller can produce a flattened filesystem tar and unpack it during the Bazel +build: + +```starlark +load("@rules_web_e2e//playwright:archive.bzl", "browser_runtime_archive") +load("@rules_web_e2e//playwright:defs.bzl", "browser_runtime") +load("@rules_web_e2e//vrt:defs.bzl", "visual_test") + +browser_runtime_archive( + name = "runtime_files", + archive = ":runtime_tar", +) + +browser_runtime( + name = "browser", + root = ":runtime_files", + executable = "chromium/chrome-headless-shell", + node = "bin/node", + library_dirs = ["lib"], + fontconfig = "etc/fonts", +) + +visual_test( + name = "visuals", + browser = ":browser", + tests = ":compiled_visual_specs", + shell = ":app_shell", + baselines = glob(["__screenshots__/*.png"]), +) +``` + +`:runtime_tar` is a declared file target. It can be built by the caller or fetched +with Bazel's downloader using a pinned checksum. Paths above describe the +prototype's layout; select paths matching the caller's runtime. + +Archive extraction uses a Bazel-provided Python interpreter and makes no network +requests. Absolute image symlinks are resolved within the image root, then links +are materialized into regular files and directories for the output tree. Missing +link targets are errors, so runtime packaging cannot silently borrow host files. +Font configuration should use image paths or paths relative to the configuration +file, rather than a build-machine path. + +For a caller-owned OCI image layout directory, use `browser_runtime_oci` instead: + +```starlark +load("@rules_web_e2e//playwright:archive.bzl", "browser_runtime_oci") + +browser_runtime_oci( + name = "runtime_files", + image = ":caller_image", + directory = "runtime", +) +``` + +`image` supplies one declared OCI layout directory, such as an `oci_image` +output. `directory` selects the runtime subtree after applying layers; use `.` +when the whole image is a closed browser runtime. Keep the `browser_runtime` +paths relative to that selected subtree. OCI extraction verifies SHA-256 blob +digests and sizes, selects one Linux amd64 image, applies layers in order, and +handles whiteouts before same-layer additions. It supports uncompressed and +gzip layers; unsupported layer media types fail explicitly. It never fetches +missing blobs, executes image commands, or contacts a registry. + +An image tar produced by `docker save` or `oci_load` is neither a flattened +filesystem archive nor a layout directory. Pass the image layout target directly. +The runtime subtree must include every link target it needs within the declared +image, and must not rely on Docker injecting files such as `/etc/hosts`. + +Execution currently requires a patched actiond Linux amd64 worker. The remote +actions produce comparison/capture results; the local test command reports their +status and `.update` applies successful captures. Host E2E/component-browser +targets continue to use their existing browser setup. Native and component +capture/comparison through the public rules passed the Linux VM workflow. + +VRT inputs are configured for Linux amd64 even when the Bazel client runs on +macOS. A caller with additional native toolchain constraints can set +`target_platform = "//platforms:linux_x86_64_gnu"` on its visual target. That +platform must still target Linux amd64 and match the runtime's ABI. `data` and +`$(rootpath ...)` expressions in `env` are evaluated in this configuration, +including expressions nested inside JSON strings used by fixture servers. diff --git a/experiments/actiond/BUILD.bazel b/experiments/actiond/BUILD.bazel new file mode 100644 index 0000000..3fb6fa5 --- /dev/null +++ b/experiments/actiond/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_python//python:defs.bzl", "py_binary") + +exports_files(["fixture.bzl"]) + +py_binary( + name = "fixture_image", + srcs = ["fixture_image.py"], + main = "fixture_image.py", +) diff --git a/experiments/actiond/README.md b/experiments/actiond/README.md index 42b1569..1908893 100644 --- a/experiments/actiond/README.md +++ b/experiments/actiond/README.md @@ -79,3 +79,57 @@ or relaxing seccomp, validate an existing editor fixture on the patched VM. Prod backend, reviewed baseline-update handling, amd64 worker selection on Apple Silicon, and cleanup/isolation coverage. One stable fixture is not evidence of cross-architecture pixel equivalence or full Chromium compatibility. + +## Production runner diagnostic + +The `codex/actiond-vrt-runtime` migration adds a diagnostic using the real native +screenshot and component gallery runners. After building `//:native_visual_test` +and `//:component_visual_test` in `examples/react`, run: + +```sh +node experiments/actiond/prepare-production.mjs /tmp/actiond-prototype examples/react/bazel-bin +bash experiments/actiond/run-sandbox.sh /tmp/actiond-prototype --production +``` + +Both suites successfully capture baseline PNGs, then compare against those +captures; JUnit reports are produced for capture and comparison. Results are +downloaded under `results/{native_visual_test,component_visual_test}`. This +diagnostic does not modify source baselines. It exercises the production +`runner.ts`, direct declared Chromium launch, and output-only baseline capture. + +Native Playwright `webServer` commands also require `/bin/sh`. This diagnostic +supplies Bash from the caller image, with the same declared ELF loader and +libraries. The VM integration still needs a declared runtime filesystem layout +for the shell and loader. These new production-runner results are process +sandbox results, not VM/REAPI validation; the earlier VM result above remains +the standalone screenshot fixture. + +## Declared runtime root patch + +`actiond-input-rootfs.patch` is a separate local actiond change based on +`8a42c3d` (local commit `66e2dca`). It adds the `input-rootfs` execution property: +runtime directories from that declared input subtree appear at normal Linux +paths. It replaces injected runtime files for that action and preserves the +executor's device, process, temporary-directory, and network isolation. +actiond's full build and both unit-test targets pass with the patch. macOS VM +execution has not been run. + +`prepare-input-rootfs.sh` restores the image's original Node and Chromium +executables and supplies the loader and shell layout. The production VM workflow +builds actiond with this patch and the separate memory-advice patch, then runs +`run-production-actiond.sh` with local execution fallback disabled. This workflow +is the validation gate for the new rootfs support; local unit tests alone do not +establish VM compatibility. + +The production workflow now uses `prepare-public.mjs` and +`run-public-actiond.sh`: it copies the actual React example, declares the runtime +archive with `browser_runtime_archive`, and runs each public `.update` target +followed by both public test targets. Source baselines are modified only in that +temporary example copy. The native and gallery capture jobs generated by these +public rules both pass under actiond's process isolation. + +The earlier hand-staged VM diagnostic failed because its `glob` omitted an npm +file beneath a nested `BUILD.bazel` package boundary. That failure is not evidence +against the runtime-root patch. The public rules collect the actual declared +runfiles and tree artifacts; their end-to-end VM result remains the validation +gate. The old scripts remain available for reproducing the diagnostic. diff --git a/experiments/actiond/actiond-input-rootfs.patch b/experiments/actiond/actiond-input-rootfs.patch new file mode 100644 index 0000000..3ef9ff6 --- /dev/null +++ b/experiments/actiond/actiond-input-rootfs.patch @@ -0,0 +1,222 @@ +diff --git a/README.md b/README.md +index 4b113c5..691b936 100644 +--- a/README.md ++++ b/README.md +@@ -105,6 +105,29 @@ into a Linux toolchain automatically. + + ## Runtime Selection + ++An action can supply its own runtime as a directory in its REAPI input tree: ++ ++```python ++exec_properties = {"input-rootfs": "path/to/declared/runtime"} ++``` ++ ++When the path is determined during action analysis (for example a Bazel tree ++artifact), use `input-rootfs-env` instead, naming a declared command environment ++variable containing the path. The two properties are mutually exclusive; missing ++or duplicate variables fail. The environment value participates in the action ++digest just like other declared inputs. ++ ++The path is relative to the input root and must traverse directory entries, not ++symlinks. actiond exposes its `bin`, `sbin`, `lib`, `lib64`, `usr`, `etc`, and ++`opt` entries at their usual absolute paths. These entries remain backed by the ++action's declared inputs; actiond does not download or unpack an image. Callers ++must package any OCI image into the input directory before execution. ++ ++This replaces the packaged runtime and common `/etc` files for that action and ++cannot be combined with `libc` or `requires-bash`. `/dev`, `/proc`, `/tmp`, ++`/var/tmp`, and `/workspace` retain the executor's existing behavior. Networking, ++privilege restrictions, and seccomp remain unchanged. ++ + The embedded runtime image currently includes: + + - `glibc2.31` +diff --git a/src/action_executor.zig b/src/action_executor.zig +index 29a208f..f4e2417 100644 +--- a/src/action_executor.zig ++++ b/src/action_executor.zig +@@ -304,6 +304,9 @@ pub fn executeDecodedActionWithOptions( + + const libc_runtime = try libcRuntimeFromPlatform(platform); + const shell_runtime = try shellRuntimeFromPlatform(platform); ++ const input_rootfs = try resolveInputRootfs(platform, command); ++ if (input_rootfs != null and (libc_runtime != null or shell_runtime != null)) ++ return error.ConflictingRuntimeProperties; + var bind_mounts: std.ArrayListUnmanaged(action_runner.BindMount) = .empty; + var borrowed_source_count: usize = 0; + defer { +@@ -313,7 +316,9 @@ pub fn executeDecodedActionWithOptions( + } + bind_mounts.deinit(allocator); + } +- if (options.runtime_mount_cache) |cache| { ++ if (input_rootfs) |rootfs| { ++ try prepareInputRootfs(io, allocator, store, input_root_digest, work_root, rootfs, options.cancellation); ++ } else if (options.runtime_mount_cache) |cache| { + borrowed_source_count = std.math.maxInt(usize); + try checkExecutionCancellation(options.cancellation); + try appendCachedRuntimeMount(io, allocator, work_root, work_root_path, cache.common_etc, "etc", &bind_mounts); +@@ -442,6 +447,75 @@ fn shellRuntimeFromPlatform(platform: ?reapi.Platform) !?[]const u8 { + return null; + } + ++fn inputRootfsFromPlatform(platform: ?reapi.Platform) !?[]const u8 { ++ var result: ?[]const u8 = null; ++ for ((platform orelse return null).properties) |property| { ++ if (!std.mem.eql(u8, property.name, "input-rootfs")) continue; ++ if (result != null) return error.DuplicateInputRootfs; ++ try validatePath(property.value); ++ result = property.value; ++ } ++ return result; ++} ++ ++fn resolveInputRootfs(platform: ?reapi.Platform, command: reapi.Command) !?[]const u8 { ++ var result = try inputRootfsFromPlatform(platform); ++ var found_env = false; ++ for ((platform orelse return result).properties) |property| { ++ if (!std.mem.eql(u8, property.name, "input-rootfs-env")) continue; ++ if (found_env or result != null) return error.ConflictingRuntimeProperties; ++ found_env = true; ++ if (property.value.len == 0) return error.MissingInputRootfsVariable; ++ for (command.environment_variables) |variable| { ++ if (!std.mem.eql(u8, variable.name, property.value)) continue; ++ if (result != null) return error.DuplicateInputRootfs; ++ try validatePath(variable.value); ++ result = variable.value; ++ } ++ if (result == null) return error.MissingInputRootfsVariable; ++ } ++ return result; ++} ++ ++// Link only runtime locations. /dev, /proc, /tmp, /var/tmp and /workspace ++// remain owned by the executor. Targets resolve after chroot, against CAS inputs. ++fn prepareInputRootfs( ++ io: std.Io, ++ allocator: std.mem.Allocator, ++ store: cas.Store, ++ input_root_digest: cas.Digest, ++ work_root: std.Io.Dir, ++ path: []const u8, ++ cancellation: ?*const std.atomic.Value(bool), ++) !void { ++ try validatePath(path); ++ var inputs: OutputParentValidator = .{ ++ .io = io, ++ .allocator = allocator, ++ .store = store, ++ .root_digest = input_root_digest, ++ .cancellation = cancellation, ++ }; ++ defer inputs.deinit(); ++ var digest = input_root_digest; ++ var parts = std.mem.splitScalar(u8, path, '/'); ++ while (parts.next()) |part| { ++ const directory = try inputs.getDirectory(digest); ++ const child = findInputDirectoryEntry(reapi.DirectoryNode, directory.directories, part) orelse ++ return error.InputRootfsNotDirectory; ++ digest = try cas.Digest.fromReapi(child.digest orelse return error.MissingInputDirectoryDigest); ++ } ++ const directory = try inputs.getDirectory(digest); ++ for ([_][]const u8{ "bin", "sbin", "lib", "lib64", "usr", "etc", "opt" }) |name| { ++ try checkExecutionCancellation(cancellation); ++ if (findInputDirectoryEntry(reapi.DirectoryNode, directory.directories, name) == null and ++ findInputDirectoryEntry(reapi.SymlinkNode, directory.symlinks, name) == null) continue; ++ const target = try std.fmt.allocPrint(allocator, "workspace/{s}/{s}", .{ path, name }); ++ defer allocator.free(target); ++ try work_root.symLink(io, target, name, .{ .is_directory = true }); ++ } ++} ++ + fn stressCaseFromCommand(command: reapi.Command) []const u8 { + for (command.environment_variables) |variable| { + if (std.mem.eql(u8, variable.name, "ACTIOND_STRESS_CASE") and variable.value.len != 0) return variable.value; +@@ -2406,6 +2480,86 @@ test "libc runtime platform property accepts pinned runtimes" { + })); + } + ++test "input-rootfs rejects paths outside declared inputs and duplicate properties" { ++ for ([_][]const u8{ "/host", "../host", "root/../host", "root//fs", "root/.", "root\x00fs" }) |path| { ++ try std.testing.expectError(error.EscapingExecPath, inputRootfsFromPlatform(.{ ++ .properties = &.{.{ .name = "input-rootfs", .value = path }}, ++ })); ++ } ++ try std.testing.expectError(error.EmptyExecPath, inputRootfsFromPlatform(.{ ++ .properties = &.{.{ .name = "input-rootfs", .value = "" }}, ++ })); ++ try std.testing.expectError(error.DuplicateInputRootfs, inputRootfsFromPlatform(.{ ++ .properties = &.{ ++ .{ .name = "input-rootfs", .value = "a" }, ++ .{ .name = "input-rootfs", .value = "b" }, ++ }, ++ })); ++} ++ ++test "input-rootfs-env resolves only a unique declared command variable" { ++ const platform: reapi.Platform = .{ .properties = &.{.{ .name = "input-rootfs-env", .value = "RUNTIME" }} }; ++ try std.testing.expectEqualStrings("bazel-out/runtime", (try resolveInputRootfs(platform, .{ ++ .environment_variables = &.{.{ .name = "RUNTIME", .value = "bazel-out/runtime" }}, ++ })).?); ++ try std.testing.expectError(error.MissingInputRootfsVariable, resolveInputRootfs(platform, .{})); ++ try std.testing.expectError(error.EscapingExecPath, resolveInputRootfs(platform, .{ ++ .environment_variables = &.{.{ .name = "RUNTIME", .value = "../host" }}, ++ })); ++ try std.testing.expectError(error.DuplicateInputRootfs, resolveInputRootfs(platform, .{ ++ .environment_variables = &.{ ++ .{ .name = "RUNTIME", .value = "one" }, ++ .{ .name = "RUNTIME", .value = "two" }, ++ }, ++ })); ++ try std.testing.expectError(error.ConflictingRuntimeProperties, resolveInputRootfs(.{ ++ .properties = &.{ ++ .{ .name = "input-rootfs", .value = "literal" }, ++ .{ .name = "input-rootfs-env", .value = "RUNTIME" }, ++ }, ++ }, .{})); ++} ++ ++test "input-rootfs links CAS runtime directories but preserves executor-owned locations" { ++ var tmp = std.testing.tmpDir(.{}); ++ defer tmp.cleanup(); ++ var cas_dir = try tmp.dir.createDirPathOpen(std.testing.io, "cas", .{}); ++ defer cas_dir.close(std.testing.io); ++ var work = try tmp.dir.createDirPathOpen(std.testing.io, "work", .{}); ++ defer work.close(std.testing.io); ++ try prepareChrootBaseDirs(std.testing.io, work); ++ try work.createDir(std.testing.io, "workspace", .default_dir); ++ const store = cas.Store.init(cas_dir); ++ const empty = try putProto(std.testing.io, std.testing.allocator, store, reapi.Directory{}); ++ var empty_hash: [64]u8 = undefined; ++ const rootfs = try putProto(std.testing.io, std.testing.allocator, store, reapi.Directory{ ++ .directories = &.{ ++ .{ .name = "dev", .digest = empty.toReapi(&empty_hash) }, ++ .{ .name = "etc", .digest = empty.toReapi(&empty_hash) }, ++ .{ .name = "proc", .digest = empty.toReapi(&empty_hash) }, ++ .{ .name = "tmp", .digest = empty.toReapi(&empty_hash) }, ++ .{ .name = "usr", .digest = empty.toReapi(&empty_hash) }, ++ .{ .name = "workspace", .digest = empty.toReapi(&empty_hash) }, ++ }, ++ .symlinks = &.{.{ .name = "bin", .target = "usr/bin" }}, ++ }); ++ var rootfs_hash: [64]u8 = undefined; ++ const root = try putProto(std.testing.io, std.testing.allocator, store, reapi.Directory{ ++ .directories = &.{.{ .name = "runtime", .digest = rootfs.toReapi(&rootfs_hash) }}, ++ .symlinks = &.{.{ .name = "alias", .target = "runtime" }}, ++ }); ++ try std.testing.expectError(error.InputRootfsNotDirectory, prepareInputRootfs(std.testing.io, std.testing.allocator, store, root, work, "alias", null)); ++ try std.testing.expectError(error.InputRootfsNotDirectory, prepareInputRootfs(std.testing.io, std.testing.allocator, store, root, work, "missing", null)); ++ try prepareInputRootfs(std.testing.io, std.testing.allocator, store, root, work, "runtime", null); ++ var buffer: [1024]u8 = undefined; ++ const length = try work.readLink(std.testing.io, "bin", &buffer); ++ try std.testing.expectEqualStrings("workspace/runtime/bin", buffer[0..length]); ++ for ([_][]const u8{ "dev", "proc", "tmp", "workspace" }) |name| { ++ var directory = try work.openDir(std.testing.io, name, .{ .follow_symlinks = false }); ++ directory.close(std.testing.io); ++ } ++} ++ + test "execution platform falls back to command platform" { + const platform = executionPlatform( + .{}, diff --git a/experiments/actiond/capture.bzl b/experiments/actiond/capture.bzl index 8c82d7d..b2bafbf 100644 --- a/experiments/actiond/capture.bzl +++ b/experiments/actiond/capture.bzl @@ -44,3 +44,31 @@ kernel_probe = rule( implementation = _kernel_probe_impl, attrs = {"binary": attr.label(allow_single_file = True)}, ) + +def _production_impl(ctx): + out = ctx.actions.declare_directory("production-results") + ctx.actions.run( + executable = ctx.file.node, + arguments = [ctx.file.script.path], + inputs = depset(ctx.files.inputs + [ctx.file.script]), + outputs = [out], + env = { + "HOME": "/tmp", + "TMPDIR": "/tmp", + "LANG": "C.UTF-8", + "TZ": "UTC", + "LD_LIBRARY_PATH": "/workspace/runtime/lib", + "OUTPUT_DIR": out.path, + }, + mnemonic = "ActiondProductionVrt", + ) + return [DefaultInfo(files = depset([out]))] + +production = rule( + implementation = _production_impl, + attrs = { + "node": attr.label(allow_single_file = True), + "script": attr.label(allow_single_file = True), + "inputs": attr.label_list(allow_files = True), + }, +) diff --git a/experiments/actiond/failure.visual.spec.ts b/experiments/actiond/failure.visual.spec.ts new file mode 100644 index 0000000..1920e03 --- /dev/null +++ b/experiments/actiond/failure.visual.spec.ts @@ -0,0 +1,9 @@ +import {expect, test} from '@playwright/test' + +test('a failure after capture must not apply partial baselines', async ({page}) => { + test.setTimeout(120_000) + await page.goto('/') + await expect(page.getByRole('button', {name: 'Save', exact: true})).toHaveScreenshot('partial.png') + if (process.env.ACTIOND_HANG) await new Promise(() => {}) + throw new Error('Intentional failure after screenshot capture') +}) diff --git a/experiments/actiond/fixture.bzl b/experiments/actiond/fixture.bzl new file mode 100644 index 0000000..c49f357 --- /dev/null +++ b/experiments/actiond/fixture.bzl @@ -0,0 +1,21 @@ +"""Build a caller-owned OCI fixture from the example's declared runtime tar.""" + +def _image_impl(ctx): + image = ctx.actions.declare_directory(ctx.label.name) + ctx.actions.run( + executable = ctx.executable._tool, + arguments = [ctx.file.archive.path, image.path], + inputs = [ctx.file.archive], + tools = [ctx.attr._tool[DefaultInfo].files_to_run], + outputs = [image], + mnemonic = "VrtFixtureImage", + ) + return [DefaultInfo(files = depset([image]))] + +runtime_image = rule( + implementation = _image_impl, + attrs = { + "archive": attr.label(mandatory = True, allow_single_file = True), + "_tool": attr.label(default = Label("//experiments/actiond:fixture_image"), executable = True, cfg = "exec"), + }, +) diff --git a/experiments/actiond/fixture_image.py b/experiments/actiond/fixture_image.py new file mode 100644 index 0000000..cf25ac2 --- /dev/null +++ b/experiments/actiond/fixture_image.py @@ -0,0 +1,31 @@ +"""Package the fixture's uncompressed runtime tar as a single-layer OCI image.""" +import hashlib +import json +from pathlib import Path +import shutil +import sys + +archive, output = map(Path, sys.argv[1:]) +blobs = output / "blobs/sha256" +blobs.mkdir(parents=True) +with archive.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() +shutil.copyfile(archive, blobs / digest) +layer = {"digest": "sha256:" + digest, "size": archive.stat().st_size, + "mediaType": "application/vnd.oci.image.layer.v1.tar"} + + +def put(value, media): + data = json.dumps(value, sort_keys=True).encode() + digest = hashlib.sha256(data).hexdigest() + (blobs / digest).write_bytes(data) + return {"digest": "sha256:" + digest, "size": len(data), "mediaType": media} + + +config = put({"os": "linux", "architecture": "amd64", + "rootfs": {"type": "layers", "diff_ids": [layer["digest"]]}}, + "application/vnd.oci.image.config.v1+json") +manifest = put({"schemaVersion": 2, "config": config, "layers": [layer]}, + "application/vnd.oci.image.manifest.v1+json") +(output / "oci-layout").write_text('{"imageLayoutVersion":"1.0.0"}') +(output / "index.json").write_text(json.dumps({"schemaVersion": 2, "manifests": [manifest]})) diff --git a/experiments/actiond/isolation.visual.spec.ts b/experiments/actiond/isolation.visual.spec.ts new file mode 100644 index 0000000..3ea2d56 --- /dev/null +++ b/experiments/actiond/isolation.visual.spec.ts @@ -0,0 +1,27 @@ +import {expect, test} from '@playwright/test' +import fs from 'node:fs' +import net from 'node:net' + +test('the whole VRT action is offline and can still serve its fixture', async ({page}) => { + expect(process.platform).toBe('linux') + expect(process.arch).toBe('x64') + expect(fs.existsSync('/var/run/docker.sock')).toBe(false) + const fixture = JSON.parse(process.env.ACTIOND_FIXTURE!) + expect(JSON.parse(fs.readFileSync(fixture.package, 'utf8')).name).toBeTruthy() + const error = await new Promise(resolve => { + const socket = net.connect({host: '1.1.1.1', port: 443}) + socket.once('connect', () => { + socket.destroy() + resolve(new Error('External connection unexpectedly succeeded')) + }) + socket.once('error', resolve) + socket.setTimeout(2000, () => { + socket.destroy() + resolve(new Error('Connection timed out instead of being isolated')) + }) + }) + expect(error.code).toBe('ENETUNREACH') + await page.goto('/') + await expect(page.getByRole('button', {name: 'Save', exact: true})).toBeVisible() + await expect(page.goto('http://1.1.1.1/', {timeout: 2000})).rejects.toThrow() +}) diff --git a/experiments/actiond/package-runtime.sh b/experiments/actiond/package-runtime.sh index 138e0be..40baba5 100755 --- a/experiments/actiond/package-runtime.sh +++ b/experiments/actiond/package-runtime.sh @@ -2,7 +2,7 @@ # Setup-only prototype: run inside the pinned Playwright image. set -euo pipefail out=${1:-/output} -mkdir -p "$out"/{bin,lib,chromium,etc/fonts,fonts} +mkdir -p "$out"/{bin,lib,lib64,chromium,etc/fonts,fonts} cp /usr/bin/node /bin/bash "$out/bin/" cp -a /ms-playwright/chromium_headless_shell-1243/chrome-headless-shell-linux64/. "$out/chromium/" for binary in /usr/bin/node /bin/bash /ms-playwright/chromium_headless_shell-1243/chrome-headless-shell-linux64/chrome-headless-shell; do @@ -11,12 +11,14 @@ for binary in /usr/bin/node /bin/bash /ms-playwright/chromium_headless_shell-124 done done cp -L /lib64/ld-linux-x86-64.so.2 "$out/lib/" -cp -a /usr/share/fonts/. "$out/fonts/" +cp -L /lib64/ld-linux-x86-64.so.2 "$out/lib64/" +ln -s bash "$out/bin/sh" +cp -aL /usr/share/fonts/. "$out/fonts/" cat > "$out/etc/fonts/fonts.conf" <<'XML' - /workspace/runtime/fonts + ../../fonts /tmp/fontconfig XML diff --git a/experiments/actiond/prepare-input-rootfs.sh b/experiments/actiond/prepare-input-rootfs.sh new file mode 100644 index 0000000..ee9a35d --- /dev/null +++ b/experiments/actiond/prepare-input-rootfs.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Restore original ELF interpreters and expose the minimal image runtime layout. +set -euo pipefail +work=${1:?usage: prepare-input-rootfs.sh ABSOLUTE_WORK_DIRECTORY} +[[ $work = /* ]] || exit 1 +tar -xf "$work/runtime.tar" -C "$work/workspace/runtime" +mkdir -p "$work/workspace/runtime/lib64" +cp "$work/workspace/runtime/lib/ld-linux-x86-64.so.2" "$work/workspace/runtime/lib64/" +ln -sf bash "$work/workspace/runtime/bin/sh" diff --git a/experiments/actiond/prepare-production.mjs b/experiments/actiond/prepare-production.mjs new file mode 100644 index 0000000..b32a114 --- /dev/null +++ b/experiments/actiond/prepare-production.mjs @@ -0,0 +1,48 @@ +// Diagnostic preparation only: stage the actual Bazel-built production runners. +import fs from 'node:fs' +import path from 'node:path' +import {pathToFileURL} from 'node:url' + +const [work, bazelBin] = process.argv.slice(2).map(value => path.resolve(value)) +if (!work || !bazelBin) throw new Error('usage: prepare-production.mjs WORK BAZEL_BIN') +const {stageRunfiles} = await import(pathToFileURL(path.join(bazelBin, 'external/rules_web_e2e+/runtime/isolation.js'))) +for (const target of ['native_visual_test', 'component_visual_test']) { + const directory = path.join(work, 'workspace', target) + fs.rmSync(directory, {recursive: true, force: true}) + fs.mkdirSync(directory, {recursive: true}) + stageRunfiles(path.join(bazelBin, `${target}_/${target}.runfiles_manifest`), directory) + const descriptorPath = path.join(directory, `_main/${target}_inputs.json`) + const descriptor = JSON.parse(fs.readFileSync(descriptorPath, 'utf8')) + descriptor.browser = { + root: '_runtime', executable: 'chromium/chrome-headless-shell', node: 'bin/node', + libraryDirs: ['lib'], fontconfig: 'etc/fonts', arch: 'x64', + } + fs.chmodSync(descriptorPath, 0o644) + fs.writeFileSync(descriptorPath, JSON.stringify(descriptor)) + const manifest = [] + const visit = relative => { + const file = path.join(directory, relative) + const stat = fs.lstatSync(file) + if (stat.isDirectory()) { + for (const name of fs.readdirSync(file)) visit(path.join(relative, name)) + } else { + manifest.push(`${relative} ${stat.isSymbolicLink() ? fs.readlinkSync(file) : `/workspace/${target}/${relative}`}`) + } + } + visit('') + manifest.push('_runtime /workspace/runtime') + fs.writeFileSync(path.join(directory, 'MANIFEST'), manifest.join('\n') + '\n') +} +fs.copyFileSync(new URL('./production-smoke.mjs', import.meta.url), path.join(work, 'workspace', 'production-smoke.mjs')) +fs.copyFileSync(new URL('./capture.bzl', import.meta.url), path.join(work, 'workspace', 'capture.bzl')) +const build = path.join(work, 'workspace', 'BUILD.bazel') +const original = fs.readFileSync(new URL('./BUILD.bazel.template', import.meta.url), 'utf8') +fs.writeFileSync(build, 'load(":capture.bzl", "production")\n' + original + ` +production( + name = "production", + node = "runtime/bin/node", + script = "production-smoke.mjs", + inputs = glob(["runtime/**", "native_visual_test/**", "component_visual_test/**"]), + exec_properties = {"input-rootfs": "runtime"}, +) +`) diff --git a/experiments/actiond/prepare-public.mjs b/experiments/actiond/prepare-public.mjs new file mode 100644 index 0000000..d3b9da4 --- /dev/null +++ b/experiments/actiond/prepare-public.mjs @@ -0,0 +1,88 @@ +// Copy the real example workspace so .update can be tested without source edits. +import fs from 'node:fs' +import path from 'node:path' +import {fileURLToPath} from 'node:url' + +const work = path.resolve(process.argv[2]) +const repository = fileURLToPath(new URL('../../', import.meta.url)) +const example = path.join(repository, 'examples/react') +const destination = path.join(work, 'public') +fs.mkdirSync(destination, {recursive: true}) +for (const name of fs.readdirSync(example)) { + if (name === 'node_modules' || name.startsWith('bazel-') || name.startsWith('.')) continue + fs.cpSync(path.join(example, name), path.join(destination, name), {recursive: true}) +} +const module = fs.readFileSync(path.join(destination, 'MODULE.bazel'), 'utf8') +fs.writeFileSync(path.join(destination, 'MODULE.bazel'), module.replace('path = "../.."', `path = ${JSON.stringify(repository)}`)) +fs.copyFileSync(path.join(work, 'runtime.tar'), path.join(destination, 'runtime.tar')) +for (const name of ['isolation', 'failure']) + fs.copyFileSync(new URL(`./${name}.visual.spec.ts`, import.meta.url), path.join(destination, `actiond-${name}.visual.spec.ts`)) +const build = path.join(destination, 'BUILD.bazel') +fs.writeFileSync(build, + 'load("@rules_web_e2e//playwright:archive.bzl", "browser_runtime_oci")\n' + + 'load("@rules_web_e2e//experiments/actiond:fixture.bzl", "runtime_image")\n' + + 'load("@rules_web_e2e//playwright:defs.bzl", "browser_runtime")\n' + + fs.readFileSync(build, 'utf8') + ` +runtime_image(name = "actiond_image", archive = "runtime.tar") +browser_runtime_oci(name = "actiond_runtime_files", image = ":actiond_image") +browser_runtime( + name = "actiond_browser", + root = ":actiond_runtime_files", + executable = "chromium/chrome-headless-shell", + node = "bin/node", + library_dirs = ["lib"], + fontconfig = "etc/fonts", +) +visual_test( + name = "actiond_native_test", + browser = ":actiond_browser", + config = ":native_config", + tests = ":native_visual_specs", + baseline_dir = "__actiond_native__", + baselines = glob(["__actiond_native__/*.png"], allow_empty = True), +) +component_visual_test( + name = "actiond_gallery_test", + browser = ":actiond_browser", + shell = ":component_shell", + matching = ":matching", + baseline_dir = "__actiond_gallery__", + baselines = glob(["__actiond_gallery__/*.png"], allow_empty = True), +) +js_library( + name = "actiond_isolation_specs", + srcs = ["actiond-isolation.visual.spec.js"], + deps = [":typecheck_project"], +) +js_library( + name = "actiond_failure_specs", + srcs = ["actiond-failure.visual.spec.js"], + deps = [":typecheck_project"], +) +visual_test( + name = "actiond_isolation_test", + browser = ":actiond_browser", + config = ":native_config", + tests = ":actiond_isolation_specs", + baseline_dir = "__actiond_isolation__", + data = ["package.json"], + env = {"ACTIOND_FIXTURE": json.encode({"package": "$(rootpath package.json)"})}, +) +visual_test( + name = "actiond_failure_test", + browser = ":actiond_browser", + config = ":native_config", + tests = ":actiond_failure_specs", + baseline_dir = "__actiond_failed__", + baselines = glob(["__actiond_failed__/*.png"], allow_empty = True), +) +visual_test( + name = "actiond_timeout_test", + browser = ":actiond_browser", + config = ":native_config", + tests = ":actiond_failure_specs", + baseline_dir = "__actiond_timeout__", + env = {"ACTIOND_HANG": "1"}, + execution_timeout_seconds = 8, +) +`) diff --git a/experiments/actiond/prepare.sh b/experiments/actiond/prepare.sh index 4528a8e..2c090fb 100755 --- a/experiments/actiond/prepare.sh +++ b/experiments/actiond/prepare.sh @@ -22,11 +22,14 @@ printf 'pub const executor_timing_logs = false;\npub const actiondfs_fstype: [:0 ZIG_GLOBAL_CACHE_DIR="$work/zig-cache" "$work/zig-x86_64-linux-0.16.0/zig" build-exe -O ReleaseFast -target x86_64-linux-musl \ --dep actiond_build_options -Mroot="$work/build/sandbox-smoke.zig" \ -Mactiond_build_options="$work/build/options.zig" -femit-bin="$work/build/sandbox-smoke" -if [[ ! -f $work/runtime.tar ]]; then +runtime_image=mcr.microsoft.com/playwright:v1.63.0-noble@sha256:bc6ab0d6d44ff4826e4cb8c1e6d801e185bfc42bb0753f8e2a30efc70db054c7 +runtime_key=$({ printf '%s\n' "$runtime_image"; cat "$here/package-runtime.sh"; } | sha256sum | cut -d ' ' -f 1) +if [[ ! -f $work/runtime.tar || $(cat "$work/runtime.key" 2>/dev/null || true) != "$runtime_key" ]]; then docker run --rm -i --network none --pull=never \ - mcr.microsoft.com/playwright:v1.63.0-noble@sha256:bc6ab0d6d44ff4826e4cb8c1e6d801e185bfc42bb0753f8e2a30efc70db054c7 \ + "$runtime_image" \ bash -s /output < "$here/package-runtime.sh" > "$work/runtime.tar.tmp" mv "$work/runtime.tar.tmp" "$work/runtime.tar" + printf '%s\n' "$runtime_key" > "$work/runtime.key" fi mkdir -p "$work/workspace/runtime" tar -xf "$work/runtime.tar" -C "$work/workspace/runtime" diff --git a/experiments/actiond/production-smoke.mjs b/experiments/actiond/production-smoke.mjs new file mode 100644 index 0000000..58085bc --- /dev/null +++ b/experiments/actiond/production-smoke.mjs @@ -0,0 +1,58 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import {spawnSync} from 'node:child_process' +import path from 'node:path' + +for (const [target, mode, baselines] of [ + ['native_visual_test', 'visual-spec', '__native_screenshots__'], + ['component_visual_test', 'visual', '__component_screenshots__'], +]) { + const runfiles = `/tmp/${target}` + fs.cpSync(`/workspace/${target}`, runfiles, {recursive: true}) + const output = path.join(path.resolve(process.env.OUTPUT_DIR || '/workspace/outputs'), target) + const captured = `${output}/update/baselines` + for (const update of [true, false]) { + const resultDirectory = `${output}/${update ? 'update' : 'compare'}` + const artifacts = `${resultDirectory}/artifacts` + fs.mkdirSync(artifacts, {recursive: true}) + const result = spawnSync('/workspace/runtime/bin/node', [ + `${runfiles}/rules_web_e2e+/runtime/runner.js`, ...(update ? ['--update'] : []), + ], { + stdio: 'inherit', + env: { + ...process.env, + RUNFILES_DIR: runfiles, + RUNFILES_MANIFEST_FILE: `${runfiles}/MANIFEST`, + JS_BINARY__NODE_BINARY: '/workspace/runtime/bin/node', + VRT_DESCRIPTOR: `_main/${target}_inputs.json`, + VRT_BASE_URL: '', VRT_BASE_URL_ENV: '', VRT_MODE: mode, + VRT_BASELINE_RELATIVE: baselines, + VRT_NETWORK_ORIGINS: '[]', VRT_NETWORK_ORIGINS_ENV: '[]', + VRT_ENV_NAMES: '[]', VRT_TIMEOUT_MS: '60000', + VRT_CAPTURE_OUTPUT: captured, + TEST_UNDECLARED_OUTPUTS_DIR: artifacts, + }, + }) + fs.writeFileSync(`${resultDirectory}/result.json`, JSON.stringify({ + schemaVersion: 1, + mode: update ? 'capture' : 'compare', + exitCode: result.status ?? 1, + })) + assert.ifError(result.error) + assert.equal(result.status, 0, `${target} ${update ? 'capture' : 'compare'} failed`) + assert.ok(fs.existsSync(`${artifacts}/junit.xml`)) + if (update) { + const images = fs.readdirSync(captured).filter(name => name.endsWith('.png')) + assert.ok(images.length > 0, 'capture must return PNGs') + // Feed downloaded captures back as declared baseline inputs to comparison. + const baselineInputs = `${runfiles}/_main/${baselines}` + fs.rmSync(baselineInputs, {recursive: true, force: true}) + fs.mkdirSync(baselineInputs, {recursive: true}) + for (const name of images) fs.copyFileSync(`${captured}/${name}`, `${baselineInputs}/${name}`) + const manifest = fs.readFileSync(`${runfiles}/MANIFEST`, 'utf8').split('\n') + .filter(line => !line.startsWith(`_main/${baselines}/`)) + for (const name of images) manifest.push(`_main/${baselines}/${name} ${baselineInputs}/${name}`) + fs.writeFileSync(`${runfiles}/MANIFEST`, manifest.filter(Boolean).join('\n') + '\n') + } + } +} diff --git a/experiments/actiond/run-production-actiond.sh b/experiments/actiond/run-production-actiond.sh new file mode 100644 index 0000000..4fcc842 --- /dev/null +++ b/experiments/actiond/run-production-actiond.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +work=${1:?usage: run-production-actiond.sh ABSOLUTE_WORK_DIRECTORY ENDPOINT} +endpoint=${2:?missing actiond endpoint} +cd "$work/workspace" +"${ACTIOND_BAZEL:-bazelisk}" --output_base="$work/production-bazel-output" build //:production \ + --host_platform=//:linux_amd64 --platforms=//:linux_amd64 \ + --remote_executor="$endpoint" --remote_cache="$endpoint" \ + --spawn_strategy=remote --remote_local_fallback=false \ + --remote_upload_local_results=false --noremote_cache_compression \ + --remote_download_outputs=all +mkdir -p "$work/results/production" +cp -R bazel-bin/production-results/. "$work/results/production/" diff --git a/experiments/actiond/run-public-actiond.sh b/experiments/actiond/run-public-actiond.sh new file mode 100644 index 0000000..b77e14a --- /dev/null +++ b/experiments/actiond/run-public-actiond.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail +work=${1:?usage: run-public-actiond.sh ABSOLUTE_WORK_DIRECTORY ENDPOINT} +endpoint=${2:?missing actiond endpoint} +cd "$work/public" +collect() { + mkdir -p "$work/results/public" + for item in __actiond_native__ __actiond_gallery__ __actiond_failed__ __actiond_isolation__ bazel-testlogs/actiond_*_test bazel-bin/actiond_*_test_*.results; do + if [[ -e $item ]]; then cp -RL "$item" "$work/results/public/"; fi + done +} +trap collect EXIT +flags=( + --jobs=2 + --remote_executor="$endpoint" --remote_cache="$endpoint" + --spawn_strategy=sandboxed,local --strategy=VrtCapture=remote --strategy=VrtCompare=remote + --remote_local_fallback=false --remote_upload_local_results=false + --noremote_cache_compression --remote_download_outputs=all +) +bazel_cmd=("${ACTIOND_BAZEL:-bazelisk}" --output_base="$work/public-bazel-output") +"${bazel_cmd[@]}" run //:actiond_native_test.update "${flags[@]}" +"${bazel_cmd[@]}" run //:actiond_gallery_test.update "${flags[@]}" +test -s __actiond_native__/saved.png +test -s __actiond_gallery__/counter.png +"${bazel_cmd[@]}" test //:actiond_native_test //:actiond_gallery_test //:actiond_isolation_test "${flags[@]}" --test_output=errors + +# A successful suite with no screenshots must not erase existing references. +mkdir -p __actiond_isolation__ +cp __actiond_native__/saved.png __actiond_isolation__/keep.png +if "${bazel_cmd[@]}" run //:actiond_isolation_test.update "${flags[@]}"; then + echo 'Expected an empty capture to be rejected' >&2 + exit 1 +fi +cmp __actiond_native__/saved.png __actiond_isolation__/keep.png +python3 - <<'PY' +import json +from pathlib import Path +result = json.loads(Path('bazel-bin/actiond_isolation_test_capture.results/result.json').read_text()) +assert result['mode'] == 'capture' and result['exitCode'] != 0, result +PY + +# Failed captures produce artifacts but must not modify any source baseline. +mkdir -p __actiond_failed__ +cp __actiond_native__/saved.png __actiond_failed__/keep.png +if "${bazel_cmd[@]}" run //:actiond_failure_test.update "${flags[@]}"; then + echo 'Expected the deliberate capture failure' >&2 + exit 1 +fi +cmp __actiond_native__/saved.png __actiond_failed__/keep.png +test ! -e __actiond_failed__/partial.png +test -s bazel-bin/actiond_failure_test_capture.results/artifacts/reference/partial.png +test -s bazel-bin/actiond_failure_test_capture.results/artifacts/junit.xml +python3 - <<'PY' +import json +from pathlib import Path +result = json.loads(Path('bazel-bin/actiond_failure_test_capture.results/result.json').read_text()) +assert result['mode'] == 'capture' and result['exitCode'] != 0, result +PY + +# The rule deadline terminates a stuck suite and preserves its partial capture. +mkdir -p __actiond_timeout__ +cp __actiond_native__/saved.png __actiond_timeout__/keep.png +if "${bazel_cmd[@]}" run //:actiond_timeout_test.update "${flags[@]}"; then + echo 'Expected the stuck suite to time out' >&2 + exit 1 +fi +cmp __actiond_native__/saved.png __actiond_timeout__/keep.png +test -s bazel-bin/actiond_timeout_test_capture.results/artifacts/reference/partial.png +python3 - <<'PY' +import json +from pathlib import Path +result = json.loads(Path('bazel-bin/actiond_timeout_test_capture.results/result.json').read_text()) +assert result['mode'] == 'capture' and result['exitCode'] != 0, result +PY + +# A mismatched reference must fail the local test, with downloaded diff images. +cp __actiond_native__/saved.png "$work/native-baseline.png" +cp __actiond_gallery__/counter.png __actiond_native__/saved.png +if "${bazel_cmd[@]}" test //:actiond_native_test "${flags[@]}" --test_output=errors; then + echo 'Expected screenshot comparison to fail' >&2 + exit 1 +fi +python3 - <<'PY' +import json +from pathlib import Path +directory = Path('bazel-bin/actiond_native_test_compare.results') +result = json.loads((directory / 'result.json').read_text()) +assert result['mode'] == 'compare' and result['exitCode'] != 0, result +assert list((directory / 'artifacts').rglob('*-diff.png')), 'Missing screenshot diff' +PY +cp "$work/native-baseline.png" __actiond_native__/saved.png diff --git a/experiments/actiond/run-sandbox.sh b/experiments/actiond/run-sandbox.sh index f80e700..3f71031 100755 --- a/experiments/actiond/run-sandbox.sh +++ b/experiments/actiond/run-sandbox.sh @@ -8,11 +8,22 @@ mkdir -p "$root"/{dev,proc,tmp,var/tmp,workspace/outputs} "$work/results" touch "$root/dev/null" chmod 1777 "$root/tmp" "$root/var/tmp" "$root/workspace/outputs" cp -a "$work/workspace/runtime" "$work/workspace/playwright-core" "$work/workspace/capture.mjs" "$work/workspace/ld.so" "$root/workspace/" +script=/workspace/capture.mjs +if [[ ${1:-} == --production ]]; then + shift + script=/workspace/production-smoke.mjs + cp -a "$work/workspace/native_visual_test" "$work/workspace/component_visual_test" "$work/workspace/production-smoke.mjs" "$root/workspace/" + # Native Playwright webServer commands invoke /bin/sh. The diagnostic supplies + # the caller image's Bash; production VM integration must declare this too. + mkdir -p "$root/bin" + cp "$work/workspace/runtime/bin/bash" "$root/bin/sh" + python3 "$(dirname "$0")/patch-interpreter.py" "$root/bin/sh" +fi rm -f "$root/workspace/outputs/"*.png container=$(docker create --network none --cap-add SYS_ADMIN --cap-add NET_ADMIN \ --security-opt seccomp=unconfined --security-opt apparmor=unconfined \ --pull=never ubuntu:24.04@sha256:224a1869083a311ef3f13648a154ba79832fbef6364d31493642ca03082da254 \ - /bin/bash -c 'mkdir /cas; exec /smoke /action-root /cas /workspace/runtime/bin/node /workspace/capture.mjs "$@"' prototype "$@") + /bin/bash -c 'mkdir /cas; exec /smoke /action-root /cas /workspace/runtime/bin/node "$@"' prototype "$script" "$@") trap 'docker rm -f "$container" >/dev/null' EXIT docker cp "$root" "$container:/action-root" docker cp "$work/build/sandbox-smoke" "$container:/smoke" diff --git a/internal/BUILD.bazel b/internal/BUILD.bazel index ff6ba80..4a63176 100644 --- a/internal/BUILD.bazel +++ b/internal/BUILD.bazel @@ -1 +1,7 @@ -exports_files(["browser.bzl"]) +exports_files(["browser.bzl", "remote.bzl"]) + +platform( + name = "linux_amd64", + constraint_values = ["@platforms//os:linux", "@platforms//cpu:x86_64"], + visibility = ["//visibility:public"], +) diff --git a/internal/browser.bzl b/internal/browser.bzl index be44b3f..506fe6a 100644 --- a/internal/browser.bzl +++ b/internal/browser.bzl @@ -1,7 +1,8 @@ """Execute compiled browser inputs with a reusable Playwright runtime.""" load("@aspect_rules_js//js:defs.bzl", "js_binary", "js_library", "js_test") -load("//playwright:defs.bzl", "PlaywrightInfo", "runfile", _PLAYWRIGHT_IMAGE = "PLAYWRIGHT_IMAGE") +load("//playwright:defs.bzl", "BrowserRuntimeInfo", "PlaywrightInfo", "runfile", _PLAYWRIGHT_IMAGE = "PLAYWRIGHT_IMAGE") +load(":remote.bzl", "remote_browser_test") PLAYWRIGHT_IMAGE = _PLAYWRIGHT_IMAGE ShellInfo = provider(fields = ["directory", "entry_point"]) @@ -43,7 +44,7 @@ def _inputs_impl(ctx): fail("tests must supply compiled JavaScript specs") runtime = ctx.attr.playwright[PlaywrightInfo] visual = ctx.attr.mode in ["visual", "visual-spec"] - if visual and not runtime.images: + if visual and not ctx.attr.browser and not runtime.images: fail("VRT requires a matching digest-pinned browser image on playwright_runtime") shell = ctx.attr.shell[ShellInfo] if ctx.attr.shell else None result = ctx.actions.declare_file(ctx.label.name + ".json") @@ -54,9 +55,10 @@ def _inputs_impl(ctx): "server": _compiled(ctx.attr.server, "server"), "shell": {"directory": shell.directory, "entryPoint": shell.entry_point} if shell else None, "playwright": {"test": runtime.test, "core": runtime.core, "version": runtime.version, "images": runtime.images if visual else []}, + "browser": ctx.attr.browser[BrowserRuntimeInfo].descriptor if ctx.attr.browser else None, })) inputs = ctx.runfiles(files = [result]) - for target in [ctx.attr.tests, ctx.attr.config, ctx.attr.matching, ctx.attr.server, ctx.attr.shell, ctx.attr.playwright, ctx.attr.sources]: + for target in [ctx.attr.tests, ctx.attr.config, ctx.attr.matching, ctx.attr.server, ctx.attr.shell, ctx.attr.playwright, ctx.attr.browser, ctx.attr.sources]: if target: inputs = inputs.merge(target[DefaultInfo].default_runfiles) inputs = inputs.merge(ctx.runfiles(transitive_files = target[DefaultInfo].files)) @@ -73,6 +75,7 @@ _inputs = rule( "server": attr.label(allow_files = True), "shell": attr.label(providers = [ShellInfo]), "playwright": attr.label(providers = [PlaywrightInfo]), + "browser": attr.label(providers = [BrowserRuntimeInfo]), "sources": attr.label(), "mode": attr.string(), }, @@ -86,6 +89,8 @@ def browser_test( base_url = None, base_url_env = None, playwright = Label("//runtime:playwright"), + browser = None, + target_platform = Label("//internal:linux_amd64"), config = None, matching = None, baselines = [], @@ -106,6 +111,14 @@ def browser_test( fail("VRT_* environment names are reserved for the browser runtime") if visual and component: fail("Visual and component modes are separate targets") + if browser and not visual: + fail("browser is only supported for VRT; other browser tests use host browsers") + if browser and (network_origins or network_origins_env): + fail("Declared browser actions have loopback-only networking; supply local fixture servers") + if browser and env_inherit: + fail("Remote VRT requires explicit env values instead of env_inherit") + if browser and base_url_env and base_url_env not in env: + fail("Remote VRT base_url_env must have an explicit env value") if not visual and (network_origins or network_origins_env): fail("network_origins and network_origins_env are VRT-only; host browsers use the host network") if not visual and matching: @@ -131,6 +144,7 @@ def browser_test( server = server, shell = shell, playwright = playwright, + browser = browser, config = config, matching = matching, sources = ":" + name + "_sources", @@ -139,7 +153,7 @@ def browser_test( common = dict( copy_data_to_bin = False, entry_point = Label("//runtime:runner_entry"), - data = [":" + name + "_inputs", Label("//runtime:files")] + ([Label("//runtime:container_files")] if visual else []) + data, + data = [":" + name + "_inputs", Label("//runtime:files")] + ([Label("//runtime:container_files")] if visual and not browser else []) + data, env = env | { "VRT_DESCRIPTOR": "$(rlocationpath :%s_inputs)" % name, "VRT_BASE_URL": base_url or "", @@ -152,11 +166,14 @@ def browser_test( "VRT_TIMEOUT_MS": str(execution_timeout_seconds * 1000), }, ) - browser_env_inherit = ["DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH", "DOCKER_CONFIG"] if visual else [ + browser_env_inherit = [] if browser else ["DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH", "DOCKER_CONFIG"] if visual else [ key for key in ["PLAYWRIGHT_BROWSERS_PATH"] if key not in env and key not in env_inherit ] + if browser: + remote_browser_test(name, browser, common["env"], args, tags, timeout, data, target_platform) + return js_test( name = name, args = args, diff --git a/internal/remote.bzl b/internal/remote.bzl new file mode 100644 index 0000000..901c2b0 --- /dev/null +++ b/internal/remote.bzl @@ -0,0 +1,115 @@ +"""VRT execution actions and local result consumers.""" + +load("@aspect_rules_js//js:defs.bzl", "js_binary", "js_test") +load("//playwright:defs.bzl", "BrowserRuntimeInfo", "runfile") + +def _linux_impl(_settings, attr): + return {"//command_line_option:platforms": [str(attr.target_platform)]} + +_linux = transition( + implementation = _linux_impl, + inputs = [], + outputs = ["//command_line_option:platforms"], +) + +def _remote_impl(ctx): + browser = ctx.attr.browser[0][BrowserRuntimeInfo] + if browser.descriptor["arch"] != "x64": + fail("Remote VRT currently requires a Linux amd64 runtime and worker") + root = ctx.file.browser + output = ctx.actions.declare_directory(ctx.label.name + ".results") + files = [] + manifest = {} + for target in [ctx.attr.inputs[0], ctx.attr._runtime, ctx.attr._runner]: + info = target[DefaultInfo] + runfiles = info.default_runfiles + for file in depset(transitive = [info.files, runfiles.files]).to_list(): + files.append(file) + manifest[runfile(file)] = file.path + for link in runfiles.symlinks.to_list(): + files.append(link.target_file) + manifest["_main/" + link.path] = link.target_file.path + for link in runfiles.root_symlinks.to_list(): + files.append(link.target_file) + manifest[link.path] = link.target_file.path + locations = ctx.attr.data + ctx.attr.inputs + env = {key: ctx.expand_location(value, targets = locations) for key, value in ctx.attr.env.items() if key != "VRT_DESCRIPTOR"} + env["VRT_DESCRIPTOR"] = runfile(ctx.file.inputs) + job = ctx.actions.declare_file(ctx.label.name + ".job.json") + ctx.actions.write(job, json.encode({ + "runfiles": manifest, + "runner": ctx.file._runner.path, + "env": env, + "args": [ctx.expand_location(value, targets = locations) for value in ctx.attr.args], + "output": output.path, + "capture": ctx.attr.capture, + })) + descriptor = browser.descriptor + ctx.actions.run( + executable = "/workspace/" + root.path + "/" + descriptor["node"], + arguments = [ctx.file._bootstrap.path, job.path], + inputs = depset(files + [root, job, ctx.file._bootstrap]), + outputs = [output], + env = { + "VRT_RUNTIME_ROOT": root.path, + "HOME": "/tmp", + "TMPDIR": "/tmp", + "LANG": "C.UTF-8", + "TZ": "UTC", + "LD_LIBRARY_PATH": ":".join(["/workspace/" + root.path + "/" + p for p in descriptor["libraryDirs"]]), + }, + execution_requirements = {"no-local": "1"}, + mnemonic = "VrtCapture" if ctx.attr.capture else "VrtCompare", + ) + return [ + DefaultInfo(files = depset([output]), runfiles = ctx.runfiles(files = [output])), + OutputGroupInfo(inputs = depset(files + [root, job, ctx.file._bootstrap])), + ] + +_remote = rule( + implementation = _remote_impl, + attrs = { + "inputs": attr.label(mandatory = True, allow_single_file = True, cfg = _linux), + "browser": attr.label(mandatory = True, providers = [BrowserRuntimeInfo], allow_single_file = True, cfg = _linux), + "data": attr.label_list(allow_files = True, cfg = _linux), + "target_platform": attr.label(mandatory = True), + "env": attr.string_dict(), + "args": attr.string_list(), + "capture": attr.bool(), + "_runtime": attr.label(default = Label("//runtime:files")), + "_runner": attr.label(default = Label("//runtime:runner_entry"), allow_single_file = True), + "_bootstrap": attr.label(default = Label("//runtime:remote_runner_entry"), allow_single_file = True), + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), + }, +) + +def remote_browser_test(name, browser, env, args, tags, timeout, data, target_platform): + """Build comparisons/captures remotely, then consume their downloaded results.""" + for capture in [False, True]: + action = name + ("_capture" if capture else "_compare") + _remote( + name = action, + inputs = ":" + name + "_inputs", + browser = browser, + env = env, + args = args, + capture = capture, + data = data, + target_platform = target_platform, + exec_properties = {"input-rootfs-env": "VRT_RUNTIME_ROOT"}, + exec_compatible_with = [Label("@platforms//os:linux"), Label("@platforms//cpu:x86_64")], + tags = ["manual"], + ) + common = dict( + entry_point = Label("//runtime:remote_result_entry"), + data = [":" + action, Label("//runtime:remote_result_files")], + env = { + "VRT_RESULT": "$(rlocationpath :%s)" % action, + "VRT_APPLY_BASELINES": "1" if capture else "0", + "VRT_BASELINE_RELATIVE": env["VRT_BASELINE_RELATIVE"], + }, + ) + if capture: + js_binary(name = name + ".update", tags = ["manual"], **common) + else: + js_test(name = name, tags = ["manual", "visual_test", "no-remote"] + tags, timeout = timeout, **common) diff --git a/playwright/BUILD.bazel b/playwright/BUILD.bazel index 5a203b3..d3c8a4e 100644 --- a/playwright/BUILD.bazel +++ b/playwright/BUILD.bazel @@ -1 +1,29 @@ -exports_files(["defs.bzl"]) +load("@rules_python//python:defs.bzl", "py_binary", "py_test") + +exports_files(["defs.bzl", "archive.bzl"]) + +py_binary( + name = "unpack_runtime", + srcs = ["unpack_runtime.py"], + main = "unpack_runtime.py", +) + +py_test( + name = "unpack_runtime_test", + size = "small", + srcs = ["unpack_runtime.py", "unpack_runtime_test.py"], + main = "unpack_runtime_test.py", +) + +py_binary( + name = "unpack_oci", + srcs = ["unpack_oci.py", "unpack_runtime.py"], + main = "unpack_oci.py", +) + +py_test( + name = "unpack_oci_test", + size = "small", + srcs = ["unpack_oci.py", "unpack_runtime.py", "unpack_oci_test.py"], + main = "unpack_oci_test.py", +) diff --git a/playwright/archive.bzl b/playwright/archive.bzl new file mode 100644 index 0000000..f6af8e2 --- /dev/null +++ b/playwright/archive.bzl @@ -0,0 +1,46 @@ +"""Turn a caller-produced flattened runtime archive into declared files.""" + +def _archive_impl(ctx): + root = ctx.actions.declare_directory(ctx.label.name) + ctx.actions.run( + executable = ctx.executable._unpack, + arguments = [ctx.file.archive.path, root.path], + inputs = [ctx.file.archive], + tools = [ctx.attr._unpack[DefaultInfo].files_to_run], + outputs = [root], + mnemonic = "BrowserRuntimeUnpack", + ) + return [DefaultInfo(files = depset([root]), runfiles = ctx.runfiles(files = [root]))] + +browser_runtime_archive = rule( + implementation = _archive_impl, + doc = "Unpack a flattened Linux runtime tar into a closed directory for browser_runtime(root=...).", + attrs = { + "archive": attr.label(mandatory = True, allow_single_file = True), + "_unpack": attr.label(default = Label("//playwright:unpack_runtime"), executable = True, cfg = "exec"), + }, +) + +def _oci_impl(ctx): + if not ctx.file.image.is_directory: + fail("image must provide a declared OCI image layout directory") + root = ctx.actions.declare_directory(ctx.label.name) + ctx.actions.run( + executable = ctx.executable._unpack, + arguments = [ctx.file.image.path, root.path, ctx.attr.directory, "amd64"], + inputs = [ctx.file.image], + tools = [ctx.attr._unpack[DefaultInfo].files_to_run], + outputs = [root], + mnemonic = "BrowserRuntimeOci", + ) + return [DefaultInfo(files = depset([root]), runfiles = ctx.runfiles(files = [root]))] + +browser_runtime_oci = rule( + implementation = _oci_impl, + doc = "Apply a caller-owned Linux amd64 OCI image's declared layers and materialize its runtime directory.", + attrs = { + "image": attr.label(mandatory = True, allow_single_file = True), + "directory": attr.string(default = "."), + "_unpack": attr.label(default = Label("//playwright:unpack_oci"), executable = True, cfg = "exec"), + }, +) diff --git a/playwright/defs.bzl b/playwright/defs.bzl index 7a15363..bd565e3 100644 --- a/playwright/defs.bzl +++ b/playwright/defs.bzl @@ -6,6 +6,42 @@ REAPER_IMAGE = "testcontainers/ryuk:0.14.0@sha256:f0456560ea5b4acdbed0da0efc33b5 PlaywrightInfo = provider(fields = ["test", "core", "version", "image", "images"]) +BrowserRuntimeInfo = provider(fields = ["descriptor"]) + +def _relative_path(value, name): + if not value or value.startswith("/") or any([p in ["", ".", ".."] for p in value.split("/")]): + fail(name + " must be a nonempty relative path without dot segments") + return value + +def _browser_runtime_impl(ctx): + if not ctx.file.root.is_directory: + fail("browser_runtime root must be a declared directory containing the Linux runtime") + descriptor = { + "root": runfile(ctx.file.root), + "executable": _relative_path(ctx.attr.executable, "executable"), + "node": _relative_path(ctx.attr.node, "node"), + "libraryDirs": [_relative_path(p, "library_dirs") for p in ctx.attr.library_dirs], + "fontconfig": _relative_path(ctx.attr.fontconfig, "fontconfig"), + "arch": ctx.attr.arch, + } + return [ + DefaultInfo(files = depset([ctx.file.root]), runfiles = ctx.runfiles(files = [ctx.file.root])), + BrowserRuntimeInfo(descriptor = descriptor), + ] + +browser_runtime = rule( + implementation = _browser_runtime_impl, + doc = "Caller-owned Linux Chromium, Node, libraries, and fonts for execution inside an isolated action.", + attrs = { + "root": attr.label(mandatory = True, allow_single_file = True), + "executable": attr.string(mandatory = True), + "node": attr.string(mandatory = True), + "library_dirs": attr.string_list(mandatory = True), + "fontconfig": attr.string(mandatory = True), + "arch": attr.string(default = "x64", values = ["x64", "arm64"]), + }, +) + def _images(image): return [ {"image": image, "platform": "linux/amd64", "roles": ["browser", "control-relay"]}, diff --git a/playwright/unpack_oci.py b/playwright/unpack_oci.py new file mode 100644 index 0000000..8a1a91f --- /dev/null +++ b/playwright/unpack_oci.py @@ -0,0 +1,124 @@ +"""Materialize a runtime from declared OCI blobs, without registry access.""" +import hashlib +import json +from pathlib import Path +import re +import shutil +import sys +import tarfile +import tempfile + +from playwright.unpack_runtime import image_filter, materialize + + +def blob(layout, descriptor): + digest = descriptor["digest"] + if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): + raise ValueError(f"Unsupported OCI digest: {digest}") + file = layout / "blobs" / "sha256" / digest.split(":")[1] + with file.open("rb") as stream: + actual = hashlib.file_digest(stream, "sha256").hexdigest() + if actual != digest.split(":")[1] or file.stat().st_size != descriptor["size"]: + raise ValueError(f"OCI blob does not match its descriptor: {digest}") + return file + + +def select_manifest(layout, architecture): + index = json.loads((layout / "index.json").read_text()) + candidates = [] + + def visit(descriptor, depth=0): + if depth > 8: + raise ValueError("OCI indexes are nested too deeply") + document = json.loads(blob(layout, descriptor).read_text()) + if "manifests" in document: + for child in document["manifests"]: + visit(child, depth + 1) + else: + config = json.loads(blob(layout, document["config"]).read_text()) + if config.get("os") == "linux" and config.get("architecture") == architecture: + candidates.append(document) + + for descriptor in index["manifests"]: + visit(descriptor) + if len(candidates) != 1: + raise ValueError(f"Expected one Linux {architecture} image, found {len(candidates)}") + return candidates[0] + + +def remove(file): + if file.is_symlink() or file.is_file(): + file.unlink() + elif file.is_dir(): + shutil.rmtree(file) + + +def destination(root, name): + if name.startswith("/") or ".." in name.split("/"): + raise ValueError(f"OCI path escapes image root: {name}") + file = root / name + if file == root: + return file + if not file.parent.resolve().is_relative_to(root): + raise ValueError(f"OCI parent escapes image root: {name}") + return file + + +def apply_layer(root, archive): + def layer_filter(member, target): + member = image_filter(member, target) + # Layer application needs writable staging directories. Final directory + # permissions are normalized when producing the Bazel tree artifact. + return member.replace(mode=(member.mode or 0o755) | 0o700) if member.isdir() else member + + with tarfile.open(archive, "r:*") as source: + entries = source.getmembers() + # OCI whiteouts affect only lower layers, regardless of tar entry order. + for entry in entries: + name = Path(entry.name).name + if not name.startswith(".wh."): + continue + marker = destination(root, entry.name) + if not entry.isfile() or entry.size != 0 or name[4:] in ("", ".", ".."): + raise ValueError(f"Invalid OCI whiteout: {entry.name}") + if name == ".wh..wh..opq": + if marker.parent.exists(): + for child in marker.parent.iterdir(): + remove(child) + else: + remove(marker.parent / name[4:]) + for entry in entries: + if Path(entry.name).name.startswith(".wh."): + continue + file = destination(root, entry.name) + if file == root or entry.name in (".", "./"): + continue + # Replacing an existing link must not modify its former target. + if not (entry.isdir() and file.is_dir() and not file.is_symlink()): + remove(file) + source.extract(entry, root, filter=layer_filter) + + +def unpack_oci(layout, output, directory=".", architecture="amd64"): + layout = Path(layout) + if json.loads((layout / "oci-layout").read_text()).get("imageLayoutVersion") != "1.0.0": + raise ValueError("Unsupported OCI image layout version") + manifest = select_manifest(layout, architecture) + with tempfile.TemporaryDirectory(prefix="oci-runtime-") as temporary: + root = Path(temporary).resolve() + for layer in manifest["layers"]: + if layer["mediaType"] not in ( + "application/vnd.oci.image.layer.v1.tar", + "application/vnd.oci.image.layer.v1.tar+gzip", + "application/vnd.docker.image.rootfs.diff.tar.gzip", + ): + raise ValueError(f"Unsupported OCI layer media type: {layer['mediaType']}") + apply_layer(root, blob(layout, layer)) + selected = root if directory == "." else destination(root, directory) + if not selected.is_dir(): + raise ValueError(f"Missing runtime directory in OCI image: {directory}") + materialize(root, selected, Path(output)) + + +if __name__ == "__main__": + unpack_oci(*sys.argv[1:]) diff --git a/playwright/unpack_oci_test.py b/playwright/unpack_oci_test.py new file mode 100644 index 0000000..9fb46fe --- /dev/null +++ b/playwright/unpack_oci_test.py @@ -0,0 +1,99 @@ +import hashlib +import io +import json +from pathlib import Path +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +from playwright.unpack_oci import unpack_oci + + +class OciRuntimeTest(unittest.TestCase): + def image(self, root, layers, architecture="amd64"): + layout = root / "image" + blobs = layout / "blobs/sha256" + blobs.mkdir(parents=True) + (layout / "oci-layout").write_text('{"imageLayoutVersion":"1.0.0"}') + + def put(data, media): + digest = hashlib.sha256(data).hexdigest() + (blobs / digest).write_bytes(data) + return {"digest": "sha256:" + digest, "size": len(data), "mediaType": media} + + descriptors = [] + for entries in layers: + data = io.BytesIO() + with tarfile.open(fileobj=data, mode="w:gz") as archive: + for name, value, kind in entries: + member = tarfile.TarInfo(name) + if kind in ("link", "hardlink"): + member.type = tarfile.SYMTYPE if kind == "link" else tarfile.LNKTYPE + member.linkname = value + archive.addfile(member) + else: + content = value.encode() + member.size = len(content) + member.mode = 0o755 + archive.addfile(member, io.BytesIO(content)) + descriptors.append(put(data.getvalue(), "application/vnd.oci.image.layer.v1.tar+gzip")) + config = put(json.dumps({"os": "linux", "architecture": architecture}).encode(), "application/vnd.oci.image.config.v1+json") + manifest = put(json.dumps({"schemaVersion": 2, "config": config, "layers": descriptors}).encode(), "application/vnd.oci.image.manifest.v1+json") + (layout / "index.json").write_text(json.dumps({"schemaVersion": 2, "manifests": [manifest]})) + return layout, descriptors + + def test_layers_whiteouts_hardlinks_and_replaced_symlinks(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layout, _ = self.image(root, [ + [("runtime/bin/node", "old", "file"), + ("runtime/bin/alias", "runtime/bin/node", "hardlink"), + ("runtime/fonts/obsolete", "old font", "file"), + ("runtime/link", "/runtime/bin/node", "link")], + [("runtime/fonts/current", "new font", "file"), + ("runtime/fonts/.wh..wh..opq", "", "file"), + ("runtime/bin/.wh.node", "", "file"), + ("runtime/bin/node", "new", "file"), + ("runtime/link", "replacement", "file")], + ]) + output = root / "output" + alias = root / "temporary-alias" + alias.symlink_to(root.resolve(), target_is_directory=True) + with patch.object(tempfile, "tempdir", str(alias)): + unpack_oci(layout, output, "runtime") + self.assertEqual((output / "bin/node").read_text(), "new") + self.assertEqual((output / "bin/alias").read_text(), "old") + self.assertEqual((output / "link").read_text(), "replacement") + self.assertEqual([p.name for p in (output / "fonts").iterdir()], ["current"]) + self.assertEqual((output / "bin/node").stat().st_mode & 0o777, 0o755) + + def test_rejects_wrong_platform_and_corrupt_blob(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layout, descriptors = self.image(root, [[("runtime/node", "node", "file")]], "arm64") + with self.assertRaisesRegex(ValueError, "Expected one Linux amd64"): + unpack_oci(layout, root / "output") + digest = descriptors[0]["digest"].split(":")[1] + (layout / "blobs/sha256" / digest).write_bytes(b"corrupt") + with self.assertRaisesRegex(ValueError, "does not match"): + unpack_oci(layout, root / "output", architecture="arm64") + + def test_whiteouts_and_links_cannot_escape_the_image(self): + for entries in [ + [(".wh...", "", "file")], + [("../.wh.outside", "", "file")], + [("runtime/link", "../../outside", "link")], + ]: + with self.subTest(entries=entries), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + keep = root / "keep" + keep.write_text("keep") + layout, _ = self.image(root, [entries]) + with self.assertRaises((ValueError, tarfile.FilterError)): + unpack_oci(layout, root / "output") + self.assertEqual(keep.read_text(), "keep") + + +if __name__ == "__main__": + unittest.main() diff --git a/playwright/unpack_runtime.py b/playwright/unpack_runtime.py new file mode 100644 index 0000000..a6291c9 --- /dev/null +++ b/playwright/unpack_runtime.py @@ -0,0 +1,58 @@ +"""Unpack a flattened image filesystem without consulting host libraries. + +This consumes a filesystem tar, not an OCI layout or an ordered set of layers. +Image-root absolute links are rewritten before extraction, then all links are +materialized so the Bazel tree artifact contains only files and directories. +""" +import os +from pathlib import Path +import posixpath +import shutil +import sys +import tarfile +import tempfile + + +def image_filter(member, destination): + if member.name.startswith("/") or ".." in member.name.split("/"): + raise ValueError(f"Runtime archive path escapes image root: {member.name}") + if member.name.rsplit("/", 1)[-1].startswith(".wh."): + raise ValueError("Supply a flattened runtime archive, not OCI layers with whiteouts") + if member.issym() and member.linkname.startswith("/"): + target = posixpath.normpath(member.linkname).lstrip("/") + member = member.replace(linkname=posixpath.relpath(target or ".", posixpath.dirname(member.name) or ".")) + if member.islnk() and member.linkname.startswith("/"): + member = member.replace(linkname=posixpath.normpath(member.linkname).lstrip("/")) + return tarfile.data_filter(member, destination) + + +def materialize(root, source, target, parents=frozenset()): + try: + resolved = source.resolve(strict=True) + except FileNotFoundError as error: + raise ValueError(f"Runtime contains a dangling link: {source.relative_to(root)}") from error + if not resolved.is_relative_to(root): + raise ValueError(f"Runtime link escapes image root: {source.relative_to(root)}") + if resolved.is_dir(): + if resolved in parents: + raise ValueError(f"Runtime contains a directory link cycle: {source.relative_to(root)}") + target.mkdir(parents=True, exist_ok=True) + for entry in sorted(resolved.iterdir()): + materialize(root, entry, target / entry.name, parents | {resolved}) + elif resolved.is_file(): + shutil.copyfile(resolved, target) + os.chmod(target, 0o755 if resolved.stat().st_mode & 0o111 else 0o644) + else: + raise ValueError(f"Runtime contains an unsupported file: {source.relative_to(root)}") + + +def unpack(archive, output): + with tempfile.TemporaryDirectory(prefix="runtime-unpack-") as temporary: + root = Path(temporary).resolve() + with tarfile.open(archive, "r:*") as source: + source.extractall(root, filter=image_filter) + materialize(root, root, Path(output)) + + +if __name__ == "__main__": + unpack(*sys.argv[1:]) diff --git a/playwright/unpack_runtime_test.py b/playwright/unpack_runtime_test.py new file mode 100644 index 0000000..6ec323d --- /dev/null +++ b/playwright/unpack_runtime_test.py @@ -0,0 +1,64 @@ +import io +from pathlib import Path +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +from playwright.unpack_runtime import unpack + + +class UnpackRuntimeTest(unittest.TestCase): + def archive(self, directory, entries): + archive = directory / "runtime.tar" + with tarfile.open(archive, "w") as output: + for name, content, kind in entries: + member = tarfile.TarInfo(name) + if kind == "link": + member.type = tarfile.SYMTYPE + member.linkname = content + output.addfile(member) + else: + data = content.encode() + member.size = len(data) + member.mode = 0o755 if kind == "executable" else 0o644 + output.addfile(member, io.BytesIO(data)) + return archive + + def test_materializes_image_links_and_preserves_executability(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + archive = self.archive(root, [ + ("usr/bin/node", "declared node", "executable"), + ("usr/lib/loader", "declared loader", "file"), + ("bin", "/usr/bin", "link"), + ("lib64/ld-linux-x86-64.so.2", "/usr/lib/loader", "link"), + ]) + result = root / "result" + alias = root / "temporary-alias" + alias.symlink_to(root.resolve(), target_is_directory=True) + with patch.object(tempfile, "tempdir", str(alias)): + unpack(archive, result) + self.assertEqual((result / "bin/node").read_text(), "declared node") + self.assertEqual((result / "lib64/ld-linux-x86-64.so.2").read_text(), "declared loader") + self.assertEqual((result / "bin/node").stat().st_mode & 0o777, 0o755) + self.assertFalse(any(p.is_symlink() for p in result.rglob("*"))) + + def test_refuses_host_paths_dangling_links_cycles_and_unflattened_layers(self): + for entries in [ + [("../escape", "bad", "file")], + [("/escape", "bad", "file")], + [("link", "../../escape", "link")], + [("missing", "/not-in-image", "link")], + [("loop", "/", "link")], + [("usr/.wh.removed", "", "file")], + ]: + with self.subTest(entries=entries), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + archive = self.archive(root, entries) + with self.assertRaises((ValueError, tarfile.FilterError, FileNotFoundError)): + unpack(archive, root / "result") + + +if __name__ == "__main__": + unittest.main() diff --git a/runtime/BUILD.bazel b/runtime/BUILD.bazel index 3ef3c94..0e690a2 100644 --- a/runtime/BUILD.bazel +++ b/runtime/BUILD.bazel @@ -39,6 +39,7 @@ js_library( srcs = [ "arguments.js", "baselines.js", + "browser-runtime.js", "capture.js", "config.js", "config-url.js", @@ -69,6 +70,31 @@ filegroup( visibility = ["//visibility:public"], ) +js_library( + name = "remote_result_files", + srcs = ["remote-result.js", "baselines.js", "package.json"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "remote_result_entry", + srcs = ["remote-result-entry.js"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "remote_runner_entry", + srcs = ["remote-runner.js"], + visibility = ["//visibility:public"], +) + +js_test( + name = "remote_result_test", + size = "small", + data = ["package.json", ":typecheck"], + entry_point = "remote-result.test.js", +) + npm_package( name = "package", srcs = [ @@ -150,6 +176,13 @@ js_test( entry_point = "config.test.js", ) +js_test( + name = "browser_runtime_test", + size = "small", + data = ["package.json", ":typecheck"], + entry_point = "browser-runtime.test.js", +) + js_test( name = "visuals_test", size = "small", diff --git a/runtime/browser-runtime.test.ts b/runtime/browser-runtime.test.ts new file mode 100644 index 0000000..e3999d7 --- /dev/null +++ b/runtime/browser-runtime.test.ts @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import {test} from 'node:test' +import {browserRuntime, type BrowserRuntime} from './browser-runtime.js' + +test('declared browser resolution refuses host fallback and escaping files', { + skip: process.platform !== 'linux', +}, () => { + const inputs = fs.mkdtempSync(path.join(os.tmpdir(), 'browser-runtime-')) + const root = path.join(inputs, 'runtime') + fs.mkdirSync(root) + for (const file of ['node', 'chrome']) fs.writeFileSync(path.join(root, file), '') + for (const directory of ['lib', 'fonts']) fs.mkdirSync(path.join(root, directory)) + const runtime: BrowserRuntime = { + root: 'runtime', node: 'node', executable: 'chrome', + libraryDirs: ['lib'], fontconfig: 'fonts', arch: process.arch as 'x64' | 'arm64', + } + try { + assert.deepEqual(browserRuntime(inputs, runtime), { + node: path.join(root, 'node'), + env: { + VRT_CHROMIUM_EXECUTABLE: path.join(root, 'chrome'), + LD_LIBRARY_PATH: path.join(root, 'lib'), + FONTCONFIG_PATH: path.join(root, 'fonts'), + }, + }) + assert.throws(() => browserRuntime(inputs, {...runtime, executable: 'missing'}), /ENOENT/) + assert.throws(() => browserRuntime(inputs, {...runtime, executable: '../chrome'}), /Invalid browser runtime path/) + fs.symlinkSync(process.execPath, path.join(root, 'host')) + assert.throws(() => browserRuntime(inputs, {...runtime, executable: 'host'}), /escapes declared root/) + assert.throws(() => browserRuntime(inputs, {...runtime, arch: process.arch === 'x64' ? 'arm64' : 'x64'}), /select a matching execution platform/) + } finally { + fs.rmSync(inputs, {recursive: true, force: true}) + } +}) diff --git a/runtime/browser-runtime.ts b/runtime/browser-runtime.ts new file mode 100644 index 0000000..8cf2e3e --- /dev/null +++ b/runtime/browser-runtime.ts @@ -0,0 +1,34 @@ +import fs from 'node:fs' +import path from 'node:path' + +export interface BrowserRuntime { + root: string + executable: string + node: string + libraryDirs: string[] + fontconfig: string + arch: 'x64' | 'arm64' +} + +/** Resolve only declared runtime files; never fall back to a host browser. */ +export function browserRuntime(inputs: string, runtime: BrowserRuntime) { + if (process.platform !== 'linux' || process.arch !== runtime.arch) + throw new Error(`Browser runtime requires Linux ${runtime.arch}; select a matching execution platform`) + const root = fs.realpathSync(path.join(inputs, runtime.root)) + const resolve = (relative: string) => { + if (!relative || path.isAbsolute(relative) || relative.split('/').some(p => !p || p === '.' || p === '..')) + throw new Error(`Invalid browser runtime path: ${relative}`) + const file = fs.realpathSync(path.join(root, relative)) + if (!file.startsWith(root + path.sep)) + throw new Error(`Browser runtime path escapes declared root: ${relative}`) + return file + } + return { + node: resolve(runtime.node), + env: { + VRT_CHROMIUM_EXECUTABLE: resolve(runtime.executable), + LD_LIBRARY_PATH: runtime.libraryDirs.map(resolve).join(path.delimiter), + FONTCONFIG_PATH: resolve(runtime.fontconfig), + }, + } +} diff --git a/runtime/config.test.ts b/runtime/config.test.ts index 69f9554..aa07353 100644 --- a/runtime/config.test.ts +++ b/runtime/config.test.ts @@ -133,6 +133,23 @@ test('compiled suite config preserves runner paths and accepts a pixel-count bud ['junit', {outputFile: join(temp, 'junit.xml')}], [realpathSync(join(reporterPackage, 'index.js'))], ]) + // Consumer launch/connection overrides must not replace a declared browser. + writeFileSync(join(temp, 'declared.js'), `export default {use: { + connectOptions: {wsEndpoint: 'ws://elsewhere'}, + launchOptions: {executablePath: '/host/chrome', args: ['--proxy-server=elsewhere']}, + viewport: {width: 800, height: 600} + }}`) + process.env.VRT_CONFIG_OVERRIDE = join(temp, 'declared.js') + process.env.VRT_CHROMIUM_EXECUTABLE = '/inputs/runtime/chrome' + delete process.env.VRT_WS_ENDPOINT + const declared = (await import(new URL('./suite-config.js?declared', import.meta.url).href)).default + assert.equal(declared.use.connectOptions, undefined) + assert.deepEqual(declared.use.launchOptions, { + executablePath: '/inputs/runtime/chrome', + chromiumSandbox: false, + args: ['--no-zygote'], + }) + assert.deepEqual(declared.use.viewport, {width: 800, height: 600}) } finally { for (const key of Object.keys(process.env)) if (!(key in previous)) delete process.env[key] diff --git a/runtime/config.ts b/runtime/config.ts index 6aaab8d..8e16e19 100644 --- a/runtime/config.ts +++ b/runtime/config.ts @@ -70,13 +70,23 @@ export function visualConfig({ ...defaults, use: { ...defaults.use, - connectOptions: { - wsEndpoint: required('VRT_WS_ENDPOINT'), - exposeNetwork: networkTargets( - required('VRT_APP_URL'), - JSON.parse(required('VRT_NETWORK_ORIGINS')) as string[] - ), - }, + ...(process.env.VRT_CHROMIUM_EXECUTABLE + ? { + launchOptions: { + executablePath: process.env.VRT_CHROMIUM_EXECUTABLE, + chromiumSandbox: false, + args: ['--no-zygote'], + }, + } + : { + connectOptions: { + wsEndpoint: required('VRT_WS_ENDPOINT'), + exposeNetwork: networkTargets( + required('VRT_APP_URL'), + JSON.parse(required('VRT_NETWORK_ORIGINS')) as string[] + ), + }, + }), }, testMatch: '**/.rules-visual.spec.ts', testIgnore: [], diff --git a/runtime/remote-result-entry.ts b/runtime/remote-result-entry.ts new file mode 100644 index 0000000..38fa1b8 --- /dev/null +++ b/runtime/remote-result-entry.ts @@ -0,0 +1,29 @@ +import path from 'node:path' +import {consumeRemoteResult} from './remote-result.js' + +function required(name: string) { + const value = process.env[name] + if (!value) throw new Error(`Missing ${name}`) + return value +} + +try { + if (process.argv.length > 2) + throw new Error('Remote VRT selection is declared by the Bazel target; use separate targets for subsets') + const runfiles = process.env.RUNFILES_DIR || required('JS_BINARY__RUNFILES') + process.exitCode = consumeRemoteResult( + path.join(runfiles, required('VRT_RESULT')), + { + artifacts: process.env.TEST_UNDECLARED_OUTPUTS_DIR, + ...(process.env.VRT_APPLY_BASELINES === '1' ? { + update: { + workspace: required('BUILD_WORKSPACE_DIRECTORY'), + baselineRelative: required('VRT_BASELINE_RELATIVE'), + }, + } : {}), + } + ) +} catch (error) { + console.error(error) + process.exitCode = 1 +} diff --git a/runtime/remote-result.test.ts b/runtime/remote-result.test.ts new file mode 100644 index 0000000..e19e73a --- /dev/null +++ b/runtime/remote-result.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import {test} from 'node:test' +import {spawnSync} from 'node:child_process' +import {fileURLToPath} from 'node:url' +import {consumeRemoteResult} from './remote-result.js' + +test('failed remote capture preserves local baselines and returns failure artifacts', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vrt-result-')) + t.after(() => fs.rmSync(root, {recursive: true, force: true})) + const remote = path.join(root, 'remote') + const workspace = path.join(root, 'workspace') + fs.mkdirSync(path.join(remote, 'artifacts'), {recursive: true}) + fs.mkdirSync(path.join(remote, 'baselines')) + fs.mkdirSync(path.join(workspace, 'screenshots'), {recursive: true}) + fs.writeFileSync(path.join(workspace, 'screenshots/old.png'), 'keep') + fs.writeFileSync(path.join(remote, 'baselines/partial.png'), 'partial') + fs.writeFileSync(path.join(remote, 'artifacts/junit.xml'), '') + fs.writeFileSync(path.join(remote, 'result.json'), JSON.stringify({schemaVersion: 1, mode: 'capture', exitCode: 7})) + const artifacts = path.join(root, 'downloaded') + const update = {workspace, baselineRelative: 'screenshots'} + assert.equal(consumeRemoteResult(remote, {artifacts, update}), 7) + assert.equal(fs.readFileSync(path.join(artifacts, 'junit.xml'), 'utf8'), '') + assert.deepEqual(fs.readdirSync(path.join(workspace, 'screenshots')), ['old.png']) + + fs.writeFileSync(path.join(remote, 'result.json'), JSON.stringify({schemaVersion: 1, mode: 'capture', exitCode: 0})) + fs.unlinkSync(path.join(remote, 'baselines/partial.png')) + assert.throws(() => consumeRemoteResult(remote, {update}), /no screenshots/) + assert.equal(fs.readFileSync(path.join(workspace, 'screenshots/old.png'), 'utf8'), 'keep') + fs.writeFileSync(path.join(remote, 'baselines/new.png'), 'complete') + assert.equal(consumeRemoteResult(remote, {update}), 0) + assert.deepEqual(fs.readdirSync(path.join(workspace, 'screenshots')), ['new.png']) + assert.equal(fs.readFileSync(path.join(workspace, 'screenshots/new.png'), 'utf8'), 'complete') +}) + +test('result metadata and artifact links cannot bypass local validation', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vrt-result-')) + t.after(() => fs.rmSync(root, {recursive: true, force: true})) + fs.mkdirSync(path.join(root, 'artifacts')) + for (const result of [ + {schemaVersion: 2, mode: 'compare', exitCode: 0}, + {schemaVersion: 1, mode: 'capture', exitCode: 0}, + {schemaVersion: 1, mode: 'compare', exitCode: -1}, + {schemaVersion: 1, mode: 'compare', exitCode: null}, + ]) { + fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify(result)) + assert.throws(() => consumeRemoteResult(root), /Invalid remote VRT result/) + } + fs.writeFileSync(path.join(root, 'result.json'), JSON.stringify({schemaVersion: 1, mode: 'compare', exitCode: 1})) + fs.symlinkSync(process.execPath, path.join(root, 'artifacts/escape')) + assert.throws(() => consumeRemoteResult(root, {artifacts: path.join(root, 'downloads')}), /only regular files/) +}) + +test('remote bootstrap preserves a failing subprocess result for the local test', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vrt-bootstrap-')) + t.after(() => fs.rmSync(root, {recursive: true, force: true})) + const runner = path.join(root, 'consumer.mjs') + fs.writeFileSync(runner, `import fs from 'node:fs'; + fs.writeFileSync(process.env.TEST_UNDECLARED_OUTPUTS_DIR + '/junit.xml', ''); + process.exitCode = 7;`) + const output = path.join(root, 'output') + const job = path.join(root, 'job.json') + fs.writeFileSync(job, JSON.stringify({ + runfiles: {}, runner, env: {}, args: [], output, capture: false, + })) + // The VM invokes Node directly, without rules_js's process.execPath wrapper. + const node = fs.realpathSync(process.env.JS_BINARY__NODE_BINARY || process.execPath) + const result = spawnSync(node, [ + fileURLToPath(new URL('./remote-runner.js', import.meta.url)), job, + ], {encoding: 'utf8', env: {...process.env, NODE_OPTIONS: ''}, timeout: 5000}) + assert.equal(result.status, 0, result.stderr) + const artifacts = path.join(root, 'test-artifacts') + assert.equal(consumeRemoteResult(output, {artifacts}), 7) + assert.equal(fs.readFileSync(path.join(artifacts, 'junit.xml'), 'utf8'), '') +}) diff --git a/runtime/remote-result.ts b/runtime/remote-result.ts new file mode 100644 index 0000000..227956a --- /dev/null +++ b/runtime/remote-result.ts @@ -0,0 +1,52 @@ +import fs from 'node:fs' +import path from 'node:path' +import {baselineDestination, updateBaselines} from './baselines.js' + +export interface RemoteVrtResult { + schemaVersion: 1 + mode: 'compare' | 'capture' + exitCode: number +} + +function copyArtifacts(source: string, destination: string) { + const stat = fs.lstatSync(source) + if (stat.isDirectory()) { + fs.mkdirSync(destination, {recursive: true}) + for (const name of fs.readdirSync(source)) + copyArtifacts(path.join(source, name), path.join(destination, name)) + } else if (stat.isFile()) { + fs.copyFileSync(source, destination) + } else { + throw new Error('Remote VRT artifacts must contain only regular files and directories') + } +} + +/** Consume downloaded action outputs without executing consumer code locally. */ +export function consumeRemoteResult( + directory: string, + options: { + artifacts?: string + update?: {workspace: string; baselineRelative: string} + } = {} +) { + const result = JSON.parse( + fs.readFileSync(path.join(directory, 'result.json'), 'utf8') + ) as RemoteVrtResult + if ( + result.schemaVersion !== 1 || + result.mode !== (options.update ? 'capture' : 'compare') || + !Number.isInteger(result.exitCode) || + result.exitCode < 0 || result.exitCode > 255 + ) throw new Error('Invalid remote VRT result') + + if (options.artifacts) + copyArtifacts(path.join(directory, 'artifacts'), options.artifacts) + if (result.exitCode !== 0) return result.exitCode + if (options.update) { + const destination = baselineDestination( + options.update.workspace, options.update.baselineRelative + ) + updateBaselines(path.join(directory, 'baselines'), destination) + } + return 0 +} diff --git a/runtime/remote-runner.ts b/runtime/remote-runner.ts new file mode 100644 index 0000000..142e579 --- /dev/null +++ b/runtime/remote-runner.ts @@ -0,0 +1,47 @@ +// Runs inside the Linux execution action. Result consumption stays local. +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import {spawnSync} from 'node:child_process' + +const job = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')) as { + runfiles: Record + runner: string + env: Record + args: string[] + output: string + capture: boolean +} +const output = path.resolve(job.output) +const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'vrt-action-')) +const artifacts = path.join(output, 'artifacts') +fs.mkdirSync(artifacts, {recursive: true}) +const escape = (value: string) => value.replaceAll('\\', '\\b').replaceAll(' ', '\\s').replaceAll('\n', '\\n') +const manifest = path.join(temp, 'MANIFEST') +fs.writeFileSync(manifest, Object.entries(job.runfiles).map(([name, source]) => + ` ${escape(name)} ${escape(path.resolve(source))}\n` +).join('')) +const result = spawnSync(process.execPath, [ + path.resolve(job.runner), ...job.args, ...(job.capture ? ['--update'] : []), +], { + env: { + ...process.env, + ...job.env, + RUNFILES_DIR: temp, + RUNFILES_MANIFEST_FILE: manifest, + JS_BINARY__NODE_BINARY: process.execPath, + TEST_TMPDIR: temp, + TEST_UNDECLARED_OUTPUTS_DIR: artifacts, + VRT_CAPTURE_OUTPUT: job.capture ? path.join(output, 'baselines') : '', + }, + stdio: 'inherit', +}) +if (result.error) console.error(result.error) +// Return a successful build action even when tests fail, so Bazel downloads +// their reports and screenshots. The local test wrapper returns this status. +fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify({ + schemaVersion: 1, + mode: job.capture ? 'capture' : 'compare', + exitCode: result.status ?? 1, +})) +fs.rmSync(temp, {recursive: true, force: true}) diff --git a/runtime/runner.ts b/runtime/runner.ts index 2962280..eb82cde 100644 --- a/runtime/runner.ts +++ b/runtime/runner.ts @@ -10,6 +10,7 @@ import {testArguments} from './arguments.js' import {baselineDestination, updateBaselines} from './baselines.js' import {stageRunfiles, testEnvironment} from './isolation.js' import {hostBrowserEnvironment} from './host-browser.js' +import {browserRuntime, type BrowserRuntime} from './browser-runtime.js' function required(name: string) { const value = process.env[name] @@ -38,7 +39,8 @@ async function main() { const args = process.argv.slice(2) const update = args.includes('--update') if (!visual && update) throw new Error('E2E tests do not update baselines') - const destination = update + const captureOutput = update ? process.env.VRT_CAPTURE_OUTPUT : undefined + const destination = update && !captureOutput ? baselineDestination( required('BUILD_WORKSPACE_DIRECTORY'), required('VRT_BASELINE_RELATIVE') @@ -64,6 +66,7 @@ async function main() { matching: string | null server: string | null shell: {directory: string; entryPoint: string} | null + browser?: BrowserRuntime | null playwright: { test: string core: string @@ -72,7 +75,10 @@ async function main() { } } const selectors = testArguments(visual, args, descriptor.tests) - const node = fs.realpathSync(required('JS_BINARY__NODE_BINARY')) + const declaredBrowser = descriptor.browser + ? browserRuntime(inputs, descriptor.browser) + : undefined + const node = declaredBrowser?.node || fs.realpathSync(required('JS_BINARY__NODE_BINARY')) const testRoot = path.dirname(descriptorPath) const generated = path.join(testRoot, '.rules-browser') fs.mkdirSync(generated) @@ -151,6 +157,7 @@ async function main() { temp ), ...hostEnv, + ...declaredBrowser?.env, VRT_NETWORK_ORIGINS: JSON.stringify(origins), VRT_INPUTS: inputs, VRT_MODE: required('VRT_MODE'), @@ -250,7 +257,7 @@ async function main() { }) }) } - if (visual) { + if (visual && !declaredBrowser) { networkTargets(appUrl!, origins) // Docker settings and helper images apply only to VRT. for (const key of Object.keys(process.env)) @@ -291,8 +298,10 @@ async function main() { } ) children.push(child) + let timedOut = false const timer = setTimeout( () => { + timedOut = true console.error('VRT exceeded its execution timeout') killChildren() }, @@ -304,7 +313,7 @@ async function main() { }) child.once('exit', code => { clearTimeout(timer) - resolve(code ?? 1) + resolve(timedOut || interrupted ? 1 : code ?? 1) }) }) const discoveryCode = gallery ? await run(true) : 0 @@ -320,6 +329,10 @@ async function main() { updateBaselines(baselines, destination) console.log(`Updated baselines: ${destination}`) } + if (captureOutput) { + updateBaselines(baselines, captureOutput) + console.log(`Captured baselines: ${captureOutput}`) + } succeeded = true } finally { killChildren() diff --git a/runtime/suite-config.ts b/runtime/suite-config.ts index 1b37471..4db50f5 100644 --- a/runtime/suite-config.ts +++ b/runtime/suite-config.ts @@ -115,6 +115,9 @@ const testMatch = const managedUse = { ...merged.use, connectOptions: defaults.use!.connectOptions, + ...(visual && process.env.VRT_CHROMIUM_EXECUTABLE + ? {launchOptions: defaults.use!.launchOptions} + : {}), browserName: 'chromium' as const, } export default defineConfig(