From 54fa80c07c73bea3efd4dc087cac1223ca1d96e4 Mon Sep 17 00:00:00 2001 From: dank-openai Date: Fri, 10 Apr 2026 12:04:32 -0400 Subject: [PATCH] toolchain: defer driver materialization for runtime-only builds Problem Solved ChiliRT wheel builds need the XLS runtime files from a registered workspace toolchain without paying the cost of materializing `xlsynth-driver`. Before this change, loading a registered toolchain could still require driver configuration or stage placeholder driver artifacts, which made runtime-only consumers depend on a tool they do not use. Implementation Split the module-extension surface into runtime and toolchain repositories. The runtime repository materializes tools, DSLX stdlib, `libxls`, and runtime metadata without resolving any driver input. The toolchain repository remains loadable metadata until a driver-backed action is built. Real driver actions now materialize `xlsynth-driver` lazily from a declared Bazel action input when the consumer supplied a local or installed driver path. `auto` driver inputs are validated before use and fall back to download-backed materialization when the declared installed driver is not usable. `local_paths` driver materialization requires the configured `local_driver_path` and rejects declared inputs that do not resolve to that file, so placeholder driver files cannot masquerade as real drivers. The registered runtime-only smoke coverage now runs directly from presubmit and checks both halves of the contract: runtime-only builds do not materialize the driver, while explicit missing-driver builds fail with an actionable configuration error. Validation - python3 artifact_resolution_test.py - env BAZELISK_HOME=/tmp/rue-bazelisk-cache python3 run_presubmit.py -k registered - env BAZELISK_HOME=/tmp/rue-bazelisk-cache bazel test //:artifact_resolution_test //:external_bundle_exports_test --test_output=errors - python3 -m py_compile materialize_xls_bundle.py artifact_resolution_test.py registered_toolchain_smoke.py run_presubmit.py pr:chilirt-runtime-only-xls-toolchain --- DESIGN.md | 42 ++- README.md | 60 ++-- artifact_resolution_test.py | 190 ++++++++++++ extensions.bzl | 180 ++++++++++-- materialize_xls_bundle.py | 280 +++++++++++++++--- registered_toolchain_smoke.py | 538 ++++++++++++++++++++++++++++++++++ run_presubmit.py | 18 ++ xls_toolchain.bzl | 104 +++++++ 8 files changed, 1321 insertions(+), 91 deletions(-) create mode 100644 registered_toolchain_smoke.py diff --git a/DESIGN.md b/DESIGN.md index 088fe35..27382a8 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -10,11 +10,15 @@ flags. ## Bundle repos and exported targets -Each `xls.toolchain(...)` call materializes two repos. `@_runtime` -contains the selected tool binaries, the DSLX stdlib tree, the matching -`libxls` shared library, and the runtime-facing exports. `@_toolchain` -contains the local `xlsynth-driver`, the `:bundle` target, and the registered -toolchain target. The public split is: +Each `xls.toolchain(...)` call exposes one runtime repo and one toolchain repo. +`@_runtime` contains the selected tool binaries, the DSLX stdlib tree, +the matching `libxls` shared library, and the runtime-facing exports. +`@_toolchain` contains the `:bundle` target, the registered toolchain +target, and a declared `:xlsynth-driver` target. Loading or registering the +toolchain repo is metadata-only; the driver target copies, validates, +downloads, or installs `xlsynth-driver` only when `:xlsynth-driver` is built +directly or a rule action consumes the driver from `:bundle`. The public split +is: - `@_runtime//:libxls` - `@_runtime//:libxls_link` @@ -26,8 +30,17 @@ toolchain target. The public split is: - `@_runtime//:xlsynth_sys_runtime_files` - `@_runtime//:xlsynth_sys_link_dep` - `@_runtime//:libxls_runtime_files` +- `@_toolchain//:xlsynth-driver` - `@_toolchain//:bundle` -- `@_toolchain//:all` +- `@_toolchain//:toolchain` + +Workspaces still register the selected default with +`register_toolchains("@_toolchain//:all")`; that is the registration +pattern for the package, not a separate exported target. When a local or +installed driver path is configured, the module extension creates a private +generated repo that exposes that host driver file as a declared input to +`@_toolchain//:xlsynth-driver`. Downstream workspaces do not use that +repo directly. The `xlsynth_sys_*` exports are the intended downstream contract for `rules_rust` `crate_extension.annotation(...)` wiring. The preferred modern @@ -48,12 +61,19 @@ generic bundle internals. - `local_paths` uses explicit local paths and is the documented escape hatch for `/tmp/xls-local-dev/` style setups. -For the installed-layout modes, the provider derives the concrete paths from -the toolchain declaration instead of hard-coding a repository-global install -root: `/v` for the tools tree and +For the installed-layout modes, runtime materialization derives the concrete +tools and library paths from the toolchain declaration instead of hard-coding a +repository-global install root: `/v` +for the tools tree, DSLX stdlib, and `libxls`. Driver materialization derives `//bin/xlsynth-driver` -for the driver binary. The provider owns the version-derived suffixes; the -consumer workspace owns the root prefixes. +inside the declared driver target. The provider owns the version-derived +suffixes; the consumer workspace owns the root prefixes. + +For `local_paths`, runtime materialization uses `local_tools_path`, +`local_dslx_stdlib_path`, and `local_libxls_path`. The local driver path is +needed only by driver-backed actions. This lets runtime consumers depend on +`@_runtime` or register `@_toolchain` without materializing, +probing, downloading, or compiling `xlsynth-driver`. ## Default bundles and explicit overrides diff --git a/README.md b/README.md index 810865f..b720baa 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,7 @@ register_toolchains("@workspace_xls_toolchain//:all") release artifacts. - `installed_only` requires the matching installed layout. - `download_only` always downloads the release artifacts. -- `local_paths` uses `local_tools_path`, `local_dslx_stdlib_path`, - `local_driver_path`, and `local_libxls_path`. +- `local_paths` uses explicit paths supplied by the consumer workspace. For the installed-layout modes, `rules_xlsynth` derives exact-version paths as: @@ -59,27 +58,46 @@ For the installed-layout modes, `rules_xlsynth` derives exact-version paths as: - `//bin/xlsynth-driver` for the driver binary -The attributes accepted by each mode are strict: - -- `local_paths` requires all four `local_*` attrs and does not accept - `xls_version` or `xlsynth_driver_version`. -- `auto` and `installed_only` require `xls_version`, - `xlsynth_driver_version`, `installed_tools_root_prefix`, and - `installed_driver_root_prefix`, and do not accept any `local_*` attrs. -- `download_only` requires `xls_version` and `xlsynth_driver_version`, and - does not accept any `local_*` or `installed_*` attrs. - -Download-backed modes also have one host prerequisite: when `auto` falls back -to downloading, or when `download_only` is selected, the repository rule -installs `xlsynth-driver` with `rustup run nightly cargo install`. The host -running module resolution must have `rustup` available. If the nightly -toolchain is missing, `rules_xlsynth` bootstraps a repo-local `rustup` home -before installing the driver. - -Each `xls.toolchain(...)` call now exports two repos: +The attributes accepted by each mode are strict, but runtime and driver inputs +are resolved at different times: + +- The runtime repo for `local_paths` requires `local_tools_path`, + `local_dslx_stdlib_path`, and `local_libxls_path`. `local_driver_path` is + required only when a driver-backed bundle/action is built. +- The runtime repo for `auto` and `installed_only` requires `xls_version` and + `installed_tools_root_prefix`. `xlsynth_driver_version` and + `installed_driver_root_prefix` are required only when a driver-backed + bundle/action is built. +- `download_only` requires `xls_version` for the runtime repo. + `xlsynth_driver_version` is required only when the driver action has to + install the driver. +- `local_paths` does not accept `xls_version` or `xlsynth_driver_version`; + the other modes do not accept any `local_*` attrs. +- `download_only` does not accept any `installed_*` attrs. + +Registering or loading `@_toolchain` is metadata-only: it defines the +toolchain, bundle, and `xlsynth-driver` targets, but does not copy, execute, +download, or compile the driver. Driver materialization is a declared Bazel +action behind `@_toolchain//:xlsynth-driver` and behind rule actions that +consume the driver from `@_toolchain//:bundle`. + +Download-backed driver actions have one host prerequisite: when `auto` falls +back to downloading the driver, or when `download_only` is selected and a +driver-backed action is built, that action installs `xlsynth-driver` with +`rustup run nightly cargo install`. The execution host must have `rustup` +available. If the nightly toolchain is missing, `rules_xlsynth` bootstraps a +repo-local `rustup` home before installing the driver. + +Each `xls.toolchain(...)` call now exports two public repos: - `@_runtime` for runtime files, `xlsynth-sys` wiring, tools, and `libxls` -- `@_toolchain` for `:bundle`, `:toolchain`, and `register_toolchains(...)` +- `@_toolchain` for `:bundle`, `:toolchain`, `:xlsynth-driver`, and + the `@_toolchain//:all` registration pattern + +When a local or installed driver path is configured, the module extension +creates a private generated repo that exposes that host driver file as a +declared input to `@_toolchain//:xlsynth-driver`. Downstream workspaces +do not use that repo directly or publish it with `use_repo(...)`. The runtime repo exposes: diff --git a/artifact_resolution_test.py b/artifact_resolution_test.py index 8da7329..f8aa51d 100644 --- a/artifact_resolution_test.py +++ b/artifact_resolution_test.py @@ -43,6 +43,47 @@ def test_auto_falls_back_to_download(self): self.assertEqual(plan["xls_version"], "0.38.0") self.assertEqual(plan["driver_version"], "0.33.0") + def test_runtime_surface_does_not_require_or_select_driver(self): + observed_paths = [] + + def exists_fn(path): + observed_paths.append(path) + return path != "/tools/xlsynth/v0.38.0/xlsynth-driver-sentinel" + + plan = materialize_xls_bundle.resolve_artifact_plan( + artifact_source = "auto", + xls_version = "0.38.0", + driver_version = "", + surface = "runtime", + installed_tools_root_prefix = "/tools/xlsynth", + installed_driver_root_prefix = "", + exists_fn = exists_fn, + ) + self.assertEqual(plan["mode"], "installed") + self.assertNotIn("driver", plan) + self.assertNotIn("driver_version", plan) + self.assertEqual( + observed_paths, + [ + "/tools/xlsynth/v0.38.0", + "/tools/xlsynth/v0.38.0/xls/dslx/stdlib", + "/tools/xlsynth/v0.38.0/libxls.dylib" if sys.platform == "darwin" else "/tools/xlsynth/v0.38.0/libxls.so", + ], + ) + + def test_runtime_local_paths_does_not_require_local_driver_path(self): + plan = materialize_xls_bundle.resolve_artifact_plan( + artifact_source = "local_paths", + xls_version = "", + driver_version = "", + surface = "runtime", + local_tools_path = "/tmp/xls-local-dev/tools", + local_dslx_stdlib_path = "/tmp/xls-local-dev/stdlib", + local_libxls_path = "/tmp/xls-local-dev/libxls.so", + ) + self.assertEqual(plan["mode"], "local_paths") + self.assertNotIn("driver", plan) + def test_auto_requires_installed_prefixes(self): with self.assertRaises(ValueError): materialize_xls_bundle.resolve_artifact_plan( @@ -95,6 +136,129 @@ def test_local_paths_bypass_versioned_selection(self): "/tmp/xls-local-dev", ) + def test_resolve_driver_plan_prefers_installed_driver(self): + plan = materialize_xls_bundle.resolve_driver_plan( + artifact_source = "auto", + driver_version = "0.33.0", + installed_driver_root_prefix = "/tools/xlsynth-driver", + exists_fn = lambda path: True, + ) + self.assertEqual(plan["mode"], "installed") + self.assertEqual(plan["driver"], Path("/tools/xlsynth-driver/0.33.0/bin/xlsynth-driver")) + + def test_resolve_driver_plan_auto_falls_back_to_download(self): + plan = materialize_xls_bundle.resolve_driver_plan( + artifact_source = "auto", + driver_version = "0.33.0", + installed_driver_root_prefix = "/tools/xlsynth-driver", + exists_fn = lambda path: False, + ) + self.assertEqual(plan, {"mode": "download", "driver_version": "0.33.0"}) + + def test_resolve_driver_plan_uses_declared_installed_driver_input(self): + plan = materialize_xls_bundle.resolve_driver_plan( + artifact_source = "auto", + driver_version = "0.33.0", + installed_driver_root_prefix = "/unavailable/xlsynth-driver", + driver_input = "external/toolchain/host_xlsynth-driver", + exists_fn = lambda path: False, + ) + self.assertEqual(plan["mode"], "auto_driver_input") + self.assertEqual(plan["driver"], Path("external/toolchain/host_xlsynth-driver")) + self.assertEqual(plan["driver_version"], "0.33.0") + self.assertEqual(plan["installed_driver_root_prefix"], "/unavailable/xlsynth-driver") + + def test_resolve_driver_plan_rejects_declared_local_driver_input_without_plan_path(self): + with self.assertRaisesRegex(ValueError, "local_paths driver materialization requires local_driver_path"): + materialize_xls_bundle.resolve_driver_plan( + artifact_source = "local_paths", + driver_version = "", + local_driver_path = "", + driver_input = "external/toolchain/host_xlsynth-driver", + ) + + def test_resolve_driver_plan_rejects_declared_local_driver_input_mismatch(self): + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + configured_driver = root / "configured" / "xlsynth-driver" + configured_driver.parent.mkdir() + configured_driver.write_text("#!/bin/sh\nexit 0\n", encoding = "utf-8") + configured_driver.chmod(0o755) + declared_driver = root / "declared" / "host_xlsynth-driver" + declared_driver.parent.mkdir() + declared_driver.write_text("#!/bin/sh\nexit 127\n", encoding = "utf-8") + declared_driver.chmod(0o755) + + with self.assertRaisesRegex(ValueError, "local_paths declared driver input must match local_driver_path"): + materialize_xls_bundle.resolve_driver_plan( + artifact_source = "local_paths", + driver_version = "", + local_driver_path = str(configured_driver), + driver_input = str(declared_driver), + ) + + def test_resolve_driver_plan_uses_declared_local_driver_input_matching_plan_path(self): + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + configured_driver = root / "configured" / "xlsynth-driver" + configured_driver.parent.mkdir() + configured_driver.write_text("#!/bin/sh\nexit 0\n", encoding = "utf-8") + configured_driver.chmod(0o755) + declared_driver = root / "declared" / "host_xlsynth-driver" + declared_driver.parent.mkdir() + declared_driver.symlink_to(configured_driver) + + plan = materialize_xls_bundle.resolve_driver_plan( + artifact_source = "local_paths", + driver_version = "", + local_driver_path = str(configured_driver), + driver_input = str(declared_driver), + ) + self.assertEqual( + plan, + { + "mode": "local_paths", + "driver": declared_driver, + }, + ) + + def test_auto_driver_input_materialization_falls_back_when_declared_input_fails(self): + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + declared_driver = root / "declared" / "host_xlsynth-driver" + declared_driver.parent.mkdir() + declared_driver.write_text("", encoding = "utf-8") + declared_driver.chmod(0o755) + + fallback_driver = root / "installed" / "0.33.0" / "bin" / "xlsynth-driver" + fallback_driver.parent.mkdir(parents = True) + fallback_driver.write_text( + """#!/bin/sh +if [ "${1:-}" = "--version" ]; then + printf 'xlsynth-driver 0.33.0 fallback\\n' + exit 0 +fi +printf 'fallback body\\n' +""", + encoding = "utf-8", + ) + fallback_driver.chmod(0o755) + + output_driver = root / "out" / "xlsynth-driver" + materialize_xls_bundle.materialize_driver_binary( + repo_root = root / "repo", + plan = { + "mode": "auto_driver_input", + "driver": declared_driver, + "driver_version": "0.33.0", + "installed_driver_root_prefix": str(root / "installed"), + }, + driver_output = output_driver, + libxls_path = root / "runtime" / "libxls.so", + dslx_stdlib_path = root / "stdlib", + ) + self.assertIn("fallback body", output_driver.read_text(encoding = "utf-8")) + def test_download_only_rejects_installed_prefixes(self): with self.assertRaises(ValueError): materialize_xls_bundle.resolve_artifact_plan( @@ -489,6 +653,32 @@ def test_materialize_toolchain_surface_records_driver_capabilities(self): self.assertEqual(metadata["driver_supports_sv_struct_field_ordering"], "false") self.assertTrue((repo_root / "xlsynth-driver").exists()) + def test_materialize_driver_binary_copies_local_driver_output(self): + with tempfile.TemporaryDirectory() as tempdir: + repo_root = Path(tempdir) + source_driver = repo_root / "input-driver" + source_driver.write_text("#!/bin/sh\nexit 0\n", encoding = "utf-8") + source_driver.chmod(0o755) + driver_output = repo_root / "out" / "xlsynth-driver" + libxls_path = repo_root / "libxls.so" + libxls_path.write_text("xls\n", encoding = "utf-8") + stdlib_root = repo_root / "stdlib" + stdlib_root.mkdir() + + materialize_xls_bundle.materialize_driver_binary( + repo_root, + { + "mode": "local_paths", + "driver": source_driver, + }, + driver_output, + libxls_path, + stdlib_root, + ) + + self.assertEqual(driver_output.read_text(encoding = "utf-8"), "#!/bin/sh\nexit 0\n") + self.assertTrue(os.access(driver_output, os.X_OK)) + if __name__ == "__main__": unittest.main() diff --git a/extensions.bzl b/extensions.bzl index 3c58c2b..d5b0c37 100644 --- a/extensions.bzl +++ b/extensions.bzl @@ -29,6 +29,53 @@ def _runtime_repo_name(name): def _toolchain_repo_name(name): return name + "_toolchain" +def _driver_repo_name(name): + return name + "_driver" + +def _installed_driver_path(driver_version, installed_driver_root_prefix): + return "{}/{}/bin/xlsynth-driver".format( + installed_driver_root_prefix.rstrip("/"), + _normalize_version_text(driver_version), + ) + +def _quoted(value): + return "\"{}\"".format(value.replace("\\", "\\\\").replace("\"", "\\\"")) + +def _normalize_version_text(version): + if version.startswith("v"): + return version[1:] + return version + +def _version_components(version): + core = _normalize_version_text(version).split("-", 1)[0].split("+", 1)[0] + components = [] + for raw_part in core.split("."): + components.append(int(raw_part or "0")) + for _unused in range(3): + if len(components) < 3: + components.append(0) + return [components[0], components[1], components[2]] + +def _version_at_least(version, minimum): + version_components = _version_components(version) + minimum_components = _version_components(minimum) + for index in range(3): + if version_components[index] > minimum_components[index]: + return True + if version_components[index] < minimum_components[index]: + return False + return True + +def _driver_supports_sv_enum_case_naming_policy(driver_version): + if not driver_version: + return True + return _version_at_least(driver_version, "0.33.0") + +def _driver_supports_sv_struct_field_ordering(driver_version): + if not driver_version: + return True + return _version_at_least(driver_version, "0.36.0") + def _runtime_build_file(libxls_name, runtime_files, runtime_aliases): tool_list = ",\n ".join(['"{}"'.format(name) for name in _TOOL_BINARIES]) exported_files = ",\n ".join( @@ -152,14 +199,44 @@ xls_runtime_surface( lib_file_rule = lib_file_rule.strip(), ) -def _toolchain_build_file(repo_alias, runtime_repo_name, driver_supports_sv_enum_case_naming_policy, driver_supports_sv_struct_field_ordering): +def _string_attr_line(name, value): + if not value: + return "" + return " {} = {},\n".format(name, _quoted(value)) + +def _toolchain_build_file( + repo_alias, + runtime_repo_name, + action_path, + action_dyld_library_path, + action_ld_library_path, + artifact_source, + host_driver_label, + installed_driver_root_prefix, + local_driver_path, + rustup_path, + xlsynth_driver_version): + action_env_attrs = "".join([ + _string_attr_line("action_dyld_library_path", action_dyld_library_path), + _string_attr_line("action_ld_library_path", action_ld_library_path), + ]) + driver_attrs = "".join([ + _string_attr_line("host_driver", host_driver_label), + _string_attr_line("installed_driver_root_prefix", installed_driver_root_prefix), + _string_attr_line("local_driver_path", local_driver_path), + _string_attr_line("rustup_path", rustup_path), + _string_attr_line("xlsynth_driver_version", xlsynth_driver_version), + ]) return """# SPDX-License-Identifier: Apache-2.0 -load("@rules_xlsynth//:xls_toolchain.bzl", "xls_bundle", "xls_toolchain") +load("@rules_xlsynth//:xls_toolchain.bzl", "xls_bundle", "xls_toolchain", "xlsynth_driver_binary") -exports_files([ - "xlsynth-driver", -]) +xlsynth_driver_binary( + name = "xlsynth-driver", + action_path = {action_path}, +{action_env_attrs} artifact_source = {artifact_source}, + runtime = "@{runtime_repo_name}//:runtime", +{driver_attrs}) xls_bundle( name = "bundle", @@ -188,12 +265,56 @@ toolchain( visibility = ["//visibility:public"], ) """.format( - driver_supports_sv_enum_case_naming_policy = "True" if driver_supports_sv_enum_case_naming_policy else "False", - driver_supports_sv_struct_field_ordering = "True" if driver_supports_sv_struct_field_ordering else "False", + artifact_source = _quoted(artifact_source), + action_env_attrs = action_env_attrs, + action_path = _quoted(action_path), + driver_attrs = driver_attrs, + driver_supports_sv_enum_case_naming_policy = "True" if _driver_supports_sv_enum_case_naming_policy(xlsynth_driver_version) else "False", + driver_supports_sv_struct_field_ordering = "True" if _driver_supports_sv_struct_field_ordering(xlsynth_driver_version) else "False", repo_alias = repo_alias, runtime_repo_name = runtime_repo_name, ) +def _driver_path_to_stage(repo_ctx): + if not repo_ctx.attr.driver_path: + return None + driver_path = repo_ctx.path(repo_ctx.attr.driver_path) + if driver_path.exists: + return driver_path + if repo_ctx.attr.required: + fail("Missing required host xlsynth-driver at {}".format(driver_path)) + return None + +def _driver_repo_impl(repo_ctx): + link_name = "host_xlsynth-driver" + driver_path = _driver_path_to_stage(repo_ctx) + if driver_path == None: + repo_ctx.file(link_name, "#!/bin/sh\nexit 127\n", executable = True) + else: + repo_ctx.symlink(driver_path, link_name) + repo_ctx.file( + "BUILD.bazel", + """# SPDX-License-Identifier: Apache-2.0 + +exports_files(["host_xlsynth-driver"]) +""", + ) + +def _host_driver_path(toolchain): + if toolchain.artifact_source == "local_paths": + return toolchain.local_driver_path + if toolchain.artifact_source in ("auto", "installed_only") and toolchain.installed_driver_root_prefix and toolchain.xlsynth_driver_version: + return _installed_driver_path(toolchain.xlsynth_driver_version, toolchain.installed_driver_root_prefix) + return "" + +def _host_driver_required(toolchain): + return toolchain.artifact_source in ("local_paths", "installed_only") + +def _host_driver_label(toolchain, driver_name): + if not _host_driver_path(toolchain): + return "" + return "@{}//:host_xlsynth-driver".format(driver_name) + def _materialize_bundle_args(repo_ctx, surface): args = [ str(repo_ctx.path(Label("//:materialize_xls_bundle.py"))), @@ -254,24 +375,24 @@ def _runtime_repo_impl(repo_ctx): ) def _toolchain_repo_impl(repo_ctx): - python3 = repo_ctx.which("python3") - if python3 == None: - fail("python3 is required to materialize XLS bundles") - result = repo_ctx.execute([str(python3)] + _materialize_bundle_args(repo_ctx, "toolchain"), quiet = False) - if result.return_code != 0: - fail("Failed to materialize XLS toolchain surface {}:\nstdout:\n{}\nstderr:\n{}".format( - repo_ctx.name, - result.stdout, - result.stderr, - )) - metadata = _metadata_dict(repo_ctx, "toolchain_metadata.txt") + rustup = repo_ctx.which("rustup") + action_path = repo_ctx.os.environ.get("PATH", "") + action_dyld_library_path = repo_ctx.os.environ.get("DYLD_LIBRARY_PATH", "") + action_ld_library_path = repo_ctx.os.environ.get("LD_LIBRARY_PATH", "") repo_ctx.file( "BUILD.bazel", _toolchain_build_file( repo_alias = repo_ctx.attr.repo_alias, runtime_repo_name = repo_ctx.attr.runtime_repo_name, - driver_supports_sv_enum_case_naming_policy = metadata["driver_supports_sv_enum_case_naming_policy"] == "true", - driver_supports_sv_struct_field_ordering = metadata["driver_supports_sv_struct_field_ordering"] == "true", + action_path = action_path, + action_dyld_library_path = action_dyld_library_path, + action_ld_library_path = action_ld_library_path, + artifact_source = repo_ctx.attr.artifact_source, + host_driver_label = repo_ctx.attr.host_driver_label, + installed_driver_root_prefix = repo_ctx.attr.installed_driver_root_prefix, + local_driver_path = repo_ctx.attr.local_driver_path, + rustup_path = "" if rustup == None else str(rustup), + xlsynth_driver_version = repo_ctx.attr.xlsynth_driver_version, ), ) @@ -289,6 +410,7 @@ _runtime_repo_attrs = { _toolchain_repo_attrs = { "artifact_source": attr.string(mandatory = True), + "host_driver_label": attr.string(), "installed_driver_root_prefix": attr.string(), "installed_tools_root_prefix": attr.string(), "local_driver_path": attr.string(), @@ -301,6 +423,11 @@ _toolchain_repo_attrs = { "xlsynth_driver_version": attr.string(), } +_driver_repo_attrs = { + "driver_path": attr.string(), + "required": attr.bool(), +} + _xls_runtime_repo = repository_rule( implementation = _runtime_repo_impl, attrs = _runtime_repo_attrs, @@ -309,6 +436,12 @@ _xls_runtime_repo = repository_rule( _xls_toolchain_repo = repository_rule( implementation = _toolchain_repo_impl, attrs = _toolchain_repo_attrs, + environ = ["DYLD_LIBRARY_PATH", "LD_LIBRARY_PATH", "PATH"], +) + +_xls_driver_repo = repository_rule( + implementation = _driver_repo_impl, + attrs = _driver_repo_attrs, ) _toolchain_tag = tag_class(attrs = { @@ -329,6 +462,7 @@ def _xls_extension_impl(module_ctx): for toolchain in module.tags.toolchain: runtime_name = _runtime_repo_name(toolchain.name) toolchain_name = _toolchain_repo_name(toolchain.name) + driver_name = _driver_repo_name(toolchain.name) _xls_runtime_repo( name = runtime_name, artifact_source = toolchain.artifact_source, @@ -341,9 +475,15 @@ def _xls_extension_impl(module_ctx): xls_version = toolchain.xls_version, xlsynth_driver_version = toolchain.xlsynth_driver_version, ) + _xls_driver_repo( + name = driver_name, + driver_path = _host_driver_path(toolchain), + required = _host_driver_required(toolchain), + ) _xls_toolchain_repo( name = toolchain_name, artifact_source = toolchain.artifact_source, + host_driver_label = _host_driver_label(toolchain, driver_name), installed_driver_root_prefix = toolchain.installed_driver_root_prefix, installed_tools_root_prefix = toolchain.installed_tools_root_prefix, local_driver_path = toolchain.local_driver_path, diff --git a/materialize_xls_bundle.py b/materialize_xls_bundle.py index a600160..72a8d47 100644 --- a/materialize_xls_bundle.py +++ b/materialize_xls_bundle.py @@ -78,6 +78,19 @@ def derive_installed_paths( } +def derive_installed_runtime_paths( + xls_version, + installed_tools_root_prefix, + sys_platform = sys.platform): + normalized_xls_version = normalize_version(xls_version) + tools_root = Path(installed_tools_root_prefix) / "v{}".format(normalized_xls_version) + return { + "tools_root": tools_root, + "dslx_stdlib_root": tools_root / "xls" / "dslx" / "stdlib", + "libxls": tools_root / libxls_name_for_platform(sys_platform), + } + + def validate_stdlib_root(stdlib_root): if not stdlib_root.exists(): raise ValueError("DSLX stdlib root does not exist: {}".format(stdlib_root)) @@ -92,6 +105,7 @@ def resolve_artifact_plan( artifact_source, xls_version, driver_version, + surface = "toolchain", installed_tools_root_prefix = "", installed_driver_root_prefix = "", local_tools_path = "", @@ -100,88 +114,195 @@ def resolve_artifact_plan( local_libxls_path = "", exists_fn = os.path.exists, ): + if surface not in ("runtime", "toolchain"): + raise ValueError("Unknown XLS bundle surface: {}".format(surface)) + include_driver = surface == "toolchain" + if artifact_source == "local_paths": if xls_version or driver_version: raise ValueError("local_paths does not accept xls_version or xlsynth_driver_version") required = { "local_tools_path": local_tools_path, "local_dslx_stdlib_path": local_dslx_stdlib_path, - "local_driver_path": local_driver_path, "local_libxls_path": local_libxls_path, } + if include_driver: + required["local_driver_path"] = local_driver_path missing = [name for name, value in required.items() if not value] if missing: raise ValueError("local_paths requires {}".format(", ".join(sorted(missing)))) - return { + plan = { "mode": "local_paths", "tools_root": Path(local_tools_path), "dslx_stdlib_root": Path(local_dslx_stdlib_path), - "driver": Path(local_driver_path), "libxls": Path(local_libxls_path), } + if include_driver: + plan["driver"] = Path(local_driver_path) + return plan if artifact_source not in ("auto", "installed_only", "download_only"): raise ValueError("Unknown artifact_source: {}".format(artifact_source)) - if not xls_version or not driver_version: - raise ValueError("{} requires xls_version and xlsynth_driver_version".format(artifact_source)) + if not xls_version: + raise ValueError("{} requires xls_version".format(artifact_source)) + if include_driver and not driver_version: + raise ValueError("{} toolchain surface requires xlsynth_driver_version".format(artifact_source)) if local_tools_path or local_dslx_stdlib_path or local_driver_path or local_libxls_path: raise ValueError("{} does not accept local_paths attrs".format(artifact_source)) if artifact_source == "download_only": if installed_tools_root_prefix or installed_driver_root_prefix: raise ValueError("download_only does not accept installed_* attrs") else: - missing_installed = [ - name - for name, value in { - "installed_tools_root_prefix": installed_tools_root_prefix, - "installed_driver_root_prefix": installed_driver_root_prefix, - }.items() - if not value - ] + required_installed = {"installed_tools_root_prefix": installed_tools_root_prefix} + if include_driver: + required_installed["installed_driver_root_prefix"] = installed_driver_root_prefix + missing_installed = [name for name, value in required_installed.items() if not value] if missing_installed: raise ValueError("{} requires {}".format(artifact_source, ", ".join(sorted(missing_installed)))) - installed_paths = derive_installed_paths( - xls_version = xls_version, - driver_version = driver_version, - installed_tools_root_prefix = installed_tools_root_prefix, - installed_driver_root_prefix = installed_driver_root_prefix, - ) + if include_driver: + installed_paths = derive_installed_paths( + xls_version = xls_version, + driver_version = driver_version, + installed_tools_root_prefix = installed_tools_root_prefix, + installed_driver_root_prefix = installed_driver_root_prefix, + ) + else: + installed_paths = derive_installed_runtime_paths( + xls_version = xls_version, + installed_tools_root_prefix = installed_tools_root_prefix, + ) + if artifact_source == "download_only": - return { + plan = { "mode": "download", "xls_version": normalize_version(xls_version), - "driver_version": normalize_version(driver_version), } + if include_driver: + plan["driver_version"] = normalize_version(driver_version) + return plan installed_paths_present = all(exists_fn(str(path)) for path in installed_paths.values()) if artifact_source == "auto" and installed_paths_present: - return { + plan = { "mode": "installed", "tools_root": installed_paths["tools_root"], "dslx_stdlib_root": installed_paths["dslx_stdlib_root"], - "driver": installed_paths["driver"], "libxls": installed_paths["libxls"], } + if include_driver: + plan["driver"] = installed_paths["driver"] + return plan if artifact_source == "installed_only": if not installed_paths_present: - raise ValueError( - "installed_only requires exact-version installed paths for XLS {} and driver {}".format( - normalize_version(xls_version), - normalize_version(driver_version), - ) + message = "installed_only requires exact-version installed paths for XLS {}".format( + normalize_version(xls_version), ) - return { + if include_driver: + message = "{} and driver {}".format(message, normalize_version(driver_version)) + raise ValueError(message) + plan = { "mode": "installed", "tools_root": installed_paths["tools_root"], "dslx_stdlib_root": installed_paths["dslx_stdlib_root"], - "driver": installed_paths["driver"], "libxls": installed_paths["libxls"], } - return { + if include_driver: + plan["driver"] = installed_paths["driver"] + return plan + plan = { "mode": "download", "xls_version": normalize_version(xls_version), - "driver_version": normalize_version(driver_version), + } + if include_driver: + plan["driver_version"] = normalize_version(driver_version) + return plan + + +def resolve_driver_plan( + artifact_source, + driver_version, + installed_driver_root_prefix = "", + local_driver_path = "", + driver_input = "", + exists_fn = os.path.exists, +): + if artifact_source == "local_paths": + if not local_driver_path: + raise ValueError("local_paths driver materialization requires local_driver_path") + if driver_input: + resolved_driver_input = Path(driver_input).resolve() + resolved_local_driver_path = Path(local_driver_path).resolve() + if resolved_driver_input != resolved_local_driver_path: + raise ValueError( + "local_paths declared driver input must match local_driver_path: {} != {}".format( + driver_input, + local_driver_path, + ) + ) + return { + "mode": "local_paths", + "driver": Path(driver_input), + } + return { + "mode": "local_paths", + "driver": Path(local_driver_path), + } + + if driver_input: + if artifact_source == "auto": + if not driver_version: + raise ValueError("auto declared driver input requires xlsynth_driver_version") + return { + "mode": "auto_driver_input", + "driver": Path(driver_input), + "driver_version": normalize_version(driver_version), + "installed_driver_root_prefix": installed_driver_root_prefix, + } + if artifact_source in ("auto", "installed_only"): + if not driver_version: + raise ValueError("{} declared driver input requires xlsynth_driver_version".format(artifact_source)) + return { + "mode": "installed", + "driver": Path(driver_input), + "driver_version": normalize_version(driver_version), + } + if artifact_source == "download_only": + raise ValueError("download_only driver materialization does not accept driver_input") + raise ValueError("Unknown artifact_source: {}".format(artifact_source)) + + if artifact_source not in ("auto", "installed_only", "download_only"): + raise ValueError("Unknown artifact_source: {}".format(artifact_source)) + if not driver_version: + raise ValueError("{} driver materialization requires xlsynth_driver_version".format(artifact_source)) + if artifact_source == "download_only": + if installed_driver_root_prefix: + raise ValueError("download_only driver materialization does not accept installed_driver_root_prefix") + return { + "mode": "download", + "driver_version": normalize_version(driver_version), + } + + if not installed_driver_root_prefix: + raise ValueError("{} driver materialization requires installed_driver_root_prefix".format(artifact_source)) + + normalized_driver_version = normalize_version(driver_version) + installed_driver = Path(installed_driver_root_prefix) / normalized_driver_version / "bin" / "xlsynth-driver" + if exists_fn(str(installed_driver)): + return { + "mode": "installed", + "driver": installed_driver, + "driver_version": normalized_driver_version, + } + if artifact_source == "installed_only": + raise ValueError( + "installed_only driver materialization requires installed path for driver {}".format( + normalized_driver_version, + ) + ) + return { + "mode": "download", + "driver_version": normalized_driver_version, } @@ -498,11 +619,14 @@ def ensure_rustup_nightly_toolchain(rustup_path, env): def validate_installed_driver(driver_path, env, driver_version): - result = run_captured_text_command( - [str(driver_path), "--version"], - check = False, - env = env, - ) + try: + result = run_captured_text_command( + [str(driver_path), "--version"], + check = False, + env = env, + ) + except OSError as error: + raise RuntimeError("Failed to execute installed xlsynth-driver at {}: {}".format(driver_path, error)) from error if result.returncode != 0: raise RuntimeError( "Installed xlsynth-driver is not runnable at {}\nstdout:\n{}\nstderr:\n{}".format( @@ -524,7 +648,7 @@ def validate_installed_driver(driver_path, env, driver_version): ) -def install_driver(repo_root, driver_version, libxls_path, dslx_stdlib_path): +def install_driver(repo_root, driver_version, libxls_path, dslx_stdlib_path, rustup_path = ""): host_platform = detect_host_platform() install_root = driver_install_root(repo_root, driver_version, host_platform) rustup_home = rustup_home_root(repo_root, host_platform) @@ -546,7 +670,7 @@ def install_driver(repo_root, driver_version, libxls_path, dslx_stdlib_path): ensure_clean_path(install_root) install_root.mkdir(parents = True, exist_ok = True) - rustup = shutil.which("rustup") + rustup = rustup_path or shutil.which("rustup") if rustup is None: raise RuntimeError( "rules_xlsynth download fallback requires rustup to install xlsynth-driver {}".format( @@ -729,6 +853,59 @@ def materialize_toolchain_surface(repo_root, plan): write_toolchain_metadata(repo_root, driver_capabilities) +def materialize_driver_binary( + repo_root, + plan, + driver_output, + libxls_path, + dslx_stdlib_path, + rustup_path = ""): + driver_env = build_driver_environment(libxls_path, dslx_stdlib_path) + if plan["mode"] == "auto_driver_input": + try: + validate_installed_driver( + plan["driver"], + driver_env, + plan["driver_version"], + ) + driver_path = plan["driver"] + except RuntimeError: + fallback_plan = resolve_driver_plan( + artifact_source = "auto", + driver_version = plan["driver_version"], + installed_driver_root_prefix = plan["installed_driver_root_prefix"], + ) + materialize_driver_binary( + repo_root, + fallback_plan, + driver_output, + libxls_path, + dslx_stdlib_path, + rustup_path = rustup_path, + ) + return + elif plan["mode"] == "download": + driver_path = install_driver( + repo_root, + plan["driver_version"], + libxls_path, + dslx_stdlib_path, + rustup_path = rustup_path, + ) + else: + driver_path = plan["driver"] + if plan["mode"] == "installed": + validate_installed_driver( + driver_path, + driver_env, + plan["driver_version"], + ) + + driver_output.parent.mkdir(parents = True, exist_ok = True) + copy_path(driver_path, driver_output) + driver_output.chmod(driver_output.stat().st_mode | 0o111) + + def parse_args(argv): parser = argparse.ArgumentParser() parser.add_argument("--repo-root", required = True) @@ -742,6 +919,11 @@ def parse_args(argv): parser.add_argument("--local-dslx-stdlib-path", default = "") parser.add_argument("--local-driver-path", default = "") parser.add_argument("--local-libxls-path", default = "") + parser.add_argument("--driver-output", default = "") + parser.add_argument("--driver-input", default = "") + parser.add_argument("--driver-runtime-libxls", default = "") + parser.add_argument("--driver-runtime-stdlib", default = "") + parser.add_argument("--rustup-path", default = "") return parser.parse_args(argv) @@ -749,10 +931,30 @@ def main(argv): args = parse_args(argv) repo_root = Path(args.repo_root) repo_root.mkdir(parents = True, exist_ok = True) + if args.driver_output: + if not args.driver_runtime_libxls or not args.driver_runtime_stdlib: + raise ValueError("--driver-output requires --driver-runtime-libxls and --driver-runtime-stdlib") + driver_plan = resolve_driver_plan( + artifact_source = args.artifact_source, + driver_version = args.xlsynth_driver_version, + installed_driver_root_prefix = args.installed_driver_root_prefix, + local_driver_path = args.local_driver_path, + driver_input = args.driver_input, + ) + materialize_driver_binary( + repo_root, + driver_plan, + Path(args.driver_output).resolve(), + Path(args.driver_runtime_libxls).resolve(), + Path(args.driver_runtime_stdlib).resolve(), + rustup_path = args.rustup_path, + ) + return plan = resolve_artifact_plan( artifact_source = args.artifact_source, xls_version = args.xls_version, driver_version = args.xlsynth_driver_version, + surface = args.surface, installed_tools_root_prefix = args.installed_tools_root_prefix, installed_driver_root_prefix = args.installed_driver_root_prefix, local_tools_path = args.local_tools_path, diff --git a/registered_toolchain_smoke.py b/registered_toolchain_smoke.py new file mode 100644 index 0000000..f4cccb8 --- /dev/null +++ b/registered_toolchain_smoke.py @@ -0,0 +1,538 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# This smoke suite exercises the registered-toolchain behavior that ChiliRT +# depends on. Each test creates a temporary Bazel workspace that registers +# @lazy_xls_toolchain, then checks the contract from a consumer's point of view: +# runtime-only targets and package loading may inspect toolchain metadata, but +# they must not install, compile, stage, or otherwise require xlsynth-driver. +# Tests that build @lazy_xls_toolchain//:xlsynth-driver are the explicit +# driver-user side of the same contract; they verify that driver materialization +# happens only for driver-backed targets and that local or installed driver +# files are declared Bazel action inputs. + +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +sys.dont_write_bytecode = True + +import materialize_xls_bundle + + +BAZEL_COMMAND_TIMEOUT_SECONDS = int(os.environ.get("REGISTERED_TOOLCHAIN_SMOKE_BAZEL_TIMEOUT", "240")) +RULES_XLSYNTH_REPO_ROOT = Path( + os.environ.get("RULES_XLSYNTH_REPO_ROOT", Path(__file__).resolve().parent) +).resolve() + + +def rules_xlsynth_source_file(name): + path = RULES_XLSYNTH_REPO_ROOT / name + if not path.exists(): + raise RuntimeError("{} is missing from {}".format(name, RULES_XLSYNTH_REPO_ROOT)) + return path + + +def minimal_tool_path_env(bazel_path): + path_dirs = [str(Path(bazel_path).parent), "/usr/bin", "/bin", "/usr/sbin", "/sbin"] + return os.pathsep.join(path_dirs) + + +def write_text_file(path, content, mode = None): + path.write_text(content, encoding = "utf-8") + if mode != None: + path.chmod(mode) + + +def copy_runfile(source_name, dest): + shutil.copy2(rules_xlsynth_source_file(source_name), dest) + + +def create_minimal_rules_xlsynth_repo(repo_root): + repo_root.mkdir() + write_text_file( + repo_root / "MODULE.bazel", + """ +module(name = "rules_xlsynth") + +bazel_dep(name = "bazel_skylib", version = "1.6.1") +bazel_dep(name = "rules_cc", version = "0.2.11") +""".lstrip(), + ) + write_text_file( + repo_root / "BUILD.bazel", + """ +exports_files(["materialize_xls_bundle.py"]) + +toolchain_type( + name = "toolchain_type", + visibility = ["//visibility:public"], +) +""".lstrip(), + ) + copy_runfile("extensions.bzl", repo_root / "extensions.bzl") + copy_runfile("xls_toolchain.bzl", repo_root / "xls_toolchain.bzl") + copy_runfile("materialize_xls_bundle.py", repo_root / "materialize_xls_bundle.py") + + +def build_minimal_shared_library(output_path): + source_path = output_path.with_suffix(".c") + write_text_file(source_path, "int rules_xlsynth_runtime_probe(void) { return 0; }\n") + + if sys.platform == "darwin": + command = [ + "cc", + "-dynamiclib", + "-install_name", + "@rpath/{}".format(output_path.name), + "-o", + str(output_path), + str(source_path), + ] + else: + command = [ + "cc", + "-shared", + "-fPIC", + "-Wl,-soname,{}".format(output_path.name), + "-o", + str(output_path), + str(source_path), + ] + subprocess.run(command, check = True) + + +def create_local_runtime_bundle(root): + tools_root = root / "tools" + stdlib_root = root / "stdlib" + tools_root.mkdir(parents = True) + stdlib_root.mkdir(parents = True) + write_text_file(stdlib_root / "std.x", "pub fn id(x: u1) -> u1 { x }\n") + for tool_name in materialize_xls_bundle.TOOL_BINARIES: + write_text_file(tools_root / tool_name, "#!/bin/sh\nexit 127\n", 0o755) + + libxls_name = "libxls.dylib" if sys.platform == "darwin" else "libxls.so" + libxls_path = root / libxls_name + build_minimal_shared_library(libxls_path) + return { + "tools_root": tools_root, + "stdlib_root": stdlib_root, + "libxls_path": libxls_path, + } + + +def create_installed_runtime_bundle(installed_tools_root_prefix, xls_version): + installed_version_root = installed_tools_root_prefix / "v{}".format(xls_version) + stdlib_root = installed_version_root / "xls" / "dslx" / "stdlib" + installed_version_root.mkdir(parents = True) + stdlib_root.mkdir(parents = True) + write_text_file(stdlib_root / "std.x", "pub fn id(x: u1) -> u1 { x }\n") + for tool_name in materialize_xls_bundle.TOOL_BINARIES: + write_text_file(installed_version_root / tool_name, "#!/bin/sh\nexit 127\n", 0o755) + + libxls_name = "libxls.dylib" if sys.platform == "darwin" else "libxls.so" + libxls_path = installed_version_root / libxls_name + build_minimal_shared_library(libxls_path) + return { + "installed_tools_root_prefix": installed_tools_root_prefix, + "tools_root": installed_version_root, + "stdlib_root": stdlib_root, + "libxls_path": libxls_path, + } + + +def installed_driver_path(installed_driver_root_prefix, driver_version): + return installed_driver_root_prefix / driver_version / "bin" / "xlsynth-driver" + + +def write_versioned_driver(path, marker, driver_version = "0.33.0"): + write_text_file( + path, + """#!/bin/sh +if [ "${{1:-}}" = "--version" ]; then + printf 'xlsynth-driver {driver_version} {marker}\\n' + exit 0 +fi +printf 'unexpected fake driver execution {marker}\\n' >&2 +exit 1 +""".format(driver_version = driver_version, marker = marker), + 0o755, + ) + + +def create_runtime_only_workspace(workspace_root, rules_xlsynth_root, local_bundle, local_driver_path = None): + workspace_root.mkdir() + local_driver_attr = "" if local_driver_path == None else """ + local_driver_path = "{local_driver_path}", +""".format(local_driver_path = local_driver_path) + write_text_file( + workspace_root / "MODULE.bazel", + """ +module(name = "registered_runtime_only") + +bazel_dep(name = "rules_xlsynth", version = "0.0.0") +local_path_override( + module_name = "rules_xlsynth", + path = "{rules_xlsynth_root}", +) + +xls = use_extension("@rules_xlsynth//:extensions.bzl", "xls") +xls.toolchain( + name = "lazy_xls", + artifact_source = "local_paths", + local_tools_path = "{tools_root}", + local_dslx_stdlib_path = "{stdlib_root}", + local_libxls_path = "{libxls_path}", +{local_driver_attr} +) +use_repo( + xls, + "lazy_xls_runtime", + "lazy_xls_toolchain", +) +register_toolchains("@lazy_xls_toolchain//:all") +""".format( + rules_xlsynth_root = rules_xlsynth_root, + tools_root = local_bundle["tools_root"], + stdlib_root = local_bundle["stdlib_root"], + libxls_path = local_bundle["libxls_path"], + local_driver_attr = local_driver_attr, + ).lstrip(), + ) + write_text_file( + workspace_root / "BUILD.bazel", + """ +filegroup( + name = "runtime_inputs", + srcs = ["@lazy_xls_runtime//:xlsynth_sys_runtime_files"], +) +""".lstrip(), + ) + + +def create_auto_installed_workspace( + workspace_root, + rules_xlsynth_root, + local_bundle, + installed_driver_root_prefix, + xls_version = "0.38.0", + driver_version = "0.33.0"): + workspace_root.mkdir() + write_text_file( + workspace_root / "MODULE.bazel", + """ +module(name = "auto_installed_toolchain") + +bazel_dep(name = "rules_xlsynth", version = "0.0.0") +local_path_override( + module_name = "rules_xlsynth", + path = "{rules_xlsynth_root}", +) + +xls = use_extension("@rules_xlsynth//:extensions.bzl", "xls") +xls.toolchain( + name = "lazy_xls", + artifact_source = "auto", + xls_version = "{xls_version}", + xlsynth_driver_version = "{driver_version}", + installed_tools_root_prefix = "{installed_tools_root_prefix}", + installed_driver_root_prefix = "{installed_driver_root_prefix}", +) +use_repo( + xls, + "lazy_xls_runtime", + "lazy_xls_toolchain", +) +register_toolchains("@lazy_xls_toolchain//:all") +""".format( + driver_version = driver_version, + installed_driver_root_prefix = installed_driver_root_prefix, + installed_tools_root_prefix = local_bundle["installed_tools_root_prefix"], + rules_xlsynth_root = rules_xlsynth_root, + xls_version = xls_version, + ).lstrip(), + ) + write_text_file(workspace_root / "BUILD.bazel", "") + + +def run_nested_bazel(bazel_path, output_user_root, workspace_root, env, args): + cmdline = [ + bazel_path, + "--bazelrc=/dev/null", + "--max_idle_secs=5", + "--output_user_root={}".format(output_user_root), + ] + args + print("Running nested workspace command: " + subprocess.list2cmdline(cmdline), flush = True) + return subprocess.run( + cmdline, + cwd = workspace_root, + env = env, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + universal_newlines = True, + check = False, + timeout = BAZEL_COMMAND_TIMEOUT_SECONDS, + ) + + +def query_single_output_file(bazel_path, output_user_root, workspace_root, env, label): + result = run_nested_bazel( + bazel_path, + output_user_root, + workspace_root, + env, + ["cquery", label, "--output=files"], + ) + combined_output = "{}\n{}".format(result.stdout, result.stderr) + if result.returncode != 0: + raise RuntimeError(combined_output) + outputs = [ + line.strip() + for line in result.stdout.splitlines() + if line.strip() + ] + if len(outputs) != 1: + raise RuntimeError("Expected one output for {}, got {}\n{}".format(label, outputs, combined_output)) + output_path = Path(outputs[0]) + if output_path.is_absolute(): + return output_path + return workspace_root / output_path + + +def paths_with_basename(root, basename): + return sorted(path for path in root.rglob(basename)) + + +class RegisteredRuntimeOnlyTest(unittest.TestCase): + def create_nested_workspace(self, root, local_driver_path = None): + rules_xlsynth_root = root / "rules_xlsynth" + create_minimal_rules_xlsynth_repo(rules_xlsynth_root) + local_bundle = create_local_runtime_bundle(root / "local_xls") + workspace_root = root / "workspace" + create_runtime_only_workspace(workspace_root, rules_xlsynth_root, local_bundle, local_driver_path) + return workspace_root + + def test_00_registered_toolchain_does_not_require_driver_for_runtime_files(self): + bazel_path = shutil.which("bazel") + if bazel_path == None: + self.skipTest("bazel is not on PATH") + + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + workspace_root = self.create_nested_workspace(root) + env = dict(os.environ) + env["PATH"] = minimal_tool_path_env(bazel_path) + output_user_root = root / "bazel_output_user_root" + result = run_nested_bazel( + bazel_path, + output_user_root, + workspace_root, + env, + ["build", "//:runtime_inputs"], + ) + + combined_output = "{}\n{}".format(result.stdout, result.stderr) + self.assertEqual(result.returncode, 0, combined_output) + self.assertNotIn("xlsynth-driver", combined_output) + self.assertNotIn("rustup", combined_output) + self.assertNotIn("cargo", combined_output.lower()) + + def test_03_local_paths_without_driver_path_fails_when_driver_is_built(self): + bazel_path = shutil.which("bazel") + if bazel_path == None: + self.skipTest("bazel is not on PATH") + + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + workspace_root = self.create_nested_workspace(root) + env = dict(os.environ) + env["PATH"] = minimal_tool_path_env(bazel_path) + output_user_root = root / "bazel_output_user_root" + result = run_nested_bazel( + bazel_path, + output_user_root, + workspace_root, + env, + ["build", "@lazy_xls_toolchain//:xlsynth-driver"], + ) + + combined_output = "{}\n{}".format(result.stdout, result.stderr) + self.assertNotEqual(result.returncode, 0, combined_output) + self.assertIn("local_paths driver materialization requires local_driver_path", combined_output) + + def test_03_local_driver_file_is_declared_action_input(self): + bazel_path = shutil.which("bazel") + if bazel_path == None: + self.skipTest("bazel is not on PATH") + + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + local_driver = root / "local_driver" / "xlsynth-driver" + local_driver.parent.mkdir() + write_text_file(local_driver, "#!/bin/sh\n# version-one\n", 0o755) + workspace_root = self.create_nested_workspace(root, local_driver) + env = dict(os.environ) + env["PATH"] = minimal_tool_path_env(bazel_path) + output_user_root = root / "bazel_output_user_root" + + first_result = run_nested_bazel( + bazel_path, + output_user_root, + workspace_root, + env, + ["build", "@lazy_xls_toolchain//:xlsynth-driver"], + ) + self.assertEqual(first_result.returncode, 0, "{}\n{}".format(first_result.stdout, first_result.stderr)) + driver_output = query_single_output_file( + bazel_path, + output_user_root, + workspace_root, + env, + "@lazy_xls_toolchain//:xlsynth-driver", + ) + self.assertEqual(driver_output.read_text(encoding = "utf-8"), "#!/bin/sh\n# version-one\n") + + write_text_file(local_driver, "#!/bin/sh\n# version-two\n", 0o755) + second_result = run_nested_bazel( + bazel_path, + output_user_root, + workspace_root, + env, + ["build", "@lazy_xls_toolchain//:xlsynth-driver"], + ) + self.assertEqual(second_result.returncode, 0, "{}\n{}".format(second_result.stdout, second_result.stderr)) + self.assertEqual(driver_output.read_text(encoding = "utf-8"), "#!/bin/sh\n# version-two\n") + + def test_04_auto_installed_driver_file_is_declared_action_input(self): + bazel_path = shutil.which("bazel") + if bazel_path == None: + self.skipTest("bazel is not on PATH") + + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + driver_version = "0.33.0" + installed_driver_root_prefix = root / "installed_driver" + installed_driver = installed_driver_path(installed_driver_root_prefix, driver_version) + installed_driver.parent.mkdir(parents = True) + write_versioned_driver(installed_driver, "version-one", driver_version) + + rules_xlsynth_root = root / "rules_xlsynth" + create_minimal_rules_xlsynth_repo(rules_xlsynth_root) + local_bundle = create_installed_runtime_bundle(root / "installed_tools", "0.38.0") + workspace_root = root / "workspace" + create_auto_installed_workspace( + workspace_root, + rules_xlsynth_root, + local_bundle, + installed_driver_root_prefix, + driver_version = driver_version, + ) + env = dict(os.environ) + env["PATH"] = minimal_tool_path_env(bazel_path) + output_user_root = root / "bazel_output_user_root" + + first_result = run_nested_bazel( + bazel_path, + output_user_root, + workspace_root, + env, + ["build", "@lazy_xls_toolchain//:xlsynth-driver"], + ) + self.assertEqual(first_result.returncode, 0, "{}\n{}".format(first_result.stdout, first_result.stderr)) + driver_output = query_single_output_file( + bazel_path, + output_user_root, + workspace_root, + env, + "@lazy_xls_toolchain//:xlsynth-driver", + ) + self.assertIn("version-one", driver_output.read_text(encoding = "utf-8")) + + write_versioned_driver(installed_driver, "version-two", driver_version) + second_result = run_nested_bazel( + bazel_path, + output_user_root, + workspace_root, + env, + ["build", "@lazy_xls_toolchain//:xlsynth-driver"], + ) + self.assertEqual(second_result.returncode, 0, "{}\n{}".format(second_result.stdout, second_result.stderr)) + driver_text = driver_output.read_text(encoding = "utf-8") + self.assertIn("version-two", driver_text) + self.assertNotIn("version-one", driver_text) + + def test_02_auto_installed_toolchain_load_does_not_stage_host_driver(self): + bazel_path = shutil.which("bazel") + if bazel_path == None: + self.skipTest("bazel is not on PATH") + + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + driver_version = "0.33.0" + installed_driver_root_prefix = root / "installed_driver" + installed_driver = installed_driver_path(installed_driver_root_prefix, driver_version) + installed_driver.parent.mkdir(parents = True) + write_versioned_driver(installed_driver, "version-one", driver_version) + + rules_xlsynth_root = root / "rules_xlsynth" + create_minimal_rules_xlsynth_repo(rules_xlsynth_root) + local_bundle = create_installed_runtime_bundle(root / "installed_tools", "0.38.0") + workspace_root = root / "workspace" + create_auto_installed_workspace( + workspace_root, + rules_xlsynth_root, + local_bundle, + installed_driver_root_prefix, + driver_version = driver_version, + ) + env = dict(os.environ) + env["PATH"] = minimal_tool_path_env(bazel_path) + output_user_root = root / "bazel_output_user_root" + result = run_nested_bazel( + bazel_path, + output_user_root, + workspace_root, + env, + ["query", "@lazy_xls_toolchain//:all"], + ) + + staged_host_drivers = paths_with_basename(output_user_root, "host_xlsynth-driver") + + combined_output = "{}\n{}".format(result.stdout, result.stderr) + self.assertEqual(result.returncode, 0, combined_output) + self.assertIn("@lazy_xls_toolchain//:xlsynth-driver", combined_output) + self.assertEqual(staged_host_drivers, []) + + def test_01_toolchain_package_load_does_not_materialize_driver(self): + bazel_path = shutil.which("bazel") + if bazel_path == None: + self.skipTest("bazel is not on PATH") + + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + workspace_root = self.create_nested_workspace(root) + env = dict(os.environ) + env["PATH"] = minimal_tool_path_env(bazel_path) + output_user_root = root / "bazel_output_user_root" + result = run_nested_bazel( + bazel_path, + output_user_root, + workspace_root, + env, + ["query", "@lazy_xls_toolchain//:all"], + ) + + combined_output = "{}\n{}".format(result.stdout, result.stderr) + self.assertEqual(result.returncode, 0, combined_output) + self.assertIn("@lazy_xls_toolchain//:xlsynth-driver", combined_output) + self.assertNotIn("Installing xlsynth-driver", combined_output) + self.assertNotIn("Compiling xlsynth-driver", combined_output) + self.assertNotIn("rustup", combined_output) + self.assertNotIn("cargo", combined_output.lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/run_presubmit.py b/run_presubmit.py index ab951cf..beab90d 100644 --- a/run_presubmit.py +++ b/run_presubmit.py @@ -133,6 +133,19 @@ def bazel_build_opt( resolved_workspace_dir = config.repo_root if workspace_dir is None else workspace_dir _run_bazel(resolved_workspace_dir, 'build', targets, flags, capture_output = capture_output) + +def run_python_script(config: PresubmitConfig, script_name: str, args: Tuple[str, ...] = ()): + assert isinstance(args, tuple), args + cmdline = [ + sys.executable, + str(config.repo_root / script_name), + *args, + ] + env = dict(os.environ) + env['PYTHONDONTWRITEBYTECODE'] = '1' + print('Running command: ' + subprocess.list2cmdline(cmdline)) + subprocess.run(cmdline, check = True, cwd = str(config.repo_root), env = env) + @register def run_sample(config: PresubmitConfig): bazel_test_opt(('//sample/...',), config) @@ -434,6 +447,11 @@ def run_toolchain_helper_tests(config: PresubmitConfig): ) +@register +def run_registered_toolchain_smoke(config: PresubmitConfig): + run_python_script(config, 'registered_toolchain_smoke.py') + + def _stage_local_dev_example_tree(config: PresubmitConfig) -> Path: stage_root = Path('/tmp/xls-local-dev') with tempfile.TemporaryDirectory(prefix = 'xls_local_dev_stage_', dir = str(config.repo_root)) as temp_dir: diff --git a/xls_toolchain.bzl b/xls_toolchain.bzl index 65f8758..9f1e980 100644 --- a/xls_toolchain.bzl +++ b/xls_toolchain.bzl @@ -190,6 +190,110 @@ xls_runtime_surface = rule( }, ) +def _xlsynth_driver_binary_impl(ctx): + runtime = _runtime_struct_from_provider(ctx.attr.runtime[XlsRuntimeSurfaceInfo]) + output = ctx.actions.declare_file(ctx.label.name) + host_driver_inputs = [ctx.file.host_driver] if ctx.file.host_driver else [] + action_inputs = _dedupe_artifacts( + [ctx.file._materializer, runtime.libxls, runtime.dslx_stdlib] + + host_driver_inputs + + runtime.runtime_files + ) + action_env = {"PATH": ctx.attr.action_path} + if ctx.attr.action_ld_library_path: + action_env["LD_LIBRARY_PATH"] = ctx.attr.action_ld_library_path + if ctx.attr.action_dyld_library_path: + action_env["DYLD_LIBRARY_PATH"] = ctx.attr.action_dyld_library_path + ctx.actions.run_shell( + inputs = action_inputs, + outputs = [output], + arguments = [ + ctx.file._materializer.path, + output.path, + runtime.libxls.path, + runtime.dslx_stdlib.path, + ctx.attr.artifact_source, + ctx.attr.xlsynth_driver_version, + ctx.attr.installed_driver_root_prefix, + ctx.attr.local_driver_path, + ctx.attr.rustup_path, + ctx.file.host_driver.path if ctx.file.host_driver else "", + ], + command = """ + set -euo pipefail + script="$1" + output="$2" + runtime_libxls="$3" + runtime_stdlib="$4" + artifact_source="$5" + driver_version="$6" + installed_driver_root_prefix="$7" + local_driver_path="$8" + rustup_path="$9" + host_driver="${10}" + + work="${TMPDIR:-/tmp}/rules_xlsynth_driver_${RANDOM}" + rm -rf "$work" + mkdir -p "$work" + trap 'rm -rf "$work"' EXIT + + command=( + python3 + "$script" + --repo-root "$work" + --artifact-source "$artifact_source" + --surface toolchain + --driver-output "$output" + --driver-runtime-libxls "$runtime_libxls" + --driver-runtime-stdlib "$runtime_stdlib" + ) + if [[ -n "$driver_version" ]]; then + command+=(--xlsynth-driver-version "$driver_version") + fi + if [[ -n "$installed_driver_root_prefix" ]]; then + command+=(--installed-driver-root-prefix "$installed_driver_root_prefix") + fi + if [[ -n "$local_driver_path" ]]; then + command+=(--local-driver-path "$local_driver_path") + fi + if [[ -n "$host_driver" ]]; then + command+=(--driver-input "$host_driver") + fi + if [[ -n "$rustup_path" ]]; then + command+=(--rustup-path "$rustup_path") + fi + "${command[@]}" + """, + env = action_env, + progress_message = "Materializing xlsynth-driver for {}".format(ctx.label), + mnemonic = "XlsynthDriverBinary", + ) + return DefaultInfo( + files = depset(direct = [output]), + executable = output, + ) + +xlsynth_driver_binary = rule( + implementation = _xlsynth_driver_binary_impl, + attrs = { + "action_dyld_library_path": attr.string(), + "action_ld_library_path": attr.string(), + "action_path": attr.string(mandatory = True), + "artifact_source": attr.string(mandatory = True), + "host_driver": attr.label(allow_single_file = True), + "installed_driver_root_prefix": attr.string(), + "local_driver_path": attr.string(), + "runtime": attr.label(mandatory = True, providers = [XlsRuntimeSurfaceInfo]), + "rustup_path": attr.string(), + "xlsynth_driver_version": attr.string(), + "_materializer": attr.label( + default = Label("//:materialize_xls_bundle.py"), + allow_single_file = True, + ), + }, + executable = True, +) + def _xls_bundle_impl(ctx): runtime = _runtime_struct_from_provider(ctx.attr.runtime[XlsRuntimeSurfaceInfo]) driver = _single_artifact(ctx.attr.driver, "driver")