Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 16 additions & 16 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -464,10 +464,10 @@ jobs:
python3 -m venv --system-site-packages .venv
source .venv/bin/activate
pip install --upgrade pip
# Pin the onboard a2a3 host_runtime build to the same pto-isa commit
# the scene tests check out (PTO_ISA_COMMIT), so host_runtime.so and
# the test-time kernels share one pto-isa revision (see issue #1067).
pip install --config-settings=cmake.define.SIMPLER_PTO_ISA_COMMIT="${PTO_ISA_COMMIT}" '.[test]'
# The host_runtime build resolves pto-isa from pto_isa.pin by default
# (which CI exports as PTO_ISA_COMMIT), so host_runtime.so and the
# test-time kernels share one pto-isa revision (see issue #1067).
pip install '.[test]'

- name: Run Python hardware unit tests
run: |
Expand Down Expand Up @@ -556,10 +556,10 @@ jobs:
python3 -m venv --system-site-packages .venv
source .venv/bin/activate
pip install --upgrade pip
# Pin the onboard a2a3 host_runtime build to the same pto-isa commit
# the scene tests check out (PTO_ISA_COMMIT), so host_runtime.so and
# the test-time kernels share one pto-isa revision (see issue #1067).
pip install --config-settings=cmake.define.SIMPLER_PTO_ISA_COMMIT="${PTO_ISA_COMMIT}" '.[test]'
# The host_runtime build resolves pto-isa from pto_isa.pin by default
# (which CI exports as PTO_ISA_COMMIT), so host_runtime.so and the
# test-time kernels share one pto-isa revision (see issue #1067).
pip install '.[test]'
# Warn if graphviz isn't installed on the runner — the dep_gen
# smoke step needs `dot` to render deps_viewer. The Python
# has_binary("dot") guard in the test will skip cleanly without
Expand Down Expand Up @@ -677,10 +677,10 @@ jobs:
python3 -m venv --system-site-packages .venv
source .venv/bin/activate
pip install --upgrade pip
# Pin the onboard a2a3 host_runtime build to the same pto-isa commit
# the scene tests check out (PTO_ISA_COMMIT), so host_runtime.so and
# the test-time kernels share one pto-isa revision (see issue #1067).
pip install --config-settings=cmake.define.SIMPLER_PTO_ISA_COMMIT="${PTO_ISA_COMMIT}" '.[test]'
# The host_runtime build resolves pto-isa from pto_isa.pin by default
# (which CI exports as PTO_ISA_COMMIT), so host_runtime.so and the
# test-time kernels share one pto-isa revision (see issue #1067).
pip install '.[test]'

- name: Run Python hardware unit tests (a5)
run: |
Expand Down Expand Up @@ -752,10 +752,10 @@ jobs:
python3 -m venv --system-site-packages .venv
source .venv/bin/activate
pip install --upgrade pip
# Pin the onboard a2a3 host_runtime build to the same pto-isa commit
# the scene tests check out (PTO_ISA_COMMIT), so host_runtime.so and
# the test-time kernels share one pto-isa revision (see issue #1067).
pip install --config-settings=cmake.define.SIMPLER_PTO_ISA_COMMIT="${PTO_ISA_COMMIT}" '.[test]'
# The host_runtime build resolves pto-isa from pto_isa.pin by default
# (which CI exports as PTO_ISA_COMMIT), so host_runtime.so and the
# test-time kernels share one pto-isa revision (see issue #1067).
pip install '.[test]'

- name: Run pytest scene tests (a5)
run: |
Expand Down
15 changes: 0 additions & 15 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -36,27 +36,13 @@ add_subdirectory(python/bindings)
set(SIMPLER_PTO_CLONE_PROTOCOL "ssh" CACHE STRING
"Protocol for cloning pto-isa during install (ssh or https)")

# Override the pto-isa revision used when `build_runtimes.py` builds the
# onboard a2a3 host runtime (which compiles the pto-isa SDMA headers into
# host_runtime.so). Empty (default) uses pto_isa.pin. Use latest/head/none to
# explicitly track origin/HEAD. Override at `pip install` time via
# pip install --config-settings=cmake.define.SIMPLER_PTO_ISA_COMMIT=<sha> .
set(SIMPLER_PTO_ISA_COMMIT "" CACHE STRING
"Override pto-isa commit for onboard a2a3 host build (empty = pto_isa.pin; latest/head/none = HEAD)")

# Compiler sanitizer for host-compiled targets (sim runtime/kernels and the
# onboard host runtime). Default `none`. Preset (asan/ubsan/tsan) or a raw
# -fsanitize token list. Enable at `pip install` time via
# pip install --no-build-isolation --config-settings=cmake.define.SIMPLER_SANITIZER=asan .
set(SIMPLER_SANITIZER "none" CACHE STRING
"Sanitizer for host targets during install (none/asan/ubsan/tsan)")

# Pass --pto-isa-commit only when explicitly overridden; an empty cache var
# leaves the flag off so build_runtimes uses pto_isa.pin by default.
if(NOT SIMPLER_PTO_ISA_COMMIT STREQUAL "")
set(_pto_isa_commit_arg --pto-isa-commit "${SIMPLER_PTO_ISA_COMMIT}")
endif()

# Pre-build runtime binaries (persistent build dirs for incremental compilation)
add_custom_target(build_runtimes ALL
COMMAND ${Python_EXECUTABLE}
Expand All @@ -65,7 +51,6 @@ add_custom_target(build_runtimes ALL
--cache-dir ${CMAKE_SOURCE_DIR}/build/cache
--clone-protocol ${SIMPLER_PTO_CLONE_PROTOCOL}
--sanitizer ${SIMPLER_SANITIZER}
${_pto_isa_commit_arg}
COMMENT "Building runtime binaries (incremental)..."
)

Expand Down
2 changes: 1 addition & 1 deletion docs/developer-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ This builds the nanobind `_task_interface` extension **and** pre-builds all runt
| Nanobind bindings (`python/bindings/`) | Re-run `pip install --no-build-isolation -e .` (no rebuild-on-import; `editable.rebuild = false`) |
| Python-only code (`python/*.py`, `simpler_setup/*.py`) | No rebuild needed (editable install) |
| Examples / kernels (`examples/{arch}/`, `tests/st/`) | No rebuild needed, just re-run |
| Pinned pto-isa commit (CI `PTO_ISA_COMMIT`) | Re-run `pip install` — onboard a2a3 `host_runtime.so` embeds pto-isa SDMA headers, so a commit bump must rebuild it. Pin the build to the same commit the tests use via `--config-settings=cmake.define.SIMPLER_PTO_ISA_COMMIT=<sha>` (issue #1067). |
| Pinned pto-isa commit (CI `PTO_ISA_COMMIT`) | Re-run `pip install` — onboard a2a3 `host_runtime.so` embeds pto-isa SDMA headers, so a commit bump must rebuild it. `pip install` reads `pto_isa.pin` by default; to build against a different commit without changing the pin, set `PTO_ISA_ROOT` at a pre-checked-out clone or run `python simpler_setup/build_runtimes.py --pto-isa-commit <sha>` (issue #1067). |
Comment thread
yanghaoran29 marked this conversation as resolved.

### Runtime binary lookup

Expand Down
40 changes: 17 additions & 23 deletions docs/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,20 +77,16 @@ It follows this order:

1. If `PTO_ISA_ROOT` points to an existing directory, that directory wins.
`simpler` treats it as user-managed and uses it as-is.
2. If the managed checkout is being used and `--pto-isa-commit` or the CMake
`SIMPLER_PTO_ISA_COMMIT` define supplies a value, that value wins.
3. If no CLI/CMake value is supplied and `SIMPLER_PTO_ISA_COMMIT` is set in
the environment, that environment value wins.
4. If no explicit value is requested, `simpler` checks out the commit recorded
2. If the managed checkout is being used and `--pto-isa-commit` supplies a
value, that value wins.
3. If no explicit value is requested, `simpler` checks out the commit recorded
in `pto_isa.pin`.
5. If `pto_isa.pin` is missing, `simpler` warns and falls back to the latest
4. If `pto_isa.pin` is missing, `simpler` warns and falls back to the latest
`origin/HEAD` behavior for downstream checkout compatibility.

For explicit values in steps 2 and 3, a concrete SHA/tag/ref checks out that
For the explicit value in step 2, a concrete SHA/tag/ref checks out that
revision. The values `latest`, `head`, and `none` are explicit opt-outs from
the pin and use `origin/HEAD`. For the CMake cache variable, the empty string is
the default value and means "use `pto_isa.pin`"; use `latest`, `head`, or `none`
for an explicit CMake opt-out.
the pin and use `origin/HEAD`.

When the managed checkout is selected, `simpler` uses `build/pto-isa`, cloning
it on first use if necessary.
Expand All @@ -105,14 +101,14 @@ checkout `--pto-isa-commit "$PTO_ISA_COMMIT"` inside that directory. To run
with a specific commit while using a custom `PTO_ISA_ROOT`, checkout that
commit in the PTO-ISA repository yourself before running `simpler`.

At install time, pass a concrete PTO-ISA commit through CMake when building
a2a3 onboard runtimes:
At install time, `pip install` builds a2a3 onboard runtimes against the
commit recorded in `pto_isa.pin` by default. To build against a different
commit without changing the pin, either point `PTO_ISA_ROOT` at a
pre-checked-out pto-isa clone, or run `build_runtimes.py` manually:

```bash
export PTO_ISA_COMMIT=0123456789abcdef0123456789abcdef01234567
pip install --no-build-isolation \
--config-settings=cmake.define.SIMPLER_PTO_ISA_COMMIT="$PTO_ISA_COMMIT" \
-e .
python simpler_setup/build_runtimes.py --pto-isa-commit "$PTO_ISA_COMMIT" --platforms a2a3
```

At test/run time, pass the matching commit through pytest or the scene-test
Expand All @@ -124,11 +120,9 @@ pytest examples --platform a2a3 --pto-isa-commit "$PTO_ISA_COMMIT"
python path/to/scene_test.py --platform a2a3 --pto-isa-commit "$PTO_ISA_COMMIT"
```

`SIMPLER_PTO_ISA_COMMIT="$PTO_ISA_COMMIT"` can also provide the concrete commit
when the CLI option is omitted. Leaving both unset uses `pto_isa.pin`, so a
plain local install and test run match the repository pin. To explicitly track
the remote default branch instead, pass `--pto-isa-commit latest` or set
`SIMPLER_PTO_ISA_COMMIT=latest`.
Leaving `--pto-isa-commit` unset uses `pto_isa.pin`, so a plain local install
and test run match the repository pin. To explicitly track the remote default
branch instead, pass `--pto-isa-commit latest`.

For a2a3 onboard runtimes, `pip install` records the actual PTO-ISA git HEAD
used to build `host_runtime.so` in `build/lib/pto_isa_build.json`. The recorded
Expand All @@ -146,9 +140,9 @@ comparison:
HEAD.
2. `PTO_ISA_ROOT`'s current git HEAD, when `PTO_ISA_ROOT` points to a git
checkout and the recorded run-time commit is unavailable.
3. The resolved concrete request from `SIMPLER_PTO_ISA_COMMIT` or `pto_isa.pin`,
as a fallback for cases where no git checkout commit can be read. Explicit
`latest`/`head`/`none` without a readable git checkout remains unverifiable.
3. The resolved concrete request from `pto_isa.pin`, as a fallback for cases
where no git checkout commit can be read. Explicit `latest`/`head`/`none`
without a readable git checkout remains unverifiable.

If the build commit and run-time commit differ, `simpler` fails early on the
host side before loading the runtime binary. The error reports both commits and
Expand Down
10 changes: 5 additions & 5 deletions simpler_setup/pto_isa.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

Resolution order for ensure_pto_isa_root():
1. PTO_ISA_ROOT environment variable (if set and points to a directory)
2. Explicit commit argument / SIMPLER_PTO_ISA_COMMIT
2. Explicit commit argument (--pto-isa-commit CLI / API)
3. PROJECT_ROOT / pto_isa.pin
4. PROJECT_ROOT / build / pto-isa at origin/HEAD when explicitly unpinned
or when the pin is missing
Expand Down Expand Up @@ -76,11 +76,11 @@ def read_pto_isa_pin(pin_path: Optional[Path] = None) -> Optional[str]:
def resolve_pto_isa_commit(commit: Optional[str] = None) -> Optional[str]:
"""Resolve the pto-isa revision requested for managed clones.

Explicit CLI/API values win over SIMPLER_PTO_ISA_COMMIT. When neither is
provided, use pto_isa.pin. "latest", "head", "none", and an explicit empty
Python/API or environment value opt into the current remote HEAD behavior.
Explicit CLI/API values win over the repository pto_isa.pin. "latest",
"head", "none", and an explicit empty value opt into the current remote
HEAD behavior.
"""
requested = commit if commit is not None else os.environ.get("SIMPLER_PTO_ISA_COMMIT")
requested = commit
if requested is not None:
value = requested.strip()
if value.lower() in _UNPINNED_COMMIT_VALUES:
Expand Down
37 changes: 6 additions & 31 deletions tests/ut/py/test_pto_isa.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,50 +52,28 @@ def test_read_pto_isa_pin_rejects_invalid_content(tmp_path):
pto_isa.read_pto_isa_pin(pin)


def test_resolve_pto_isa_commit_prefers_explicit_over_env_and_pin(monkeypatch):
monkeypatch.setenv("SIMPLER_PTO_ISA_COMMIT", PIN_B)
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: PIN_C)

def test_resolve_pto_isa_commit_prefers_explicit_over_pin():
assert pto_isa.resolve_pto_isa_commit(PIN_A) == PIN_A


def test_resolve_pto_isa_commit_prefers_env_over_pin(monkeypatch):
monkeypatch.setenv("SIMPLER_PTO_ISA_COMMIT", PIN_B)
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: PIN_C)

assert pto_isa.resolve_pto_isa_commit() == PIN_B


def test_resolve_pto_isa_commit_uses_pin_by_default(monkeypatch):
monkeypatch.delenv("SIMPLER_PTO_ISA_COMMIT", raising=False)
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: PIN_C)

assert pto_isa.resolve_pto_isa_commit() == PIN_C


@pytest.mark.parametrize("value", ["latest", "head", "none", ""])
def test_resolve_pto_isa_commit_explicit_unpinned_values_return_none(value, monkeypatch):
monkeypatch.setenv("SIMPLER_PTO_ISA_COMMIT", PIN_B)
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: PIN_C)

def test_resolve_pto_isa_commit_explicit_unpinned_values_return_none(value):
assert pto_isa.resolve_pto_isa_commit(value) is None


def test_resolve_pto_isa_commit_env_latest_overrides_pin(monkeypatch):
monkeypatch.setenv("SIMPLER_PTO_ISA_COMMIT", "latest")
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: PIN_C)

assert pto_isa.resolve_pto_isa_commit() is None


def test_ensure_pto_isa_root_uses_pin_instead_of_latest_for_existing_clone(tmp_path, monkeypatch):
clone_path = tmp_path / "build" / "pto-isa"
(clone_path / "include").mkdir(parents=True)
checked_out = []
updated = []

monkeypatch.delenv("PTO_ISA_ROOT", raising=False)
monkeypatch.delenv("SIMPLER_PTO_ISA_COMMIT", raising=False)
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: PIN_A)
monkeypatch.setattr(pto_isa, "get_pto_isa_clone_path", lambda: clone_path)
monkeypatch.setattr(
Expand Down Expand Up @@ -236,7 +214,6 @@ def test_ensure_pto_isa_root_rejects_existing_clone_when_pin_checkout_fails(tmp_
updated = []

monkeypatch.delenv("PTO_ISA_ROOT", raising=False)
monkeypatch.delenv("SIMPLER_PTO_ISA_COMMIT", raising=False)
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: PIN_A)
monkeypatch.setattr(pto_isa, "get_pto_isa_clone_path", lambda: clone_path)
monkeypatch.setattr(pto_isa, "checkout_pto_isa_commit", lambda path, commit, verbose=False: False)
Expand All @@ -257,7 +234,6 @@ def clone_then_fail(target, commit, clone_protocol, verbose):
return False

monkeypatch.delenv("PTO_ISA_ROOT", raising=False)
monkeypatch.delenv("SIMPLER_PTO_ISA_COMMIT", raising=False)
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: PIN_A)
monkeypatch.setattr(pto_isa, "get_pto_isa_clone_path", lambda: clone_path)
monkeypatch.setattr(pto_isa, "_clone", clone_then_fail)
Expand Down Expand Up @@ -302,7 +278,6 @@ def test_validate_runtime_pto_isa_rejects_when_run_commit_unavailable(tmp_path,
)
monkeypatch.delenv("SIMPLER_RUN_PTO_ISA_COMMIT", raising=False)
monkeypatch.delenv("PTO_ISA_ROOT", raising=False)
monkeypatch.delenv("SIMPLER_PTO_ISA_COMMIT", raising=False)
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: None)

with pytest.raises(RuntimeError, match="Cannot verify PTO-ISA runtime revision"):
Expand All @@ -319,12 +294,12 @@ def test_validate_runtime_pto_isa_rejects_mismatch(tmp_path, monkeypatch):
pto_isa.validate_runtime_pto_isa_compatible(tmp_path)


def test_validate_runtime_pto_isa_uses_explicit_env_for_non_git_env_root(tmp_path, monkeypatch, caplog):
def test_validate_runtime_pto_isa_uses_pin_for_non_git_env_root(tmp_path, monkeypatch, caplog):
(tmp_path / pto_isa.PTO_ISA_BUILD_METADATA).write_text(
json.dumps({"schema_version": 1, "pto_isa_commit": "build_sha"}) + "\n"
)
monkeypatch.delenv("SIMPLER_RUN_PTO_ISA_COMMIT", raising=False)
monkeypatch.setenv("SIMPLER_PTO_ISA_COMMIT", "build_sha")
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: "build_sha")
monkeypatch.setenv("PTO_ISA_ROOT", str(tmp_path / "pto-isa"))
(tmp_path / "pto-isa").mkdir()
monkeypatch.setattr(pto_isa, "get_pto_isa_head", lambda root: "")
Expand All @@ -334,12 +309,12 @@ def test_validate_runtime_pto_isa_uses_explicit_env_for_non_git_env_root(tmp_pat
assert "falling back to resolved PTO-ISA commit" in caplog.text


def test_validate_runtime_pto_isa_rejects_latest_with_non_git_env_root(tmp_path, monkeypatch):
def test_validate_runtime_pto_isa_rejects_when_pin_missing_with_non_git_env_root(tmp_path, monkeypatch):
(tmp_path / pto_isa.PTO_ISA_BUILD_METADATA).write_text(
json.dumps({"schema_version": 1, "pto_isa_commit": "build_sha"}) + "\n"
)
monkeypatch.delenv("SIMPLER_RUN_PTO_ISA_COMMIT", raising=False)
monkeypatch.setenv("SIMPLER_PTO_ISA_COMMIT", "latest")
monkeypatch.setattr(pto_isa, "read_pto_isa_pin", lambda: None)
monkeypatch.setenv("PTO_ISA_ROOT", str(tmp_path / "pto-isa"))
(tmp_path / "pto-isa").mkdir()
monkeypatch.setattr(pto_isa, "get_pto_isa_head", lambda root: "")
Expand Down
Loading