diff --git a/.github/workflows/python-wheels.yaml b/.github/workflows/python-wheels.yaml index 524b6ae0a..1d3f5d42d 100644 --- a/.github/workflows/python-wheels.yaml +++ b/.github/workflows/python-wheels.yaml @@ -27,13 +27,23 @@ jobs: - name: Install capnp (Windows) if: runner.os == 'Windows' run: choco install capnproto + - name: Stage bundled client (macOS) + if: runner.os == 'macOS' + run: make stage-python-client + - name: Stage bundled client (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + cargo build --release --locked --bin gen + New-Item -ItemType Directory -Force "gen.gen.data/scripts" + Copy-Item "target/release/gen.exe" "gen.gen.data/scripts/gen.exe" - name: Build wheels (Linux) if: runner.os == 'Linux' uses: PyO3/maturin-action@v1 with: command: build args: --release --manifest-path gen-python/Cargo.toml --features extension-module --interpreter python${{ matrix.python-version }} --out gen-python/target/wheels - sccache: true manylinux: auto container: quay.io/pypa/manylinux_2_28_x86_64:latest before-script-linux: | @@ -45,6 +55,7 @@ jobs: mkdir -p gen-capnp-schemas/src/generated (cd gen-capnp-schemas && capnp compile -I . -orust:src/generated gen-core.capnp gen-models.capnp gen-schema.capnp) ls -la gen-capnp-schemas/src/generated + make stage-python-client working-directory: . env: CAPNP: /usr/bin/capnp @@ -58,6 +69,31 @@ jobs: args: --release --manifest-path gen-python/Cargo.toml --features extension-module --interpreter ${{ steps.setup-python.outputs.python-path }} --out gen-python/target/wheels sccache: true working-directory: . + - name: Verify wheel contents (macOS, Linux) + if: runner.os != 'Windows' + run: python gen-python/scripts/verify_wheel.py gen-python/target/wheels/*.whl + - name: Verify wheel contents (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $wheel = Get-ChildItem "gen-python/target/wheels/*.whl" + python gen-python/scripts/verify_wheel.py "$($wheel.FullName)" + - name: Smoke test wheel (macOS, Linux) + if: runner.os != 'Windows' + run: | + python -m pip install --force-reinstall --no-deps gen-python/target/wheels/*.whl + python -c "import gen; print(gen.__version__)" + gen --version + - name: Smoke test wheel (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $wheel = Get-ChildItem "gen-python/target/wheels/*.whl" + python -m pip install --force-reinstall --no-deps "$($wheel.FullName)" + python -c "import gen; print(gen.__version__)" + gen --version - name: Upload wheels to releases if: runner.os != 'Windows' env: diff --git a/.gitignore b/.gitignore index c1b12600e..722473361 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ gen-r/vignettes/*.html # Python bindings **/__pycache__/ gen-python/python/gen/*.so +/gen.gen.data/ # R package extension built from Rust gen-r/src/genr.so diff --git a/Cargo.lock b/Cargo.lock index 34ace980a..0cc64b406 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2658,7 +2658,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin 0.9.9", ] [[package]] @@ -2757,8 +2757,7 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libdoltlite-sys" version = "0.38.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbf72e2d56836b1c5cf51e40534a969b563994a1b726fe502ef8f8158366b32" +source = "git+https://github.com/bobvh/rusqdoltlite?rev=d1ef39b#d1ef39bb87239d67b41779c468380392edc9b1f3" dependencies = [ "cc", "pkg-config", @@ -4726,8 +4725,7 @@ dependencies = [ [[package]] name = "rusqdoltlite" version = "0.40.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6de2ed3a86b99876639d7e7af3ed30992a2f46a73131b5c9339c1c28d8aebea" +source = "git+https://github.com/bobvh/rusqdoltlite?rev=d1ef39b#d1ef39bb87239d67b41779c468380392edc9b1f3" dependencies = [ "bitflags 2.13.0", "fallible-iterator", @@ -5288,9 +5286,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spin" diff --git a/Cargo.toml b/Cargo.toml index ca0e48c1f..53aec3e92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,11 @@ members = [".", "gen-core", "gen-models", "gen-graph", "gen-diff", "gen-tui", "g default-members = [".", "gen-core", "gen-models", "gen-graph", "gen-diff", "gen-capnp-schemas", "gen-annotations"] exclude = ["gen-python", "gen-r/src/rust"] +[patch.crates-io] +# TEMPORARY: verifying the Windows MSVC build fix in CI before it's published +# to crates.io. Revert this patch before merging. +rusqdoltlite = { git = "https://github.com/bobvh/rusqdoltlite", rev = "d1ef39b" } + [features] benchmark = [] default = ["models", "cli", "diff", "remote"] diff --git a/Makefile b/Makefile index 0c3a0e8bf..6e63763c8 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,17 @@ -.PHONY: python jupyter r r-test release-check-js clean build clippy-fix docker-build gif +.PHONY: python python-wheel stage-python-client jupyter r r-test release-check-js clean build clippy-fix docker-build gif python: @[ -d .venv ] || python -m venv .venv @.venv/bin/pip show maturin >/dev/null 2>&1 || .venv/bin/pip install maturin .venv/bin/maturin develop --release --manifest-path gen-python/Cargo.toml --features extension-module +stage-python-client: + cargo build --release --locked --bin gen + mkdir -p gen.gen.data/scripts + cp target/release/gen gen.gen.data/scripts/gen + chmod +x gen.gen.data/scripts/gen +python-wheel: stage-python-client + @[ -d .venv ] || python -m venv .venv + @.venv/bin/pip show maturin >/dev/null 2>&1 || .venv/bin/pip install maturin + .venv/bin/maturin build --release --manifest-path gen-python/Cargo.toml --features extension-module # The jupyter widget requires a bundled JS file compiled from the TypeScript sources in gen-python/js/. # We check in the compiled jupyter_widget.js alongside the TS so npm is not required to build the widget. jupyter: python diff --git a/README.md b/README.md index 111b2868b..d545a3245 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,33 @@ Gen brings version control to genetic sequences. With it, you can track variants ## Install -**Gen client**: prebuilt binaries for macOS and Linux are on the [releases page](https://github.com/genhub-bio/gen/releases): [macOS (.pkg)](https://github.com/genhub-bio/gen/releases/download/nightly/gen.macos.pkg), [Linux x86_64 (.zip)](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-x86_64.zip), [Linux arm64 (.zip)](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-arm64.zip). Gen is built primarily for Unix-like systems; on Windows, you can install [WSL](https://learn.microsoft.com/en-us/windows/wsl/) to get a Linux environment, then use the Linux binary above from inside it. +Install Gen with pip: -**Python package**: install on macOS, Linux, or Windows using: ```sh pip install gen ``` -Install the `jupyter` extra to include an interactive graph widget for Jupyter and other anywidget-compatible notebooks: +Or install CLI in an isolated environment with uv: + +```sh +uv tool install gen +``` + +It is also available from crates.io: + +```sh +cargo install gen +``` + +Prebuilt binaries and installers are available on the [releases page](https://github.com/genhub-bio/gen/releases) for [macOS](https://github.com/genhub-bio/gen/releases/download/nightly/gen.macos.pkg) and Linux ([x64](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-x86_64.zip) / [arm64](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-arm64.zip)). Gen is built primarily for Unix-like systems; on Windows, you can install [WSL](https://learn.microsoft.com/en-us/windows/wsl/) to get a Linux environment, then use the Linux binary above from inside it. + +**Python package**: Installing Gen with pip also installs the Python package. Install the `jupyter` extra to include an interactive graph widget for Jupyter and other anywidget-compatible notebooks: + ```sh pip install gen[jupyter] ``` -**R package**: install on macOS (Apple silicon) using the `remotes` package: +**R package**: Install on macOS (Apple silicon) using the `remotes` package: ```r install.packages("remotes") remotes::install_url( diff --git a/gen-python/README.md b/gen-python/README.md index d46ae09e1..6978116d1 100644 --- a/gen-python/README.md +++ b/gen-python/README.md @@ -2,9 +2,10 @@ Python bindings to the Gen version control system for genetic sequences. -The bindings expose the full Gen data model — repositories, sequence graphs, -import/export pipelines — from Python and Jupyter notebooks. An optional Jupyter -widget provides interactive graph visualization. +The package installs the `gen` command-line client and exposes the full Gen data +model — repositories, sequence graphs, import/export pipelines — from Python and +Jupyter notebooks. An optional Jupyter widget provides interactive graph +visualization. ## Quick start @@ -29,6 +30,12 @@ sample.plot() # or sg.plot() The package is built from three layers: +### Client (`src/main.rs`) + +The existing Rust command-line client is compiled separately and staged in +maturin's wheel data `scripts` directory. Package installers place that executable +on `PATH` as `gen` on macOS and Linux or `gen.exe` on Windows. + ### Rust (`src/python_api/`) The core of the package. [PyO3](https://pyo3.rs) + [maturin](https://www.maturin.rs) @@ -72,6 +79,7 @@ Loaded by anywidget directly in the browser. Responsible for: ```sh make # from the project root — builds the native extension via maturin +make python-wheel # builds a wheel containing the extension and client make jupyter # also builds the JS widget bundle and installs the `jupyter` extras ``` diff --git a/gen-python/scripts/verify_wheel.py b/gen-python/scripts/verify_wheel.py new file mode 100644 index 000000000..9cfc84b81 --- /dev/null +++ b/gen-python/scripts/verify_wheel.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Verify that a Gen wheel contains the Python package and bundled client.""" + +import sys +import zipfile +from pathlib import Path + + +def fail(message: str) -> None: + raise SystemExit(message) + + +def verify_wheel(wheel_path: Path) -> None: + client_name = "gen.exe" if "-win" in wheel_path.name else "gen" + + with zipfile.ZipFile(wheel_path) as wheel: + names = {entry.filename for entry in wheel.infolist()} + + if not any(name.endswith(f".data/scripts/{client_name}") for name in names): + fail(f"{wheel_path} should contain the bundled {client_name}") + + extension_suffixes = (".pyd", ".so") + if not any( + name.startswith("gen/") and name.endswith(extension_suffixes) + for name in names + ): + fail(f"{wheel_path} should contain the compiled gen extension") + + if "gen/static/jupyter_widget.js" not in names: + fail(f"{wheel_path} should contain the Jupyter widget asset") + + print(f"Verified bundled client and Python package in {wheel_path}") + + +def main() -> None: + if len(sys.argv) != 2: + fail(f"usage: {Path(sys.argv[0]).name} WHEEL") + + wheel_path = Path(sys.argv[1]) + if not wheel_path.is_file(): + fail(f"wheel does not exist: {wheel_path}") + + verify_wheel(wheel_path) + + +if __name__ == "__main__": + main()