From e729f2f509f829039aed429eebbc1453d90a577e Mon Sep 17 00:00:00 2001 From: Bob Van Hove <1587584+bobvh@users.noreply.github.com> Date: Thu, 30 Jul 2026 05:16:46 +0200 Subject: [PATCH 1/4] Package the gen CLI in Python wheels Stage the Rust CLI in the Python package so pip, uv, and pipx install the gen command. Build abi3 wheels for Linux x86_64/aarch64, macOS arm64/x86_64, and Windows, then test them on Python 3.12 through 3.14. Sign and notarize macOS wheels for tagged releases. Upload release wheels after compatibility tests pass and send PyPI the exact producing run ID. Disable maturin sccache on Linux because before-script invokes Cargo before sccache is installed in the container. Keep manual tagged binary releases and nightly builds. --- .github/workflows/publish-pypi-tag.yaml | 112 ++++++++++---- .github/workflows/python-wheels.yaml | 190 +++++++++++++++++++++--- .github/workflows/release.yml | 26 +++- .gitignore | 1 + Cargo.lock | 6 +- Makefile | 15 +- README.md | 22 ++- gen-python/Cargo.toml | 1 + gen-python/README.md | 18 ++- gen-python/scripts/verify_wheel.py | 50 +++++++ pyproject.toml | 2 +- 11 files changed, 379 insertions(+), 64 deletions(-) create mode 100644 gen-python/scripts/verify_wheel.py diff --git a/.github/workflows/publish-pypi-tag.yaml b/.github/workflows/publish-pypi-tag.yaml index 3b1006816..f78d0c315 100644 --- a/.github/workflows/publish-pypi-tag.yaml +++ b/.github/workflows/publish-pypi-tag.yaml @@ -1,15 +1,17 @@ name: Publish PyPI Tag on: - push: - tags: - - "v*" + # Python wheels dispatch this workflow only after release gating succeeds. workflow_dispatch: inputs: tag: description: Git tag to publish (e.g. v0.4.0) required: true type: string + wheel_run_id: + description: GitHub Actions run containing the tested wheel artifacts + required: true + type: string repository: description: Target repository required: false @@ -20,7 +22,8 @@ on: - testpypi env: - PUBLISH_TAG: ${{ github.event.inputs.tag || github.ref_name || (github.ref_type == 'tag' && github.ref_name) || '' }} + PUBLISH_TAG: ${{ inputs.tag }} + WHEEL_RUN_ID: ${{ inputs.wheel_run_id }} jobs: publish: @@ -28,53 +31,108 @@ jobs: environment: name: pypi permissions: + actions: read id-token: write contents: read steps: - name: Ensure tag provided run: | - if [ -z "${{ env.PUBLISH_TAG }}" ]; then - echo "PUBLISH_TAG is required. Run on a tag or provide the workflow_dispatch input 'tag'." >&2 - exit 1 - fi + case "$PUBLISH_TAG" in + v*) ;; + *) + echo "PUBLISH_TAG must start with 'v'." >&2 + exit 1 + ;; + esac - - name: Locate wheel build run - id: wheels-run + - name: Validate wheel build run uses: actions/github-script@v7 env: - WHEEL_SHA: ${{ github.sha }} + WHEEL_RUN_ID: ${{ env.WHEEL_RUN_ID }} WHEEL_TAG: ${{ env.PUBLISH_TAG }} with: script: | - const sha = process.env.WHEEL_SHA; + const runId = Number(process.env.WHEEL_RUN_ID); const tag = process.env.WHEEL_TAG; - const { data } = await github.rest.actions.listWorkflowRuns({ + + if (!Number.isSafeInteger(runId) || runId <= 0) { + core.setFailed(`Invalid wheel run ID: ${process.env.WHEEL_RUN_ID}`); + return; + } + + let run; + for (let attempt = 1; attempt <= 12; attempt += 1) { + const response = await github.rest.actions.getWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: runId, + }); + run = response.data; + if (run.status === "completed") { + break; + } + core.info( + `Wheel run ${runId} is ${run.status}; waiting for completion (${attempt}/12).` + ); + await new Promise((resolve) => setTimeout(resolve, 5000)); + } + + if (run.path !== ".github/workflows/python-wheels.yaml") { + core.setFailed( + `Run ${runId} belongs to ${run.path}, not python-wheels.yaml.` + ); + return; + } + if (run.status !== "completed" || run.conclusion !== "success") { + core.setFailed( + `Wheel run ${runId} for tag ${tag} did not complete successfully (status: ${run.status}, conclusion: ${run.conclusion}).` + ); + return; + } + + const jobsResponse = + await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: runId, + per_page: 100, + }); + const uploadJob = jobsResponse.data.jobs.find( + (job) => job.name === "upload-release-wheels" + ); + if (!uploadJob || uploadJob.conclusion !== "success") { + core.setFailed( + `Wheel run ${runId} did not successfully upload its tested release wheels.` + ); + return; + } + + const tagResponse = await github.rest.git.getRef({ owner: context.repo.owner, repo: context.repo.repo, - workflow_id: "python-wheels.yaml", - head_sha: sha, - per_page: 100, + ref: `tags/${tag}`, }); - const run = data.workflow_runs.find( - (r) => - r.head_sha === sha && - r.status === "completed" && - r.conclusion === "success" - ); - if (!run) { + let tagTarget = tagResponse.data.object; + for (let depth = 0; tagTarget.type === "tag" && depth < 5; depth += 1) { + const annotatedTag = await github.rest.git.getTag({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_sha: tagTarget.sha, + }); + tagTarget = annotatedTag.data.object; + } + if (tagTarget.type !== "commit" || tagTarget.sha !== run.head_sha) { core.setFailed( - `No successful python-wheels run found for tag ${tag} (sha ${sha}). Trigger the wheel build for this tag first.` + `Tag ${tag} does not point to wheel run ${runId} commit ${run.head_sha}.` ); - return; } - core.setOutput("run_id", run.id.toString()); - name: Download wheel artifacts uses: actions/download-artifact@v4 with: pattern: wheels-* path: wheelhouse - run-id: ${{ steps.wheels-run.outputs.run_id }} + run-id: ${{ env.WHEEL_RUN_ID }} github-token: ${{ secrets.GITHUB_TOKEN }} - name: Collect wheels diff --git a/.github/workflows/python-wheels.yaml b/.github/workflows/python-wheels.yaml index 524b6ae0a..70cad6101 100644 --- a/.github/workflows/python-wheels.yaml +++ b/.github/workflows/python-wheels.yaml @@ -1,6 +1,10 @@ name: Python API Wheels on: + push: + branches: ["main"] + pull_request: + branches: ["**"] workflow_dispatch: jobs: @@ -11,15 +15,19 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.11", "3.12", "3.13", "3.14"] + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-15-intel, windows-latest] + include: + - os: ubuntu-latest + manylinux: quay.io/pypa/manylinux_2_28_x86_64:latest + - os: ubuntu-24.04-arm + manylinux: quay.io/pypa/manylinux_2_28_aarch64:latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 id: setup-python with: - python-version: ${{ matrix.python-version }} + python-version: "3.11" - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Install capnp (macOS) if: runner.os == 'macOS' @@ -27,15 +35,80 @@ jobs: - name: Install capnp (Windows) if: runner.os == 'Windows' run: choco install capnproto + - name: Stage bundled client (macOS) + if: runner.os == 'macOS' + run: make stage-python-client + - name: Sign bundled client (macOS) + if: runner.os == 'macOS' && github.ref_type == 'tag' + env: + MACOS_CERTIFICATE: ${{ secrets.PROD_MACOS_APP_CERTIFICATE }} + MACOS_CERTIFICATE_PWD: ${{ secrets.PROD_MACOS_APP_CERTIFICATE_PWD }} + MACOS_CERTIFICATE_NAME: ${{ secrets.PROD_MACOS_APP_CERTIFICATE_NAME }} + MACOS_CI_KEYCHAIN_PWD: ${{ secrets.PROD_MACOS_CI_KEYCHAIN_PWD }} + run: | + CERTIFICATE_PATH="$RUNNER_TEMP/gen-application-certificate.p12" + KEYCHAIN_PATH="$RUNNER_TEMP/gen-application-signing.keychain-db" + echo "$MACOS_CERTIFICATE" | base64 --decode > "$CERTIFICATE_PATH" + security create-keychain -p "$MACOS_CI_KEYCHAIN_PWD" "$KEYCHAIN_PATH" + security default-keychain -s "$KEYCHAIN_PATH" + security unlock-keychain -p "$MACOS_CI_KEYCHAIN_PWD" "$KEYCHAIN_PATH" + security import "$CERTIFICATE_PATH" \ + -k "$KEYCHAIN_PATH" \ + -P "$MACOS_CERTIFICATE_PWD" \ + -T /usr/bin/codesign + security set-key-partition-list \ + -S apple-tool:,apple:,codesign: \ + -s \ + -k "$MACOS_CI_KEYCHAIN_PWD" \ + "$KEYCHAIN_PATH" + codesign \ + --force \ + --timestamp \ + --options runtime \ + --sign "$MACOS_CERTIFICATE_NAME" \ + gen.gen.data/scripts/gen + codesign --verify --strict --verbose=2 gen.gen.data/scripts/gen + security delete-keychain "$KEYCHAIN_PATH" + - name: Notarize bundled client (macOS) + if: runner.os == 'macOS' && github.ref_type == 'tag' + env: + MACOS_CI_KEYCHAIN_PWD: ${{ secrets.PROD_MACOS_CI_KEYCHAIN_PWD }} + PROD_MACOS_NOTARIZATION_APPLE_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_APPLE_ID }} + PROD_MACOS_NOTARIZATION_TEAM_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_TEAM_ID }} + PROD_MACOS_NOTARIZATION_PWD: ${{ secrets.PROD_MACOS_NOTARIZATION_PWD }} + run: | + KEYCHAIN_PATH="$RUNNER_TEMP/gen-notarization.keychain-db" + NOTARIZATION_ARCHIVE="$RUNNER_TEMP/gen-client.zip" + ditto -c -k --keepParent gen.gen.data/scripts/gen "$NOTARIZATION_ARCHIVE" + security create-keychain -p "$MACOS_CI_KEYCHAIN_PWD" "$KEYCHAIN_PATH" + security default-keychain -s "$KEYCHAIN_PATH" + security unlock-keychain -p "$MACOS_CI_KEYCHAIN_PWD" "$KEYCHAIN_PATH" + security list-keychain -d user -s "$KEYCHAIN_PATH" + xcrun notarytool store-credentials "gen-notarytool-profile" \ + --apple-id "$PROD_MACOS_NOTARIZATION_APPLE_ID" \ + --team-id "$PROD_MACOS_NOTARIZATION_TEAM_ID" \ + --password "$PROD_MACOS_NOTARIZATION_PWD" \ + --keychain "$KEYCHAIN_PATH" + xcrun notarytool submit "$NOTARIZATION_ARCHIVE" \ + --keychain-profile "gen-notarytool-profile" \ + --wait + security delete-keychain "$KEYCHAIN_PATH" + - name: Stage bundled client (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + cargo build --release --locked --bin gen + New-Item -ItemType Directory -Force "gen.gen.data/scripts" + Copy-Item "target/release/gen.exe" "gen.gen.data/scripts/gen.exe" - name: Build wheels (Linux) if: runner.os == 'Linux' uses: PyO3/maturin-action@v1 with: command: build - args: --release --manifest-path gen-python/Cargo.toml --features extension-module --interpreter python${{ matrix.python-version }} --out gen-python/target/wheels - sccache: true + args: --release --manifest-path gen-python/Cargo.toml --features abi3,extension-module --interpreter python3.11 --out gen-python/target/wheels manylinux: auto - container: quay.io/pypa/manylinux_2_28_x86_64:latest + container: ${{ matrix.manylinux }} before-script-linux: | yum install -y epel-release yum install -y clang llvm-devel sqlite-devel capnproto capnproto-devel pkgconfig mold @@ -45,6 +118,7 @@ jobs: mkdir -p gen-capnp-schemas/src/generated (cd gen-capnp-schemas && capnp compile -I . -orust:src/generated gen-core.capnp gen-models.capnp gen-schema.capnp) ls -la gen-capnp-schemas/src/generated + make stage-python-client working-directory: . env: CAPNP: /usr/bin/capnp @@ -55,35 +129,104 @@ jobs: uses: PyO3/maturin-action@v1 with: command: build - args: --release --manifest-path gen-python/Cargo.toml --features extension-module --interpreter ${{ steps.setup-python.outputs.python-path }} --out gen-python/target/wheels + args: --release --manifest-path gen-python/Cargo.toml --features abi3,extension-module --interpreter ${{ steps.setup-python.outputs.python-path }} --out gen-python/target/wheels sccache: true working-directory: . - - name: Upload wheels to releases + - name: Verify wheel contents (macOS, Linux) + if: runner.os != 'Windows' + run: python gen-python/scripts/verify_wheel.py gen-python/target/wheels/*.whl + - name: Verify wheel contents (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $wheel = Get-ChildItem "gen-python/target/wheels/*.whl" + python gen-python/scripts/verify_wheel.py "$($wheel.FullName)" + - name: Verify bundled client signature (macOS) + if: runner.os == 'macOS' && github.ref_type == 'tag' + run: | + CHECK_DIRECTORY="$RUNNER_TEMP/gen-wheel-check" + mkdir -p "$CHECK_DIRECTORY" + unzip -q gen-python/target/wheels/*.whl -d "$CHECK_DIRECTORY" + CLIENT_PATH=$(find "$CHECK_DIRECTORY" -type f -path "*/scripts/gen" -print -quit) + test -n "$CLIENT_PATH" + codesign --verify --strict --verbose=2 "$CLIENT_PATH" + - name: Smoke test wheel (macOS, Linux) if: runner.os != 'Windows' - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} run: | - gh release upload "${TAG}" gen-python/target/wheels/*.whl - - name: Upload wheels to releases (Windows) + python -m pip install --force-reinstall --no-deps gen-python/target/wheels/*.whl + python -c "import gen; print(gen.__version__)" + gen --version + - name: Smoke test wheel (Windows) if: runner.os == 'Windows' shell: pwsh - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} run: | $ErrorActionPreference = 'Stop' - gh release upload "$env:TAG" "gen-python/target/wheels/*.whl" + $wheel = Get-ChildItem "gen-python/target/wheels/*.whl" + python -m pip install --force-reinstall --no-deps "$($wheel.FullName)" + python -c "import gen; print(gen.__version__)" + gen --version - uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-py${{ matrix.python-version }} + name: wheels-${{ matrix.os }} path: gen-python/target/wheels/*.whl if-no-files-found: error - trigger-pypi-publish: + test-newer-python: needs: build-wheels + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-15-intel, windows-latest] + python-version: ["3.12", "3.13", "3.14"] + + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: wheelhouse + - name: Smoke test wheel (macOS, Linux) + if: runner.os != 'Windows' + run: | + python -m pip install --force-reinstall --no-deps wheelhouse/*.whl + python -c "import gen; print(gen.__version__)" + gen --version + - name: Smoke test wheel (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $wheel = Get-ChildItem "wheelhouse/*.whl" + python -m pip install --force-reinstall --no-deps "$($wheel.FullName)" + python -c "import gen; print(gen.__version__)" + gen --version + + upload-release-wheels: + needs: [build-wheels, test-newer-python] + runs-on: ubuntu-latest + if: ${{ needs.build-wheels.result == 'success' && needs.test-newer-python.result == 'success' && github.ref_type == 'tag' }} + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: wheels-* + path: wheelhouse + merge-multiple: true + - name: Upload wheels to release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: gh release upload "$TAG" wheelhouse/*.whl --clobber + + trigger-pypi-publish: + needs: upload-release-wheels runs-on: ubuntu-latest - if: ${{ needs.build-wheels.result == 'success' }} + if: ${{ needs.upload-release-wheels.result == 'success' && github.ref_type == 'tag' }} permissions: actions: write contents: read @@ -91,6 +234,7 @@ jobs: TAG: ${{ github.ref_name }} GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} + WHEEL_RUN_ID: ${{ github.run_id }} steps: - name: Ensure tag is available run: | @@ -100,4 +244,8 @@ jobs: fi - name: Trigger publish-pypi-tag workflow run: | - gh workflow run publish-pypi-tag.yaml --repo "$REPO" --ref "$TAG" -f tag="$TAG" + gh workflow run publish-pypi-tag.yaml \ + --repo "$REPO" \ + --ref "$TAG" \ + -f tag="$TAG" \ + -f wheel_run_id="$WHEEL_RUN_ID" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 08e72ab11..3e76e6e00 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -138,9 +138,33 @@ jobs: gh release upload "$RELEASE_NAME" gen.linux-x86_64.zip --clobber gh release upload "$RELEASE_NAME" gen.linux-arm64.zip --clobber + release-windows: + runs-on: windows-latest + permissions: + actions: write + contents: write + + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Install Dependencies + run: choco install capnproto + - name: Build + run: cargo build --release --locked --bin gen + - name: Ensure release exists + shell: bash + run: | + gh release view "$RELEASE_NAME" >/dev/null 2>&1 || gh release create "$RELEASE_NAME" --notes "" || gh release view "$RELEASE_NAME" + - name: upload-binary + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + Compress-Archive -Path target/release/gen.exe -DestinationPath gen.windows-x86_64.zip + gh release upload "$env:RELEASE_NAME" gen.windows-x86_64.zip --clobber + update-release-source: runs-on: ubuntu-latest - needs: [release-macos, release-ubuntu] + needs: [release-macos, release-ubuntu, release-windows] permissions: actions: write contents: write diff --git a/.gitignore b/.gitignore index c1b12600e..722473361 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ gen-r/vignettes/*.html # Python bindings **/__pycache__/ gen-python/python/gen/*.so +/gen.gen.data/ # R package extension built from Rust gen-r/src/genr.so diff --git a/Cargo.lock b/Cargo.lock index 34ace980a..00c35d49c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2658,7 +2658,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin 0.9.9", ] [[package]] @@ -5288,9 +5288,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spin" diff --git a/Makefile b/Makefile index 0c3a0e8bf..6ed0d57f5 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,17 @@ -.PHONY: python jupyter r r-test release-check-js clean build clippy-fix docker-build gif +.PHONY: python python-wheel stage-python-client jupyter r r-test release-check-js clean build clippy-fix docker-build gif python: @[ -d .venv ] || python -m venv .venv @.venv/bin/pip show maturin >/dev/null 2>&1 || .venv/bin/pip install maturin - .venv/bin/maturin develop --release --manifest-path gen-python/Cargo.toml --features extension-module + .venv/bin/maturin develop --release --manifest-path gen-python/Cargo.toml --features abi3,extension-module +stage-python-client: + cargo build --release --locked --bin gen + mkdir -p gen.gen.data/scripts + cp target/release/gen gen.gen.data/scripts/gen + chmod +x gen.gen.data/scripts/gen +python-wheel: stage-python-client + @[ -d .venv ] || python -m venv .venv + @.venv/bin/pip show maturin >/dev/null 2>&1 || .venv/bin/pip install maturin + .venv/bin/maturin build --release --manifest-path gen-python/Cargo.toml --features abi3,extension-module # The jupyter widget requires a bundled JS file compiled from the TypeScript sources in gen-python/js/. # We check in the compiled jupyter_widget.js alongside the TS so npm is not required to build the widget. jupyter: python @@ -13,7 +22,7 @@ jupyter: python test -f gen-python/python/gen/static/jupyter_widget.js || \ (echo "Error: gen-python/python/gen/static/jupyter_widget.js missing. Install npm and run 'make jupyter'." && exit 1); \ fi - .venv/bin/maturin develop --release --manifest-path gen-python/Cargo.toml --features extension-module --extras jupyter + .venv/bin/maturin develop --release --manifest-path gen-python/Cargo.toml --features abi3,extension-module --extras jupyter r-test: # CI only: builds and tests inside Linux Docker container @if command -v npm >/dev/null 2>&1; then \ diff --git a/README.md b/README.md index 111b2868b..b510b17a6 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,33 @@ Gen brings version control to genetic sequences. With it, you can track variants ## Install -**Gen client**: prebuilt binaries for macOS and Linux are on the [releases page](https://github.com/genhub-bio/gen/releases): [macOS (.pkg)](https://github.com/genhub-bio/gen/releases/download/nightly/gen.macos.pkg), [Linux x86_64 (.zip)](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-x86_64.zip), [Linux arm64 (.zip)](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-arm64.zip). Gen is built primarily for Unix-like systems; on Windows, you can install [WSL](https://learn.microsoft.com/en-us/windows/wsl/) to get a Linux environment, then use the Linux binary above from inside it. +Install Gen with pip: -**Python package**: install on macOS, Linux, or Windows using: ```sh pip install gen ``` -Install the `jupyter` extra to include an interactive graph widget for Jupyter and other anywidget-compatible notebooks: +Or install CLI in an isolated environment with uv: + +```sh +uv tool install gen +``` + +It is also available from crates.io: + +```sh +cargo install gen +``` + +Prebuilt binaries and installers are available on the [releases page](https://github.com/genhub-bio/gen/releases) for [macOS](https://github.com/genhub-bio/gen/releases/download/nightly/gen.macos.pkg), Linux ([x64](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-x86_64.zip) / [arm64](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-arm64.zip)), and [Windows](https://github.com/genhub-bio/gen/releases/download/nightly/gen.windows-x86_64.zip). + +**Python package**: Installing Gen with pip also installs the Python package. Install the `jupyter` extra to include an interactive graph widget for Jupyter and other anywidget-compatible notebooks: + ```sh pip install gen[jupyter] ``` -**R package**: install on macOS (Apple silicon) using the `remotes` package: +**R package**: Install on macOS (Apple silicon) using the `remotes` package: ```r install.packages("remotes") remotes::install_url( diff --git a/gen-python/Cargo.toml b/gen-python/Cargo.toml index a6afcdcfb..d619b6a35 100644 --- a/gen-python/Cargo.toml +++ b/gen-python/Cargo.toml @@ -14,6 +14,7 @@ crate-type = ["cdylib"] [features] default = [] +abi3 = ["pyo3/abi3-py311"] extension-module = ["pyo3/extension-module"] [dependencies] diff --git a/gen-python/README.md b/gen-python/README.md index d46ae09e1..bfa710d06 100644 --- a/gen-python/README.md +++ b/gen-python/README.md @@ -2,9 +2,10 @@ Python bindings to the Gen version control system for genetic sequences. -The bindings expose the full Gen data model — repositories, sequence graphs, -import/export pipelines — from Python and Jupyter notebooks. An optional Jupyter -widget provides interactive graph visualization. +The package installs the `gen` command-line client and exposes the full Gen data +model — repositories, sequence graphs, import/export pipelines — from Python and +Jupyter notebooks. An optional Jupyter widget provides interactive graph +visualization. ## Quick start @@ -29,10 +30,18 @@ sample.plot() # or sg.plot() The package is built from three layers: +### Client (`src/main.rs`) + +The existing Rust command-line client is compiled separately and staged in +maturin's wheel data `scripts` directory. Package installers place that executable +on `PATH` as `gen` on macOS and Linux or `gen.exe` on Windows. + ### Rust (`src/python_api/`) The core of the package. [PyO3](https://pyo3.rs) + [maturin](https://www.maturin.rs) -compile the Gen engine into a native extension module (`gen.so`). This layer owns: +compile the Gen engine into a native extension module (`gen.so`). Release wheels +use CPython's stable ABI with Python 3.11 as the minimum supported version. This +layer owns: - **`Repository`** — opens a Gen workspace, drives all import/export operations (FASTA, GenBank, GFA, VCF, GAF, …), and exposes node/sample/sequence-graph @@ -72,6 +81,7 @@ Loaded by anywidget directly in the browser. Responsible for: ```sh make # from the project root — builds the native extension via maturin +make python-wheel # builds a wheel containing the extension and client make jupyter # also builds the JS widget bundle and installs the `jupyter` extras ``` diff --git a/gen-python/scripts/verify_wheel.py b/gen-python/scripts/verify_wheel.py new file mode 100644 index 000000000..50a24bd78 --- /dev/null +++ b/gen-python/scripts/verify_wheel.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Verify that a Gen wheel contains the Python package and bundled client.""" + +import sys +import zipfile +from pathlib import Path + + +def fail(message: str) -> None: + raise SystemExit(message) + + +def verify_wheel(wheel_path: Path) -> None: + client_name = "gen.exe" if "-win" in wheel_path.name else "gen" + + if "-abi3-" not in wheel_path.name: + fail(f"{wheel_path} should be tagged for the CPython stable ABI") + + with zipfile.ZipFile(wheel_path) as wheel: + names = {entry.filename for entry in wheel.infolist()} + + if not any(name.endswith(f".data/scripts/{client_name}") for name in names): + fail(f"{wheel_path} should contain the bundled {client_name}") + + extension_suffixes = (".pyd", ".so") + if not any( + name.startswith("gen/") and name.endswith(extension_suffixes) + for name in names + ): + fail(f"{wheel_path} should contain the compiled gen extension") + + if "gen/static/jupyter_widget.js" not in names: + fail(f"{wheel_path} should contain the Jupyter widget asset") + + print(f"Verified bundled client and Python package in {wheel_path}") + + +def main() -> None: + if len(sys.argv) != 2: + fail(f"usage: {Path(sys.argv[0]).name} WHEEL") + + wheel_path = Path(sys.argv[1]) + if not wheel_path.is_file(): + fail(f"wheel does not exist: {wheel_path}") + + verify_wheel(wheel_path) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 597a1c9b1..6c8969cb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,6 @@ Repository = "https://github.com/genhub-bio/gen" [tool.maturin] manifest-path = "gen-python/Cargo.toml" -features = ["extension-module"] +features = ["abi3", "extension-module"] python-source = "gen-python/python" module-name = "gen.gen" From 65008a3d825912150efe4be4faeb9547a192b9ea Mon Sep 17 00:00:00 2001 From: Bob Van Hove <1587584+bobvh@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:04:24 +0200 Subject: [PATCH 2/4] TEMPORARY: pin rusqdoltlite to Windows MSVC build fix for CI verification Points at bobvh/rusqdoltlite@812ffae (fix-windows branch), which excludes Windows from the bundled remote-auth server build to avoid the pthread_t/struct DoltliteServer parse failure under MSVC. Revert this patch once the fix ships in a published crates.io release. --- Cargo.lock | 6 ++---- Cargo.toml | 5 +++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00c35d49c..0cc64b406 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2757,8 +2757,7 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libdoltlite-sys" version = "0.38.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbf72e2d56836b1c5cf51e40534a969b563994a1b726fe502ef8f8158366b32" +source = "git+https://github.com/bobvh/rusqdoltlite?rev=d1ef39b#d1ef39bb87239d67b41779c468380392edc9b1f3" dependencies = [ "cc", "pkg-config", @@ -4726,8 +4725,7 @@ dependencies = [ [[package]] name = "rusqdoltlite" version = "0.40.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6de2ed3a86b99876639d7e7af3ed30992a2f46a73131b5c9339c1c28d8aebea" +source = "git+https://github.com/bobvh/rusqdoltlite?rev=d1ef39b#d1ef39bb87239d67b41779c468380392edc9b1f3" dependencies = [ "bitflags 2.13.0", "fallible-iterator", diff --git a/Cargo.toml b/Cargo.toml index ca0e48c1f..53aec3e92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,11 @@ members = [".", "gen-core", "gen-models", "gen-graph", "gen-diff", "gen-tui", "g default-members = [".", "gen-core", "gen-models", "gen-graph", "gen-diff", "gen-capnp-schemas", "gen-annotations"] exclude = ["gen-python", "gen-r/src/rust"] +[patch.crates-io] +# TEMPORARY: verifying the Windows MSVC build fix in CI before it's published +# to crates.io. Revert this patch before merging. +rusqdoltlite = { git = "https://github.com/bobvh/rusqdoltlite", rev = "d1ef39b" } + [features] benchmark = [] default = ["models", "cli", "diff", "remote"] From 8dc161871e236ee94a1ece2b200cabedd6bba338 Mon Sep 17 00:00:00 2001 From: Bob Van Hove <1587584+bobvh@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:06:24 +0200 Subject: [PATCH 3/4] TEMPORARY: verify macOS signing/notarization in CI without a release tag Drops the github.ref_type == 'tag' gate on the Sign/Notarize/Verify-signature steps so they run on this branch's CI. upload-release-wheels and trigger-pypi-publish keep their tag-only gates untouched, so this can't publish anything. Revert before merging. --- .github/workflows/python-wheels.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-wheels.yaml b/.github/workflows/python-wheels.yaml index 70cad6101..1874e7d46 100644 --- a/.github/workflows/python-wheels.yaml +++ b/.github/workflows/python-wheels.yaml @@ -39,7 +39,9 @@ jobs: if: runner.os == 'macOS' run: make stage-python-client - name: Sign bundled client (macOS) - if: runner.os == 'macOS' && github.ref_type == 'tag' + # TEMPORARY: dropped the `github.ref_type == 'tag'` gate to verify signing + # works in CI without cutting a release tag. Restore the gate before merging. + if: runner.os == 'macOS' env: MACOS_CERTIFICATE: ${{ secrets.PROD_MACOS_APP_CERTIFICATE }} MACOS_CERTIFICATE_PWD: ${{ secrets.PROD_MACOS_APP_CERTIFICATE_PWD }} @@ -70,7 +72,8 @@ jobs: codesign --verify --strict --verbose=2 gen.gen.data/scripts/gen security delete-keychain "$KEYCHAIN_PATH" - name: Notarize bundled client (macOS) - if: runner.os == 'macOS' && github.ref_type == 'tag' + # TEMPORARY: see Sign bundled client (macOS) above. Restore the gate before merging. + if: runner.os == 'macOS' env: MACOS_CI_KEYCHAIN_PWD: ${{ secrets.PROD_MACOS_CI_KEYCHAIN_PWD }} PROD_MACOS_NOTARIZATION_APPLE_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_APPLE_ID }} @@ -143,7 +146,8 @@ jobs: $wheel = Get-ChildItem "gen-python/target/wheels/*.whl" python gen-python/scripts/verify_wheel.py "$($wheel.FullName)" - name: Verify bundled client signature (macOS) - if: runner.os == 'macOS' && github.ref_type == 'tag' + # TEMPORARY: see Sign bundled client (macOS) above. Restore the gate before merging. + if: runner.os == 'macOS' run: | CHECK_DIRECTORY="$RUNNER_TEMP/gen-wheel-check" mkdir -p "$CHECK_DIRECTORY" From 9018efef0e529fdb9ae120d6263f319035cb93b7 Mon Sep 17 00:00:00 2001 From: Bob Van Hove <1587584+bobvh@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:28:34 +0200 Subject: [PATCH 4/4] Revert "TEMPORARY: verify macOS signing/notarization in CI without a release tag" This reverts commit 8dc161871e236ee94a1ece2b200cabedd6bba338. --- .github/workflows/python-wheels.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-wheels.yaml b/.github/workflows/python-wheels.yaml index 1874e7d46..70cad6101 100644 --- a/.github/workflows/python-wheels.yaml +++ b/.github/workflows/python-wheels.yaml @@ -39,9 +39,7 @@ jobs: if: runner.os == 'macOS' run: make stage-python-client - name: Sign bundled client (macOS) - # TEMPORARY: dropped the `github.ref_type == 'tag'` gate to verify signing - # works in CI without cutting a release tag. Restore the gate before merging. - if: runner.os == 'macOS' + if: runner.os == 'macOS' && github.ref_type == 'tag' env: MACOS_CERTIFICATE: ${{ secrets.PROD_MACOS_APP_CERTIFICATE }} MACOS_CERTIFICATE_PWD: ${{ secrets.PROD_MACOS_APP_CERTIFICATE_PWD }} @@ -72,8 +70,7 @@ jobs: codesign --verify --strict --verbose=2 gen.gen.data/scripts/gen security delete-keychain "$KEYCHAIN_PATH" - name: Notarize bundled client (macOS) - # TEMPORARY: see Sign bundled client (macOS) above. Restore the gate before merging. - if: runner.os == 'macOS' + if: runner.os == 'macOS' && github.ref_type == 'tag' env: MACOS_CI_KEYCHAIN_PWD: ${{ secrets.PROD_MACOS_CI_KEYCHAIN_PWD }} PROD_MACOS_NOTARIZATION_APPLE_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_APPLE_ID }} @@ -146,8 +143,7 @@ jobs: $wheel = Get-ChildItem "gen-python/target/wheels/*.whl" python gen-python/scripts/verify_wheel.py "$($wheel.FullName)" - name: Verify bundled client signature (macOS) - # TEMPORARY: see Sign bundled client (macOS) above. Restore the gate before merging. - if: runner.os == 'macOS' + if: runner.os == 'macOS' && github.ref_type == 'tag' run: | CHECK_DIRECTORY="$RUNNER_TEMP/gen-wheel-check" mkdir -p "$CHECK_DIRECTORY"