diff --git a/.github/workflows/publish-pypi-tag.yaml b/.github/workflows/publish-pypi-tag.yaml index 3b1006816..f78d0c315 100644 --- a/.github/workflows/publish-pypi-tag.yaml +++ b/.github/workflows/publish-pypi-tag.yaml @@ -1,15 +1,17 @@ name: Publish PyPI Tag on: - push: - tags: - - "v*" + # Python wheels dispatch this workflow only after release gating succeeds. workflow_dispatch: inputs: tag: description: Git tag to publish (e.g. v0.4.0) required: true type: string + wheel_run_id: + description: GitHub Actions run containing the tested wheel artifacts + required: true + type: string repository: description: Target repository required: false @@ -20,7 +22,8 @@ on: - testpypi env: - PUBLISH_TAG: ${{ github.event.inputs.tag || github.ref_name || (github.ref_type == 'tag' && github.ref_name) || '' }} + PUBLISH_TAG: ${{ inputs.tag }} + WHEEL_RUN_ID: ${{ inputs.wheel_run_id }} jobs: publish: @@ -28,53 +31,108 @@ jobs: environment: name: pypi permissions: + actions: read id-token: write contents: read steps: - name: Ensure tag provided run: | - if [ -z "${{ env.PUBLISH_TAG }}" ]; then - echo "PUBLISH_TAG is required. Run on a tag or provide the workflow_dispatch input 'tag'." >&2 - exit 1 - fi + case "$PUBLISH_TAG" in + v*) ;; + *) + echo "PUBLISH_TAG must start with 'v'." >&2 + exit 1 + ;; + esac - - name: Locate wheel build run - id: wheels-run + - name: Validate wheel build run uses: actions/github-script@v7 env: - WHEEL_SHA: ${{ github.sha }} + WHEEL_RUN_ID: ${{ env.WHEEL_RUN_ID }} WHEEL_TAG: ${{ env.PUBLISH_TAG }} with: script: | - const sha = process.env.WHEEL_SHA; + const runId = Number(process.env.WHEEL_RUN_ID); const tag = process.env.WHEEL_TAG; - const { data } = await github.rest.actions.listWorkflowRuns({ + + if (!Number.isSafeInteger(runId) || runId <= 0) { + core.setFailed(`Invalid wheel run ID: ${process.env.WHEEL_RUN_ID}`); + return; + } + + let run; + for (let attempt = 1; attempt <= 12; attempt += 1) { + const response = await github.rest.actions.getWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: runId, + }); + run = response.data; + if (run.status === "completed") { + break; + } + core.info( + `Wheel run ${runId} is ${run.status}; waiting for completion (${attempt}/12).` + ); + await new Promise((resolve) => setTimeout(resolve, 5000)); + } + + if (run.path !== ".github/workflows/python-wheels.yaml") { + core.setFailed( + `Run ${runId} belongs to ${run.path}, not python-wheels.yaml.` + ); + return; + } + if (run.status !== "completed" || run.conclusion !== "success") { + core.setFailed( + `Wheel run ${runId} for tag ${tag} did not complete successfully (status: ${run.status}, conclusion: ${run.conclusion}).` + ); + return; + } + + const jobsResponse = + await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: runId, + per_page: 100, + }); + const uploadJob = jobsResponse.data.jobs.find( + (job) => job.name === "upload-release-wheels" + ); + if (!uploadJob || uploadJob.conclusion !== "success") { + core.setFailed( + `Wheel run ${runId} did not successfully upload its tested release wheels.` + ); + return; + } + + const tagResponse = await github.rest.git.getRef({ owner: context.repo.owner, repo: context.repo.repo, - workflow_id: "python-wheels.yaml", - head_sha: sha, - per_page: 100, + ref: `tags/${tag}`, }); - const run = data.workflow_runs.find( - (r) => - r.head_sha === sha && - r.status === "completed" && - r.conclusion === "success" - ); - if (!run) { + let tagTarget = tagResponse.data.object; + for (let depth = 0; tagTarget.type === "tag" && depth < 5; depth += 1) { + const annotatedTag = await github.rest.git.getTag({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_sha: tagTarget.sha, + }); + tagTarget = annotatedTag.data.object; + } + if (tagTarget.type !== "commit" || tagTarget.sha !== run.head_sha) { core.setFailed( - `No successful python-wheels run found for tag ${tag} (sha ${sha}). Trigger the wheel build for this tag first.` + `Tag ${tag} does not point to wheel run ${runId} commit ${run.head_sha}.` ); - return; } - core.setOutput("run_id", run.id.toString()); - name: Download wheel artifacts uses: actions/download-artifact@v4 with: pattern: wheels-* path: wheelhouse - run-id: ${{ steps.wheels-run.outputs.run_id }} + run-id: ${{ env.WHEEL_RUN_ID }} github-token: ${{ secrets.GITHUB_TOKEN }} - name: Collect wheels diff --git a/.github/workflows/python-wheels.yaml b/.github/workflows/python-wheels.yaml index 524b6ae0a..70cad6101 100644 --- a/.github/workflows/python-wheels.yaml +++ b/.github/workflows/python-wheels.yaml @@ -1,6 +1,10 @@ name: Python API Wheels on: + push: + branches: ["main"] + pull_request: + branches: ["**"] workflow_dispatch: jobs: @@ -11,15 +15,19 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.11", "3.12", "3.13", "3.14"] + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-15-intel, windows-latest] + include: + - os: ubuntu-latest + manylinux: quay.io/pypa/manylinux_2_28_x86_64:latest + - os: ubuntu-24.04-arm + manylinux: quay.io/pypa/manylinux_2_28_aarch64:latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 id: setup-python with: - python-version: ${{ matrix.python-version }} + python-version: "3.11" - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Install capnp (macOS) if: runner.os == 'macOS' @@ -27,15 +35,80 @@ jobs: - name: Install capnp (Windows) if: runner.os == 'Windows' run: choco install capnproto + - name: Stage bundled client (macOS) + if: runner.os == 'macOS' + run: make stage-python-client + - name: Sign bundled client (macOS) + if: runner.os == 'macOS' && github.ref_type == 'tag' + env: + MACOS_CERTIFICATE: ${{ secrets.PROD_MACOS_APP_CERTIFICATE }} + MACOS_CERTIFICATE_PWD: ${{ secrets.PROD_MACOS_APP_CERTIFICATE_PWD }} + MACOS_CERTIFICATE_NAME: ${{ secrets.PROD_MACOS_APP_CERTIFICATE_NAME }} + MACOS_CI_KEYCHAIN_PWD: ${{ secrets.PROD_MACOS_CI_KEYCHAIN_PWD }} + run: | + CERTIFICATE_PATH="$RUNNER_TEMP/gen-application-certificate.p12" + KEYCHAIN_PATH="$RUNNER_TEMP/gen-application-signing.keychain-db" + echo "$MACOS_CERTIFICATE" | base64 --decode > "$CERTIFICATE_PATH" + security create-keychain -p "$MACOS_CI_KEYCHAIN_PWD" "$KEYCHAIN_PATH" + security default-keychain -s "$KEYCHAIN_PATH" + security unlock-keychain -p "$MACOS_CI_KEYCHAIN_PWD" "$KEYCHAIN_PATH" + security import "$CERTIFICATE_PATH" \ + -k "$KEYCHAIN_PATH" \ + -P "$MACOS_CERTIFICATE_PWD" \ + -T /usr/bin/codesign + security set-key-partition-list \ + -S apple-tool:,apple:,codesign: \ + -s \ + -k "$MACOS_CI_KEYCHAIN_PWD" \ + "$KEYCHAIN_PATH" + codesign \ + --force \ + --timestamp \ + --options runtime \ + --sign "$MACOS_CERTIFICATE_NAME" \ + gen.gen.data/scripts/gen + codesign --verify --strict --verbose=2 gen.gen.data/scripts/gen + security delete-keychain "$KEYCHAIN_PATH" + - name: Notarize bundled client (macOS) + if: runner.os == 'macOS' && github.ref_type == 'tag' + env: + MACOS_CI_KEYCHAIN_PWD: ${{ secrets.PROD_MACOS_CI_KEYCHAIN_PWD }} + PROD_MACOS_NOTARIZATION_APPLE_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_APPLE_ID }} + PROD_MACOS_NOTARIZATION_TEAM_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_TEAM_ID }} + PROD_MACOS_NOTARIZATION_PWD: ${{ secrets.PROD_MACOS_NOTARIZATION_PWD }} + run: | + KEYCHAIN_PATH="$RUNNER_TEMP/gen-notarization.keychain-db" + NOTARIZATION_ARCHIVE="$RUNNER_TEMP/gen-client.zip" + ditto -c -k --keepParent gen.gen.data/scripts/gen "$NOTARIZATION_ARCHIVE" + security create-keychain -p "$MACOS_CI_KEYCHAIN_PWD" "$KEYCHAIN_PATH" + security default-keychain -s "$KEYCHAIN_PATH" + security unlock-keychain -p "$MACOS_CI_KEYCHAIN_PWD" "$KEYCHAIN_PATH" + security list-keychain -d user -s "$KEYCHAIN_PATH" + xcrun notarytool store-credentials "gen-notarytool-profile" \ + --apple-id "$PROD_MACOS_NOTARIZATION_APPLE_ID" \ + --team-id "$PROD_MACOS_NOTARIZATION_TEAM_ID" \ + --password "$PROD_MACOS_NOTARIZATION_PWD" \ + --keychain "$KEYCHAIN_PATH" + xcrun notarytool submit "$NOTARIZATION_ARCHIVE" \ + --keychain-profile "gen-notarytool-profile" \ + --wait + security delete-keychain "$KEYCHAIN_PATH" + - name: Stage bundled client (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + cargo build --release --locked --bin gen + New-Item -ItemType Directory -Force "gen.gen.data/scripts" + Copy-Item "target/release/gen.exe" "gen.gen.data/scripts/gen.exe" - name: Build wheels (Linux) if: runner.os == 'Linux' uses: PyO3/maturin-action@v1 with: command: build - args: --release --manifest-path gen-python/Cargo.toml --features extension-module --interpreter python${{ matrix.python-version }} --out gen-python/target/wheels - sccache: true + args: --release --manifest-path gen-python/Cargo.toml --features abi3,extension-module --interpreter python3.11 --out gen-python/target/wheels manylinux: auto - container: quay.io/pypa/manylinux_2_28_x86_64:latest + container: ${{ matrix.manylinux }} before-script-linux: | yum install -y epel-release yum install -y clang llvm-devel sqlite-devel capnproto capnproto-devel pkgconfig mold @@ -45,6 +118,7 @@ jobs: mkdir -p gen-capnp-schemas/src/generated (cd gen-capnp-schemas && capnp compile -I . -orust:src/generated gen-core.capnp gen-models.capnp gen-schema.capnp) ls -la gen-capnp-schemas/src/generated + make stage-python-client working-directory: . env: CAPNP: /usr/bin/capnp @@ -55,35 +129,104 @@ jobs: uses: PyO3/maturin-action@v1 with: command: build - args: --release --manifest-path gen-python/Cargo.toml --features extension-module --interpreter ${{ steps.setup-python.outputs.python-path }} --out gen-python/target/wheels + args: --release --manifest-path gen-python/Cargo.toml --features abi3,extension-module --interpreter ${{ steps.setup-python.outputs.python-path }} --out gen-python/target/wheels sccache: true working-directory: . - - name: Upload wheels to releases + - name: Verify wheel contents (macOS, Linux) + if: runner.os != 'Windows' + run: python gen-python/scripts/verify_wheel.py gen-python/target/wheels/*.whl + - name: Verify wheel contents (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $wheel = Get-ChildItem "gen-python/target/wheels/*.whl" + python gen-python/scripts/verify_wheel.py "$($wheel.FullName)" + - name: Verify bundled client signature (macOS) + if: runner.os == 'macOS' && github.ref_type == 'tag' + run: | + CHECK_DIRECTORY="$RUNNER_TEMP/gen-wheel-check" + mkdir -p "$CHECK_DIRECTORY" + unzip -q gen-python/target/wheels/*.whl -d "$CHECK_DIRECTORY" + CLIENT_PATH=$(find "$CHECK_DIRECTORY" -type f -path "*/scripts/gen" -print -quit) + test -n "$CLIENT_PATH" + codesign --verify --strict --verbose=2 "$CLIENT_PATH" + - name: Smoke test wheel (macOS, Linux) if: runner.os != 'Windows' - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} run: | - gh release upload "${TAG}" gen-python/target/wheels/*.whl - - name: Upload wheels to releases (Windows) + python -m pip install --force-reinstall --no-deps gen-python/target/wheels/*.whl + python -c "import gen; print(gen.__version__)" + gen --version + - name: Smoke test wheel (Windows) if: runner.os == 'Windows' shell: pwsh - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} run: | $ErrorActionPreference = 'Stop' - gh release upload "$env:TAG" "gen-python/target/wheels/*.whl" + $wheel = Get-ChildItem "gen-python/target/wheels/*.whl" + python -m pip install --force-reinstall --no-deps "$($wheel.FullName)" + python -c "import gen; print(gen.__version__)" + gen --version - uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-py${{ matrix.python-version }} + name: wheels-${{ matrix.os }} path: gen-python/target/wheels/*.whl if-no-files-found: error - trigger-pypi-publish: + test-newer-python: needs: build-wheels + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-15-intel, windows-latest] + python-version: ["3.12", "3.13", "3.14"] + + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: wheelhouse + - name: Smoke test wheel (macOS, Linux) + if: runner.os != 'Windows' + run: | + python -m pip install --force-reinstall --no-deps wheelhouse/*.whl + python -c "import gen; print(gen.__version__)" + gen --version + - name: Smoke test wheel (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $wheel = Get-ChildItem "wheelhouse/*.whl" + python -m pip install --force-reinstall --no-deps "$($wheel.FullName)" + python -c "import gen; print(gen.__version__)" + gen --version + + upload-release-wheels: + needs: [build-wheels, test-newer-python] + runs-on: ubuntu-latest + if: ${{ needs.build-wheels.result == 'success' && needs.test-newer-python.result == 'success' && github.ref_type == 'tag' }} + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: wheels-* + path: wheelhouse + merge-multiple: true + - name: Upload wheels to release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: gh release upload "$TAG" wheelhouse/*.whl --clobber + + trigger-pypi-publish: + needs: upload-release-wheels runs-on: ubuntu-latest - if: ${{ needs.build-wheels.result == 'success' }} + if: ${{ needs.upload-release-wheels.result == 'success' && github.ref_type == 'tag' }} permissions: actions: write contents: read @@ -91,6 +234,7 @@ jobs: TAG: ${{ github.ref_name }} GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} + WHEEL_RUN_ID: ${{ github.run_id }} steps: - name: Ensure tag is available run: | @@ -100,4 +244,8 @@ jobs: fi - name: Trigger publish-pypi-tag workflow run: | - gh workflow run publish-pypi-tag.yaml --repo "$REPO" --ref "$TAG" -f tag="$TAG" + gh workflow run publish-pypi-tag.yaml \ + --repo "$REPO" \ + --ref "$TAG" \ + -f tag="$TAG" \ + -f wheel_run_id="$WHEEL_RUN_ID" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 08e72ab11..3e76e6e00 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -138,9 +138,33 @@ jobs: gh release upload "$RELEASE_NAME" gen.linux-x86_64.zip --clobber gh release upload "$RELEASE_NAME" gen.linux-arm64.zip --clobber + release-windows: + runs-on: windows-latest + permissions: + actions: write + contents: write + + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Install Dependencies + run: choco install capnproto + - name: Build + run: cargo build --release --locked --bin gen + - name: Ensure release exists + shell: bash + run: | + gh release view "$RELEASE_NAME" >/dev/null 2>&1 || gh release create "$RELEASE_NAME" --notes "" || gh release view "$RELEASE_NAME" + - name: upload-binary + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + Compress-Archive -Path target/release/gen.exe -DestinationPath gen.windows-x86_64.zip + gh release upload "$env:RELEASE_NAME" gen.windows-x86_64.zip --clobber + update-release-source: runs-on: ubuntu-latest - needs: [release-macos, release-ubuntu] + needs: [release-macos, release-ubuntu, release-windows] permissions: actions: write contents: write diff --git a/.gitignore b/.gitignore index c1b12600e..722473361 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ gen-r/vignettes/*.html # Python bindings **/__pycache__/ gen-python/python/gen/*.so +/gen.gen.data/ # R package extension built from Rust gen-r/src/genr.so diff --git a/Cargo.lock b/Cargo.lock index 34ace980a..0cc64b406 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2658,7 +2658,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin 0.9.9", ] [[package]] @@ -2757,8 +2757,7 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libdoltlite-sys" version = "0.38.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbf72e2d56836b1c5cf51e40534a969b563994a1b726fe502ef8f8158366b32" +source = "git+https://github.com/bobvh/rusqdoltlite?rev=d1ef39b#d1ef39bb87239d67b41779c468380392edc9b1f3" dependencies = [ "cc", "pkg-config", @@ -4726,8 +4725,7 @@ dependencies = [ [[package]] name = "rusqdoltlite" version = "0.40.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6de2ed3a86b99876639d7e7af3ed30992a2f46a73131b5c9339c1c28d8aebea" +source = "git+https://github.com/bobvh/rusqdoltlite?rev=d1ef39b#d1ef39bb87239d67b41779c468380392edc9b1f3" dependencies = [ "bitflags 2.13.0", "fallible-iterator", @@ -5288,9 +5286,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spin" diff --git a/Cargo.toml b/Cargo.toml index ca0e48c1f..53aec3e92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,11 @@ members = [".", "gen-core", "gen-models", "gen-graph", "gen-diff", "gen-tui", "g default-members = [".", "gen-core", "gen-models", "gen-graph", "gen-diff", "gen-capnp-schemas", "gen-annotations"] exclude = ["gen-python", "gen-r/src/rust"] +[patch.crates-io] +# TEMPORARY: verifying the Windows MSVC build fix in CI before it's published +# to crates.io. Revert this patch before merging. +rusqdoltlite = { git = "https://github.com/bobvh/rusqdoltlite", rev = "d1ef39b" } + [features] benchmark = [] default = ["models", "cli", "diff", "remote"] diff --git a/Makefile b/Makefile index 0c3a0e8bf..6ed0d57f5 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,17 @@ -.PHONY: python jupyter r r-test release-check-js clean build clippy-fix docker-build gif +.PHONY: python python-wheel stage-python-client jupyter r r-test release-check-js clean build clippy-fix docker-build gif python: @[ -d .venv ] || python -m venv .venv @.venv/bin/pip show maturin >/dev/null 2>&1 || .venv/bin/pip install maturin - .venv/bin/maturin develop --release --manifest-path gen-python/Cargo.toml --features extension-module + .venv/bin/maturin develop --release --manifest-path gen-python/Cargo.toml --features abi3,extension-module +stage-python-client: + cargo build --release --locked --bin gen + mkdir -p gen.gen.data/scripts + cp target/release/gen gen.gen.data/scripts/gen + chmod +x gen.gen.data/scripts/gen +python-wheel: stage-python-client + @[ -d .venv ] || python -m venv .venv + @.venv/bin/pip show maturin >/dev/null 2>&1 || .venv/bin/pip install maturin + .venv/bin/maturin build --release --manifest-path gen-python/Cargo.toml --features abi3,extension-module # The jupyter widget requires a bundled JS file compiled from the TypeScript sources in gen-python/js/. # We check in the compiled jupyter_widget.js alongside the TS so npm is not required to build the widget. jupyter: python @@ -13,7 +22,7 @@ jupyter: python test -f gen-python/python/gen/static/jupyter_widget.js || \ (echo "Error: gen-python/python/gen/static/jupyter_widget.js missing. Install npm and run 'make jupyter'." && exit 1); \ fi - .venv/bin/maturin develop --release --manifest-path gen-python/Cargo.toml --features extension-module --extras jupyter + .venv/bin/maturin develop --release --manifest-path gen-python/Cargo.toml --features abi3,extension-module --extras jupyter r-test: # CI only: builds and tests inside Linux Docker container @if command -v npm >/dev/null 2>&1; then \ diff --git a/README.md b/README.md index 111b2868b..b510b17a6 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,33 @@ Gen brings version control to genetic sequences. With it, you can track variants ## Install -**Gen client**: prebuilt binaries for macOS and Linux are on the [releases page](https://github.com/genhub-bio/gen/releases): [macOS (.pkg)](https://github.com/genhub-bio/gen/releases/download/nightly/gen.macos.pkg), [Linux x86_64 (.zip)](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-x86_64.zip), [Linux arm64 (.zip)](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-arm64.zip). Gen is built primarily for Unix-like systems; on Windows, you can install [WSL](https://learn.microsoft.com/en-us/windows/wsl/) to get a Linux environment, then use the Linux binary above from inside it. +Install Gen with pip: -**Python package**: install on macOS, Linux, or Windows using: ```sh pip install gen ``` -Install the `jupyter` extra to include an interactive graph widget for Jupyter and other anywidget-compatible notebooks: +Or install CLI in an isolated environment with uv: + +```sh +uv tool install gen +``` + +It is also available from crates.io: + +```sh +cargo install gen +``` + +Prebuilt binaries and installers are available on the [releases page](https://github.com/genhub-bio/gen/releases) for [macOS](https://github.com/genhub-bio/gen/releases/download/nightly/gen.macos.pkg), Linux ([x64](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-x86_64.zip) / [arm64](https://github.com/genhub-bio/gen/releases/download/nightly/gen.linux-arm64.zip)), and [Windows](https://github.com/genhub-bio/gen/releases/download/nightly/gen.windows-x86_64.zip). + +**Python package**: Installing Gen with pip also installs the Python package. Install the `jupyter` extra to include an interactive graph widget for Jupyter and other anywidget-compatible notebooks: + ```sh pip install gen[jupyter] ``` -**R package**: install on macOS (Apple silicon) using the `remotes` package: +**R package**: Install on macOS (Apple silicon) using the `remotes` package: ```r install.packages("remotes") remotes::install_url( diff --git a/gen-python/Cargo.toml b/gen-python/Cargo.toml index a6afcdcfb..d619b6a35 100644 --- a/gen-python/Cargo.toml +++ b/gen-python/Cargo.toml @@ -14,6 +14,7 @@ crate-type = ["cdylib"] [features] default = [] +abi3 = ["pyo3/abi3-py311"] extension-module = ["pyo3/extension-module"] [dependencies] diff --git a/gen-python/README.md b/gen-python/README.md index d46ae09e1..bfa710d06 100644 --- a/gen-python/README.md +++ b/gen-python/README.md @@ -2,9 +2,10 @@ Python bindings to the Gen version control system for genetic sequences. -The bindings expose the full Gen data model — repositories, sequence graphs, -import/export pipelines — from Python and Jupyter notebooks. An optional Jupyter -widget provides interactive graph visualization. +The package installs the `gen` command-line client and exposes the full Gen data +model — repositories, sequence graphs, import/export pipelines — from Python and +Jupyter notebooks. An optional Jupyter widget provides interactive graph +visualization. ## Quick start @@ -29,10 +30,18 @@ sample.plot() # or sg.plot() The package is built from three layers: +### Client (`src/main.rs`) + +The existing Rust command-line client is compiled separately and staged in +maturin's wheel data `scripts` directory. Package installers place that executable +on `PATH` as `gen` on macOS and Linux or `gen.exe` on Windows. + ### Rust (`src/python_api/`) The core of the package. [PyO3](https://pyo3.rs) + [maturin](https://www.maturin.rs) -compile the Gen engine into a native extension module (`gen.so`). This layer owns: +compile the Gen engine into a native extension module (`gen.so`). Release wheels +use CPython's stable ABI with Python 3.11 as the minimum supported version. This +layer owns: - **`Repository`** — opens a Gen workspace, drives all import/export operations (FASTA, GenBank, GFA, VCF, GAF, …), and exposes node/sample/sequence-graph @@ -72,6 +81,7 @@ Loaded by anywidget directly in the browser. Responsible for: ```sh make # from the project root — builds the native extension via maturin +make python-wheel # builds a wheel containing the extension and client make jupyter # also builds the JS widget bundle and installs the `jupyter` extras ``` diff --git a/gen-python/scripts/verify_wheel.py b/gen-python/scripts/verify_wheel.py new file mode 100644 index 000000000..50a24bd78 --- /dev/null +++ b/gen-python/scripts/verify_wheel.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Verify that a Gen wheel contains the Python package and bundled client.""" + +import sys +import zipfile +from pathlib import Path + + +def fail(message: str) -> None: + raise SystemExit(message) + + +def verify_wheel(wheel_path: Path) -> None: + client_name = "gen.exe" if "-win" in wheel_path.name else "gen" + + if "-abi3-" not in wheel_path.name: + fail(f"{wheel_path} should be tagged for the CPython stable ABI") + + with zipfile.ZipFile(wheel_path) as wheel: + names = {entry.filename for entry in wheel.infolist()} + + if not any(name.endswith(f".data/scripts/{client_name}") for name in names): + fail(f"{wheel_path} should contain the bundled {client_name}") + + extension_suffixes = (".pyd", ".so") + if not any( + name.startswith("gen/") and name.endswith(extension_suffixes) + for name in names + ): + fail(f"{wheel_path} should contain the compiled gen extension") + + if "gen/static/jupyter_widget.js" not in names: + fail(f"{wheel_path} should contain the Jupyter widget asset") + + print(f"Verified bundled client and Python package in {wheel_path}") + + +def main() -> None: + if len(sys.argv) != 2: + fail(f"usage: {Path(sys.argv[0]).name} WHEEL") + + wheel_path = Path(sys.argv[1]) + if not wheel_path.is_file(): + fail(f"wheel does not exist: {wheel_path}") + + verify_wheel(wheel_path) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 597a1c9b1..6c8969cb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,6 @@ Repository = "https://github.com/genhub-bio/gen" [tool.maturin] manifest-path = "gen-python/Cargo.toml" -features = ["extension-module"] +features = ["abi3", "extension-module"] python-source = "gen-python/python" module-name = "gen.gen"