From c67b5683a401a8b6683e863a2bd2f1ec2b94f7e1 Mon Sep 17 00:00:00 2001 From: Fernando Ayats Date: Mon, 26 May 2025 15:14:39 +0200 Subject: [PATCH 1/5] Refactor action --- .github/workflows/test-build-cache.yml | 36 +++---- .spack/default/spack.yaml | 8 +- README.md | 129 +++++------------------- action.yml | 131 ++++++++++++++----------- 4 files changed, 121 insertions(+), 183 deletions(-) diff --git a/.github/workflows/test-build-cache.yml b/.github/workflows/test-build-cache.yml index cd1ce02..1379e1f 100644 --- a/.github/workflows/test-build-cache.yml +++ b/.github/workflows/test-build-cache.yml @@ -2,9 +2,7 @@ name: Test Spack Build-Cache Action on: push: - branches: [ main ] pull_request: - branches: [ main ] jobs: smoke: @@ -14,24 +12,28 @@ jobs: packages: write steps: - # 1) Check out this action’s code - - name: Checkout action + - name: Checkout uses: actions/checkout@v4 - # 2) Run our composite action locally - - name: Run Spack Build-Cache + - name: Set up Spack + uses: spack/setup-spack@v2 + + # - name: Add spack.numpex + # uses: viperML/spack-repo-action@master + # with: + # repository: numpex/spack.numpex + + - name: Activate Spack build cache uses: ./ with: - env-variant: default - env-path: .spack - repo-packages: numpex/spack.numpex - repo-packages-path: spack.numpex - mirror: numpex-buildcache - mirror-token: ${{ secrets.GITHUB_TOKEN }} + environment: .spack/default base-image: ubuntu:24.04 - - name: spack + force: true + + - name: Check specs + run: | + spack spec + + - name: Check programs run: | - spack find -lv - - - \ No newline at end of file + which -a cmake diff --git a/.spack/default/spack.yaml b/.spack/default/spack.yaml index bb9df52..de626d2 100644 --- a/.spack/default/spack.yaml +++ b/.spack/default/spack.yaml @@ -3,19 +3,17 @@ # It describes a set of packages to be installed, along with # configuration settings. spack: - config: - install_tree: - padded_length: 128 packages: all: - compiler: [gcc@14:, clang@14:] + compiler: ["gcc@14:", "clang@14:"] require: ['target=x86_64_v3'] definitions: - compilers: [gcc@12:] specs: - $compilers + - cmake view: true concretizer: unify: true mirrors: - numpex-buildcache: oci://ghcr.io/numpex/numpex-buildcache \ No newline at end of file + numpex-buildcache: oci://ghcr.io/numpex/numpex-buildcache diff --git a/README.md b/README.md index 514ec4b..8f70d12 100644 --- a/README.md +++ b/README.md @@ -5,29 +5,14 @@ ![GitHub License](https://img.shields.io/github/license/numpex/spack-buildcache-action) ![Static Badge](https://img.shields.io/badge/packaging-spack-blue) -[spack]: https://spack.io +[spack]: https://spack.io -A GitHub Actions composite action to: +A GitHub Actions composite action to build and push a Spack environment into an OCI registry mirror. -1. Install your project (and dependencies) via [Spack][spack] -2. Push the resulting binary buildcache to a remote OCI registry mirror - ---- - -## Features - -- Checks out your repository (including submodules) -- Sets up Spack via `spack/setup-spack` -- Clones your custom Spack package recipes repo -- Adds it as a Spack repository -- Installs the specified Spack environment variant -- Pushes the buildcache to a specified OCI mirror - ---- ## Usage -Add this step to your workflow: +Add the step after your `spack/setup-spack` action: ```yaml jobs: @@ -37,101 +22,35 @@ jobs: packages: write steps: - - name: Build & Push Spack Buildcache - uses: numpex/spack-buildcache-action@v1 - with: - env-variant: default - mirror: my-org/my-spack-mirror - # Optional inputs shown with their defaults: - # repo-packages: numpex/spack.numpex - # repo-packages-path: spack.numpex - # spack-path: _spack - # base-image: ubuntu:24.04 -``` - ---- - -## Inputs - -| Input | Description | Required | Default | -| ------------------ | --------------------------------------------------------------------------- | :------: | --------------------- | -| `env-variant` | Spack environment variant to install (e.g. `default`, `omp`, etc.) | ✅ | — | -| `mirror` | Name of the OCI mirror (e.g. `my-org/my-spack-mirror`) | ✅ | — | -| `repo-packages` | GitHub repo with your Spack package recipes | | `numpex/spack.numpex` | -| `repo-packages-path` | Local directory to clone the Spack packages repo | | `spack.numpex` | -| `spack-path` | Local install path for the Spack checkout | | `_spack` | -| `base-image` | Base Docker image passed to `spack buildcache push --base-image` | | `ubuntu:24.04` | + - name: Set up Spack + uses: spack/setup-spack@v2 ---- - -## Example Workflow - -```yaml -name: CI - -on: - push: - tags: - - 'v*' - -jobs: - build-cache: - runs-on: ubuntu-24.04 - permissions: - packages: write - - steps: - - name: Build & publish Spack cache - uses: numpex/spack-buildcache-action@v1 + - name: Build and push Spack environment + uses: numpex/spack-buildcache-action@v2 with: - env-variant: default - mirror: numpex/spack-buildcache -``` - ---- - -## Test Workflow Example + environment: .spack/default -Here’s a sample workflow you can use within your Spack Build-Cache Action repository to test the action locally: - -```yaml -name: Test Spack Build-Cache Action + - name: Build app + # Use your Spack packages automatically + run: | + cmake -B build + cmake --build build +``` -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] -jobs: - smoke: - runs-on: ubuntu-24.04 - permissions: - contents: read - packages: write - - steps: - # 1) Check out this action’s code - - name: Checkout action - uses: actions/checkout@v4 +## Inputs - # 2) Run our composite action locally - - name: Run Spack Build-Cache - uses: ./ - with: - env-variant: default - repo-packages: numpex/spack.numpex - repo-packages-path: spack.numpex - mirror: numpex-buildcache +| Input | Description | Required | Default | +| ------------------ | ------------------------------------------------------------------------------- | :------: | ---------------------------------------- | +| `environment` | Relative path to the Spack environment (e.g. `.`). | ✅ | | +| `load-environment` | Whether the Spack environment should be loaded to subsequent steps. | | `true` | +| `spack-path` | Path to the Spack installation, if changed the path for the setup-spack action. | | `spack` | +| `mirror` | OCI registry to push the packages to. | | `oci://ghcr.io/${{ github.repository }}` | +| `token` | OCI token to use for pushing the packages. | | `${{ github.token }}` | +| `base-image` | Image name to attach to the packages, to use them as standalone containers. | | | +| `force` | Force push packages into the registry. | | `false` | - # 3) Verify installed specs - - name: List installed specs - run: | - . _spack/share/spack/setup-env.sh - spack find -lv -``` ---- ## License diff --git a/action.yml b/action.yml index 1810bef..ab4c90e 100644 --- a/action.yml +++ b/action.yml @@ -5,80 +5,99 @@ branding: color: purple inputs: - # which Spack environment to use (directory under your repo-packages) - env-variant: - description: "Spack environment variant (e.g. 'default')" + environment: + description: "Relative path to the Spack environment (e.g. `.`)." required: true - # path to spack environments - env-path: - description: "local path to Spack environments" + default: ".spack/default" + + load-environment: + description: "Whether the Spack environment should be loaded to subsequent steps." required: false - default: ".spack" - # where to pull your Spack packages from - repo-packages: - description: "GitHub repo with your Spack package recipes" + default: "true" + + spack-path: + description: "Path to the Spack installation, if changed the path for the setup-spack action." required: false - default: "numpex/spack.numpex" - repo-packages-path: - description: "Path to your Spack packages repo" + default: "spack" + + token: + description: "OCI token to use for pushing the packages." required: false - default: "spack.numpex" - # mirror name in Spack - mirror: - description: "Name of the Spack buildcache mirror" - required: true - default: "numpex-buildcache" + default: "${{ github.token }}" - # pass in your GitHub token or other OCI password - mirror-token: - description: "Password/token to push to the mirror (e.g. secrets.GITHUB_TOKEN)" - required: true + mirror: + description: "OCI registry to push the packages to." + required: false + default: "oci://ghcr.io/${{ github.repository }}" base-image: - description: "Base container image for buildcache metadata" + description: "Image name to attach to the packages, to use them as standalone containers." + required: false + + force: + description: "Force push packages into the registry." required: false - default: "ubuntu:24.04" + default: "false" runs: using: composite steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: true + - name: Configure Spack + shell: bash + run: | + . "${{ inputs.spack-path }}/share/spack/setup-env.sh" + spack bootstrap now - - name: Set up Spack - uses: spack/setup-spack@v2.1.1 - with: - buildcache: true - path: _spack - color: true + spack \ + mirror add \ + --scope site \ + --unsigned \ + --oci-username ${{ github.repository_owner }} \ + --oci-password ${{ inputs.token }} \ + buildcache-numpex \ + ${{ inputs.mirror }} + + spack compiler find --scope site - - name: Checkout Spack packages repo - uses: actions/checkout@v4 + # Sane defaults + spack config --scope site add config:install_tree:root:/opt/root + spack config --scope site add config:install_tree:padded_length:128 + + # Schedule for pushing before installing + # If installing fails, push will still be performed + - name: Push to cache + uses: gacts/run-and-post-run@v1 with: - repository: ${{ inputs.repo-packages }} - path: ${{ inputs.repo-packages-path }} - fetch-depth: 1 + # gacts/run-and-post-run calls each line as a separate command, so put everything in a single line + # https://github.com/gacts/run-and-post-run/issues/45 + post: | + . "${{ inputs.spack-path }}/share/spack/setup-env.sh" && \ + spack -e "${{ inputs.environment }}" buildcache \ + push \ + --update-index \ + --with-build-dependencies \ + $([[ -n "${{ inputs.base-image }}" ]] && echo --base-image "${{ inputs.base-image }}") \ + $([[ "${{ inputs.force }}" == "true" ]] && echo --force) \ + buildcache-numpex - - name: Add custom Spack repo + - name: Install Spack environment shell: bash run: | - . _spack/share/spack/setup-env.sh - spack repo add "${{ inputs.repo-packages-path }}" + . "${{ inputs.spack-path }}/share/spack/setup-env.sh" + spack -e "${{ inputs.environment }}" install - - name: Install & Push buildcache + - name: Automatically load environment + if: ${{ inputs.load-environment == 'true' }} shell: bash - env: - SPACK_MIRROR_TOKEN: ${{ inputs.mirror-token }} run: | - . _spack/share/spack/setup-env.sh - spack -e "${{ inputs.env-path }}/${{ inputs.env-variant }}" install --no-check-signature - spack -e "${{ inputs.env-path }}/${{ inputs.env-variant }}" mirror set --push \ - --oci-username "${{ github.actor }}" \ - --oci-password "${SPACK_MIRROR_TOKEN}" \ - "${{ inputs.mirror }}" - spack -e "${{ inputs.env-path }}/${{ inputs.env-variant }}" buildcache push \ - --base-image "${{ inputs.base-image }}" \ - --unsigned \ - --update-index "${{ inputs.mirror }}" \ No newline at end of file + . "${{ inputs.spack-path }}/share/spack/setup-env.sh" + env_file="/tmp/spack-env" + spack env activate "${{ inputs.environment }}" --sh > "$env_file" + while IFS= read -r line; do + if [[ "$line" =~ ^export\ (.*)=(.*)\;$ ]]; then + exporting="${BASH_REMATCH[1]}=${BASH_REMATCH[2]}" + echo ":: exporting $exporting" + echo "$exporting" >> "$GITHUB_ENV" + fi + done < "$env_file" + rm -f "$env_file" From 289b3ca278e342408a8180e26e1523be947e856b Mon Sep 17 00:00:00 2001 From: Fernando Ayats Date: Tue, 27 May 2025 09:00:46 +0200 Subject: [PATCH 2/5] Use spack-bash --- README.md | 1 - action.yml | 19 ++++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8f70d12..4eb5aa7 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,6 @@ jobs: | ------------------ | ------------------------------------------------------------------------------- | :------: | ---------------------------------------- | | `environment` | Relative path to the Spack environment (e.g. `.`). | ✅ | | | `load-environment` | Whether the Spack environment should be loaded to subsequent steps. | | `true` | -| `spack-path` | Path to the Spack installation, if changed the path for the setup-spack action. | | `spack` | | `mirror` | OCI registry to push the packages to. | | `oci://ghcr.io/${{ github.repository }}` | | `token` | OCI token to use for pushing the packages. | | `${{ github.token }}` | | `base-image` | Image name to attach to the packages, to use them as standalone containers. | | | diff --git a/action.yml b/action.yml index ab4c90e..62de914 100644 --- a/action.yml +++ b/action.yml @@ -15,11 +15,6 @@ inputs: required: false default: "true" - spack-path: - description: "Path to the Spack installation, if changed the path for the setup-spack action." - required: false - default: "spack" - token: description: "OCI token to use for pushing the packages." required: false @@ -43,9 +38,8 @@ runs: using: composite steps: - name: Configure Spack - shell: bash + shell: spack-bash {0} run: | - . "${{ inputs.spack-path }}/share/spack/setup-env.sh" spack bootstrap now spack \ @@ -71,26 +65,25 @@ runs: # gacts/run-and-post-run calls each line as a separate command, so put everything in a single line # https://github.com/gacts/run-and-post-run/issues/45 post: | - . "${{ inputs.spack-path }}/share/spack/setup-env.sh" && \ + spack-bash -c "\ spack -e "${{ inputs.environment }}" buildcache \ push \ --update-index \ --with-build-dependencies \ $([[ -n "${{ inputs.base-image }}" ]] && echo --base-image "${{ inputs.base-image }}") \ $([[ "${{ inputs.force }}" == "true" ]] && echo --force) \ - buildcache-numpex + buildcache-numpex \ + " - name: Install Spack environment - shell: bash + shell: spack-bash {0} run: | - . "${{ inputs.spack-path }}/share/spack/setup-env.sh" spack -e "${{ inputs.environment }}" install - name: Automatically load environment if: ${{ inputs.load-environment == 'true' }} - shell: bash + shell: spack-bash {0} run: | - . "${{ inputs.spack-path }}/share/spack/setup-env.sh" env_file="/tmp/spack-env" spack env activate "${{ inputs.environment }}" --sh > "$env_file" while IFS= read -r line; do From 2034dd7e9c9de7c5f57da4aee850b86d4f05deb6 Mon Sep 17 00:00:00 2001 From: Fernando Ayats Date: Tue, 27 May 2025 12:43:58 +0200 Subject: [PATCH 3/5] Save logs --- .github/workflows/test-build-cache.yml | 1 - .spack/default/spack.yaml | 1 + action.yml | 50 ++++++++++++++++---------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test-build-cache.yml b/.github/workflows/test-build-cache.yml index 1379e1f..a64bfa9 100644 --- a/.github/workflows/test-build-cache.yml +++ b/.github/workflows/test-build-cache.yml @@ -28,7 +28,6 @@ jobs: with: environment: .spack/default base-image: ubuntu:24.04 - force: true - name: Check specs run: | diff --git a/.spack/default/spack.yaml b/.spack/default/spack.yaml index de626d2..c6b0a9c 100644 --- a/.spack/default/spack.yaml +++ b/.spack/default/spack.yaml @@ -12,6 +12,7 @@ spack: specs: - $compilers - cmake + - python view: true concretizer: unify: true diff --git a/action.yml b/action.yml index 62de914..9d319e8 100644 --- a/action.yml +++ b/action.yml @@ -34,6 +34,11 @@ inputs: required: false default: "false" + log-name: + description: "If not empty, upload the build logs with the following name" + required: false + default: "" + runs: using: composite steps: @@ -57,28 +62,23 @@ runs: spack config --scope site add config:install_tree:root:/opt/root spack config --scope site add config:install_tree:padded_length:128 - # Schedule for pushing before installing - # If installing fails, push will still be performed - - name: Push to cache - uses: gacts/run-and-post-run@v1 - with: - # gacts/run-and-post-run calls each line as a separate command, so put everything in a single line - # https://github.com/gacts/run-and-post-run/issues/45 - post: | - spack-bash -c "\ - spack -e "${{ inputs.environment }}" buildcache \ - push \ - --update-index \ - --with-build-dependencies \ - $([[ -n "${{ inputs.base-image }}" ]] && echo --base-image "${{ inputs.base-image }}") \ - $([[ "${{ inputs.force }}" == "true" ]] && echo --force) \ - buildcache-numpex \ - " - - name: Install Spack environment shell: spack-bash {0} run: | - spack -e "${{ inputs.environment }}" install + spack \ + -e "${{ inputs.environment }}" \ + install \ + -vv \ + --log-format junit \ + --log-file /tmp/spack-log + + - uses: actions/upload-artifact@v4 + name: Upload logs + if: ${{ inputs.log-name != '' }} + with: + name: ${{ inputs.log-name }} + path: /tmp/spack-log.xml + overwrite: true - name: Automatically load environment if: ${{ inputs.load-environment == 'true' }} @@ -94,3 +94,15 @@ runs: fi done < "$env_file" rm -f "$env_file" + + - name: Push to cache + shell: spack-bash {0} + run: | + spack -e "${{ inputs.environment }}" buildcache \ + push \ + --update-index \ + --with-build-dependencies \ + $([[ -n "${{ inputs.base-image }}" ]] && echo --base-image "${{ inputs.base-image }}") \ + $([[ "${{ inputs.force }}" == "true" ]] && echo --force) \ + buildcache-numpex + From 4b2775f1be10ce0fc1f8abea27676116e800f950 Mon Sep 17 00:00:00 2001 From: Fernando Ayats Date: Wed, 28 May 2025 13:09:51 +0200 Subject: [PATCH 4/5] Add print-logs option --- README.md | 18 ++++++++++-------- action.yml | 9 +++++++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 4eb5aa7..11ce184 100644 --- a/README.md +++ b/README.md @@ -40,14 +40,16 @@ jobs: ## Inputs -| Input | Description | Required | Default | -| ------------------ | ------------------------------------------------------------------------------- | :------: | ---------------------------------------- | -| `environment` | Relative path to the Spack environment (e.g. `.`). | ✅ | | -| `load-environment` | Whether the Spack environment should be loaded to subsequent steps. | | `true` | -| `mirror` | OCI registry to push the packages to. | | `oci://ghcr.io/${{ github.repository }}` | -| `token` | OCI token to use for pushing the packages. | | `${{ github.token }}` | -| `base-image` | Image name to attach to the packages, to use them as standalone containers. | | | -| `force` | Force push packages into the registry. | | `false` | +| Input | Description | Required | Default | +| ------------------ | --------------------------------------------------------------------------- | :------: | ---------------------------------------- | +| `environment` | Relative path to the Spack environment (e.g. `.`). | ✅ | | +| `load-environment` | Whether the Spack environment should be loaded to subsequent steps. | | `true` | +| `mirror` | OCI registry to push the packages to. | | `oci://ghcr.io/${{ github.repository }}` | +| `token` | OCI token to use for pushing the packages. | | `${{ github.token }}` | +| `base-image` | Image name to attach to the packages, to use them as standalone containers. | | | +| `force` | Force push packages into the registry. | | `false` | +| `print-logs` | Whether to show the build logs for `spack install`. | | `false` | +| `log-name` | If not empty, upload the build logs with the following name. | | | diff --git a/action.yml b/action.yml index 9d319e8..a6d984e 100644 --- a/action.yml +++ b/action.yml @@ -35,10 +35,15 @@ inputs: default: "false" log-name: - description: "If not empty, upload the build logs with the following name" + description: "If not empty, upload the build logs with the following name." required: false default: "" + print-logs: + description: "Whether to show the build logs for spack install." + required: false + default: "false" + runs: using: composite steps: @@ -68,7 +73,7 @@ runs: spack \ -e "${{ inputs.environment }}" \ install \ - -vv \ + $([[ "${{ inputs.print-logs }}" == "true" ]] && echo -vv) \ --log-format junit \ --log-file /tmp/spack-log From 719325cb39237ab2bb52744da038f51531956b31 Mon Sep 17 00:00:00 2001 From: Fernando Ayats Date: Wed, 4 Jun 2025 09:38:07 +0200 Subject: [PATCH 5/5] Push even if install fails --- action.yml | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/action.yml b/action.yml index a6d984e..5375fa7 100644 --- a/action.yml +++ b/action.yml @@ -70,6 +70,7 @@ runs: - name: Install Spack environment shell: spack-bash {0} run: | + set +e spack \ -e "${{ inputs.environment }}" \ install \ @@ -77,6 +78,10 @@ runs: --log-format junit \ --log-file /tmp/spack-log + SPACK_INSTALL_EXIT_CODE=$? + echo "SPACK_INSTALL_EXIT_CODE=$SPACK_INSTALL_EXIT_CODE" >> "$GITHUB_ENV" + set -e + - uses: actions/upload-artifact@v4 name: Upload logs if: ${{ inputs.log-name != '' }} @@ -85,6 +90,27 @@ runs: path: /tmp/spack-log.xml overwrite: true + - name: Push to cache + shell: spack-bash {0} + run: | + spack -e "${{ inputs.environment }}" buildcache \ + push \ + --update-index \ + --with-build-dependencies \ + $([[ -n "${{ inputs.base-image }}" ]] && echo --base-image "${{ inputs.base-image }}") \ + $([[ "${{ inputs.force }}" == "true" ]] && echo --force) \ + buildcache-numpex + + - name: Check Spack install result + shell: spack-bash {0} + run: | + if [[ "$SPACK_INSTALL_EXIT_CODE" -ne 0 ]]; then + echo "Spack install failed with exit code: $SPACK_INSTALL_EXIT_CODE" + exit $SPACK_INSTALL_EXIT_CODE + else + echo "Spack install completed successfully" + fi + - name: Automatically load environment if: ${{ inputs.load-environment == 'true' }} shell: spack-bash {0} @@ -100,14 +126,4 @@ runs: done < "$env_file" rm -f "$env_file" - - name: Push to cache - shell: spack-bash {0} - run: | - spack -e "${{ inputs.environment }}" buildcache \ - push \ - --update-index \ - --with-build-dependencies \ - $([[ -n "${{ inputs.base-image }}" ]] && echo --base-image "${{ inputs.base-image }}") \ - $([[ "${{ inputs.force }}" == "true" ]] && echo --force) \ - buildcache-numpex