From 8bebef11067901a9194bb73abf14273cebf056aa Mon Sep 17 00:00:00 2001 From: 0xsl1m <85326122+0xsl1m@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:34:59 -0500 Subject: [PATCH] Make container evidence recovery deterministic --- .github/workflows/container-release.yml | 122 +++++++++++++++++------- docs/RELEASING.md | 34 +++++-- tests/test_workflow_policy.py | 27 +++++- 3 files changed, 141 insertions(+), 42 deletions(-) diff --git a/.github/workflows/container-release.yml b/.github/workflows/container-release.yml index 0078be3..f3c1993 100644 --- a/.github/workflows/container-release.yml +++ b/.github/workflows/container-release.yml @@ -1,11 +1,18 @@ name: Publish release container +run-name: Container evidence for ${{ github.event.release.tag_name || inputs.release_tag }} on: release: types: [published] + workflow_dispatch: + inputs: + release_tag: + description: Existing stable release tag whose trusted image evidence needs recovery + required: true + type: string concurrency: - group: container-release-${{ github.event.release.tag_name }} + group: container-release-${{ github.event.release.tag_name || inputs.release_tag }} cancel-in-progress: false permissions: @@ -23,7 +30,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.release.tag_name }} + ref: refs/tags/${{ github.event.release.tag_name || inputs.release_tag }} fetch-depth: 0 persist-credentials: false @@ -32,23 +39,41 @@ jobs: shell: bash env: GH_TOKEN: ${{ github.token }} - RELEASE_PRERELEASE: ${{ github.event.release.prerelease }} + RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }} + WORKFLOW_EVENT: ${{ github.event_name }} + WORKFLOW_REF: ${{ github.ref }} run: | set -euo pipefail image="ghcr.io/${GITHUB_REPOSITORY,,}" - version="${GITHUB_REF_NAME#v}" - test "$RELEASE_PRERELEASE" = "false" - [[ "$GITHUB_REF_NAME" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] + version="${RELEASE_TAG#v}" + [[ "$RELEASE_TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] + if [[ "$WORKFLOW_EVENT" == workflow_dispatch ]]; then + test "$WORKFLOW_REF" = "refs/heads/main" + git branch --remotes --contains "$GITHUB_SHA" | grep -Fxq " origin/main" + fi + release_metadata="$( + gh release view "$RELEASE_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --json isDraft,isPrerelease,publishedAt,tagName + )" + test "$(jq -r '.tagName' <<<"$release_metadata")" = "$RELEASE_TAG" + test "$(jq -r '.isDraft' <<<"$release_metadata")" = "false" + test "$(jq -r '.isPrerelease' <<<"$release_metadata")" = "false" + test "$(jq -r '.publishedAt // empty' <<<"$release_metadata")" != "" + release_sha="$(git rev-parse HEAD)" + test "$(git rev-list -n 1 "refs/tags/$RELEASE_TAG")" = "$release_sha" + if [[ "$WORKFLOW_EVENT" == release ]]; then + test "$release_sha" = "$GITHUB_SHA" + fi package_version="$(python -c \ 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" - test "$(git rev-parse HEAD)" = "$GITHUB_SHA" - git branch --remotes --contains "$GITHUB_SHA" | grep -Fxq " origin/main" + git branch --remotes --contains "$release_sha" | grep -Fxq " origin/main" test "$package_version" = "$version" successful_ci="$( gh run list \ --repo "$GITHUB_REPOSITORY" \ --workflow ci.yml \ - --commit "$GITHUB_SHA" \ + --commit "$release_sha" \ --branch main \ --event push \ --status success \ @@ -74,19 +99,25 @@ jobs: } version_digest="$(resolve_tag_digest "$image:$version")" - sha_digest="$(resolve_tag_digest "$image:sha-$GITHUB_SHA")" + sha_digest="$(resolve_tag_digest "$image:sha-$release_sha")" version_missing="$([[ "$version_digest" == missing ]] && echo true || echo false)" sha_missing="$([[ "$sha_digest" == missing ]] && echo true || echo false)" + if [[ "$WORKFLOW_EVENT" == workflow_dispatch ]]; then + if [[ "$version_missing" == true || "$sha_missing" == true ]]; then + echo "recovery dispatch requires both immutable release image tags" >&2 + exit 1 + fi + fi resolved_digest="" reused=false if [[ "$version_missing" == true && "$sha_missing" == true ]]; then docker build \ - --label "org.opencontainers.image.revision=$GITHUB_SHA" \ + --label "org.opencontainers.image.revision=$release_sha" \ --label "org.opencontainers.image.version=$version" \ --label "org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY" \ --tag "$image:$version" \ - --tag "$image:sha-$GITHUB_SHA" \ + --tag "$image:sha-$release_sha" \ . else if [[ "$version_missing" == false && "$sha_missing" == false ]]; then @@ -99,7 +130,7 @@ jobs: docker pull "$image@$resolved_digest" test "$(docker image inspect --format \ '{{index .Config.Labels "org.opencontainers.image.revision"}}' \ - "$image@$resolved_digest")" = "$GITHUB_SHA" + "$image@$resolved_digest")" = "$release_sha" test "$(docker image inspect --format \ '{{index .Config.Labels "org.opencontainers.image.version"}}' \ "$image@$resolved_digest")" = "$version" @@ -113,10 +144,10 @@ jobs: --predicate-type https://slsa.dev/provenance/v1 \ --signer-workflow \ "$GITHUB_REPOSITORY/.github/workflows/container-release.yml" \ - --source-digest "$GITHUB_SHA" \ + --source-digest "$release_sha" \ --deny-self-hosted-runners docker tag "$image@$resolved_digest" "$image:$version" - docker tag "$image@$resolved_digest" "$image:sha-$GITHUB_SHA" + docker tag "$image@$resolved_digest" "$image:sha-$release_sha" reused=true fi { @@ -126,6 +157,8 @@ jobs: echo "resolved_digest=$resolved_digest" echo "version_missing=$version_missing" echo "sha_missing=$sha_missing" + echo "release_sha=$release_sha" + echo "release_tag=$RELEASE_TAG" } >> "$GITHUB_OUTPUT" - name: Generate CycloneDX image SBOM @@ -136,17 +169,6 @@ jobs: output: shadowshield-sbom.cdx.json exit-code: "0" - - name: Canonicalize release SBOM - shell: bash - run: | - set -euo pipefail - temporary="$(mktemp)" - jq --sort-keys \ - 'del(.serialNumber, .metadata.timestamp)' \ - shadowshield-sbom.cdx.json > "$temporary" - jq -e '.bomFormat == "CycloneDX"' "$temporary" >/dev/null - mv "$temporary" shadowshield-sbom.cdx.json - - name: Reject fixable high or critical image vulnerabilities uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: @@ -169,6 +191,7 @@ jobs: RESOLVED_DIGEST: ${{ steps.image.outputs.resolved_digest }} VERSION_MISSING: ${{ steps.image.outputs.version_missing }} SHA_MISSING: ${{ steps.image.outputs.sha_missing }} + RELEASE_SHA: ${{ steps.image.outputs.release_sha }} run: | set -euo pipefail if [[ "$VERSION_MISSING" == true || "$SHA_MISSING" == true ]]; then @@ -182,7 +205,7 @@ jobs: docker push "$IMAGE_REPOSITORY:$IMAGE_VERSION" fi if [[ "$SHA_MISSING" == true ]]; then - docker push "$IMAGE_REPOSITORY:sha-$GITHUB_SHA" + docker push "$IMAGE_REPOSITORY:sha-$RELEASE_SHA" fi else push_output="$(docker push "$IMAGE_REPOSITORY:$IMAGE_VERSION")" @@ -191,7 +214,7 @@ jobs: | sed -n 's/^.*digest: \(sha256:[0-9a-f]\{64\}\).*$/\1/p' \ | tail -n 1)" test -n "$digest" - docker push "$IMAGE_REPOSITORY:sha-$GITHUB_SHA" + docker push "$IMAGE_REPOSITORY:sha-$RELEASE_SHA" fi [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] @@ -201,7 +224,7 @@ jobs: "$IMAGE_REPOSITORY:$IMAGE_VERSION" \ --format '{{json .Manifest}}' | jq -er '.digest')" \ && sha_digest="$(docker buildx imagetools inspect \ - "$IMAGE_REPOSITORY:sha-$GITHUB_SHA" \ + "$IMAGE_REPOSITORY:sha-$RELEASE_SHA" \ --format '{{json .Manifest}}' | jq -er '.digest')" \ && [[ "$version_digest" == "$digest" && "$sha_digest" == "$digest" ]]; then tags_verified=true @@ -213,7 +236,34 @@ jobs: printf '%s\n' "$digest" > container-digest.txt echo "digest=$digest" >> "$GITHUB_OUTPUT" + - name: Canonicalize release SBOM + shell: bash + env: + IMAGE_REPOSITORY: ${{ steps.image.outputs.repository }} + IMAGE_DIGEST: ${{ steps.push.outputs.digest }} + run: | + set -euo pipefail + temporary="$(mktemp)" + serial_number="$( + python3 - "$IMAGE_REPOSITORY@$IMAGE_DIGEST" <<'PY' + import sys + import uuid + + print(f"urn:uuid:{uuid.uuid5(uuid.NAMESPACE_URL, sys.argv[1])}") + PY + )" + jq --arg serial_number "$serial_number" --sort-keys \ + 'del(.metadata.timestamp) | .serialNumber = $serial_number' \ + shadowshield-sbom.cdx.json > "$temporary" + jq -e ' + .bomFormat == "CycloneDX" + and (.specVersion | type == "string" and length > 0) + and (.serialNumber | type == "string" and startswith("urn:uuid:")) + ' "$temporary" >/dev/null + mv "$temporary" shadowshield-sbom.cdx.json + - name: Attest release image provenance + if: github.event_name == 'release' && steps.image.outputs.reused != 'true' uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-name: ${{ steps.image.outputs.repository }} @@ -236,21 +286,24 @@ jobs: GH_TOKEN: ${{ github.token }} IMAGE_REPOSITORY: ${{ steps.image.outputs.repository }} IMAGE_DIGEST: ${{ steps.push.outputs.digest }} + RELEASE_SHA: ${{ steps.image.outputs.release_sha }} + SBOM_SOURCE_SHA: ${{ github.sha }} run: | set -euo pipefail for _ in $(seq 1 12); do if gh attestation verify "oci://$IMAGE_REPOSITORY@$IMAGE_DIGEST" \ --repo "$GITHUB_REPOSITORY" \ --bundle-from-oci \ + --predicate-type https://slsa.dev/provenance/v1 \ --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/container-release.yml" \ - --source-digest "$GITHUB_SHA" \ + --source-digest "$RELEASE_SHA" \ --deny-self-hosted-runners \ && gh attestation verify "oci://$IMAGE_REPOSITORY@$IMAGE_DIGEST" \ --repo "$GITHUB_REPOSITORY" \ --bundle-from-oci \ --predicate-type https://cyclonedx.org/bom \ --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/container-release.yml" \ - --source-digest "$GITHUB_SHA" \ + --source-digest "$SBOM_SOURCE_SHA" \ --deny-self-hosted-runners; then exit 0 fi @@ -291,6 +344,7 @@ jobs: shell: bash env: GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ steps.image.outputs.release_tag }} run: | set -euo pipefail download_dir="$(mktemp -d)" @@ -298,9 +352,9 @@ jobs: upload_or_verify_immutable() { local asset="$1" - if gh release view "$GITHUB_REF_NAME" --json assets \ + if gh release view "$RELEASE_TAG" --json assets \ --jq '.assets[].name' | grep -Fxq "$asset"; then - gh release download "$GITHUB_REF_NAME" \ + gh release download "$RELEASE_TAG" \ --pattern "$asset" \ --dir "$download_dir" if ! cmp --silent "$asset" "$download_dir/$asset"; then @@ -309,7 +363,7 @@ jobs: fi return fi - gh release upload "$GITHUB_REF_NAME" "$asset" + gh release upload "$RELEASE_TAG" "$asset" } upload_or_verify_immutable container-digest.txt diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 0b3d230..feda485 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -72,16 +72,29 @@ gh release create vX.Y.Z --target "$merge_sha" \ Do not create the release while required checks are pending. Once trusted source provenance exists, a partial container release is recoverable by fixing the -failed workflow and re-running it: exact matching registry content is pulled, -rescanned, re-attested, and reused. An interruption between the first image push -and provenance attachment intentionally needs the incomplete GHCR tags removed +failed workflow and using its restricted recovery dispatch from protected +`main`: + +```bash +gh workflow run container-release.yml --ref main -f release_tag=vX.Y.Z +``` + +The recovery path accepts only an existing stable release whose tag resolves to +an exact green commit contained in `main`. Exact matching registry content is +pulled only after its labels and prior release-source SLSA provenance verify, +then it is rescanned and the missing evidence is attached. The recovered SBOM +attestation records the protected `main` workflow commit that performed the +repair; the independently verified SLSA provenance continues to record the exact +release-source commit. An interruption between the first image push and +provenance attachment intentionally needs the incomplete GHCR tags removed before retry. Conflicting tags and existing evidence assets are never overwritten. The PyPI version itself cannot be replaced. Both publication workflows independently reject non-stable tags, tag/source version mismatches, commits outside `main`, and commits without a successful -exact-SHA `main` CI run. There is no manual-dispatch publication path; use -GitHub's **Re-run failed jobs** control to recover a failed release workflow. +exact-SHA `main` CI run. Manual dispatch cannot publish an unattested image: it +is a recovery-only path for an existing release digest with trusted +release-source provenance. ## Verify @@ -94,19 +107,26 @@ gh release download vX.Y.Z \ --pattern container-digest.txt --pattern shadowshield-sbom.cdx.json digest="$(cat container-digest.txt)" image="ghcr.io/0xsl1m/shadowshield@$digest" +release_sha="$(git rev-list -n 1 vX.Y.Z)" +# For a normal release, both attestations use the release SHA. If evidence was +# completed by the recovery dispatch, use that successful run's protected-main +# head SHA for evidence_sha: +evidence_sha="$release_sha" +# evidence_sha="$(gh run view RECOVERY_RUN_ID --json headSha --jq .headSha)" docker pull "$image" gh attestation verify "oci://$image" \ --repo 0xsl1m/shadowshield \ --bundle-from-oci \ + --predicate-type https://slsa.dev/provenance/v1 \ --signer-workflow 0xsl1m/shadowshield/.github/workflows/container-release.yml \ - --source-digest "$(git rev-list -n 1 vX.Y.Z)" \ + --source-digest "$release_sha" \ --deny-self-hosted-runners gh attestation verify "oci://$image" \ --repo 0xsl1m/shadowshield \ --bundle-from-oci \ --predicate-type https://cyclonedx.org/bom \ --signer-workflow 0xsl1m/shadowshield/.github/workflows/container-release.yml \ - --source-digest "$(git rev-list -n 1 vX.Y.Z)" \ + --source-digest "$evidence_sha" \ --deny-self-hosted-runners ``` diff --git a/tests/test_workflow_policy.py b/tests/test_workflow_policy.py index 2052baf..a17c738 100644 --- a/tests/test_workflow_policy.py +++ b/tests/test_workflow_policy.py @@ -104,11 +104,36 @@ def test_release_builds_install_hash_locked_dependencies() -> None: provenance_check = 'gh attestation verify "oci://$image@$resolved_digest"' assert provenance_check in container_release assert "--predicate-type https://slsa.dev/provenance/v1" in container_release - assert '--source-digest "$GITHUB_SHA"' in container_release + assert '--source-digest "$release_sha"' in container_release assert container_release.index(provenance_check) < container_release.index( "reused=true", container_release.index(provenance_check), ) + assert "workflow_dispatch:" in container_release + assert "ref: refs/tags/${{ github.event.release.tag_name || inputs.release_tag }}" in ( + container_release + ) + assert 'test "$WORKFLOW_REF" = "refs/heads/main"' in container_release + assert 'git branch --remotes --contains "$GITHUB_SHA"' in container_release + assert 'gh release view "$RELEASE_TAG"' in container_release + assert "isDraft,isPrerelease,publishedAt,tagName" in container_release + assert "'.publishedAt // empty'" in container_release + assert 'test "$(git rev-list -n 1 "refs/tags/$RELEASE_TAG")" = "$release_sha"' in ( + container_release + ) + assert 'test "$release_sha" = "$GITHUB_SHA"' in container_release + assert "recovery dispatch requires both immutable release image tags" in container_release + assert "del(.serialNumber" not in container_release + assert ".serialNumber = $serial_number" in container_release + assert ( + "if: github.event_name == 'release' && steps.image.outputs.reused != 'true'" + in container_release + ) + assert '--source-digest "$RELEASE_SHA"' in container_release + assert '--source-digest "$SBOM_SOURCE_SHA"' in container_release + assert container_release.index("id: push") < container_release.index( + "name: Canonicalize release SBOM" + ) assert 'test "$tags_verified" = true' in container_release assert "gh release upload" in container_release assert "--clobber" not in container_release