diff --git a/.gitignore b/.gitignore index 80d35f3..f89b13d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,8 @@ .idea/ target/ -# Apache RAT artifacts (see dev/release/run-rat.sh) +# Release artifacts (see dev/release/README.md) +dev/dist apache-rat-*.jar rat.txt filtered_rat.txt diff --git a/crates/datafusion/LICENSE b/crates/datafusion/LICENSE new file mode 120000 index 0000000..30cff74 --- /dev/null +++ b/crates/datafusion/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/datafusion/NOTICE b/crates/datafusion/NOTICE new file mode 120000 index 0000000..fb376cf --- /dev/null +++ b/crates/datafusion/NOTICE @@ -0,0 +1 @@ +../../NOTICE \ No newline at end of file diff --git a/dev/release/README.md b/dev/release/README.md new file mode 100644 index 0000000..24b0014 --- /dev/null +++ b/dev/release/README.md @@ -0,0 +1,253 @@ + + +# Release Process + +This guide is for maintainers creating a release of Apache DataFusion Iceberg. + +As part of the Apache governance model, official releases consist of signed source tarballs approved by the DataFusion PMC. The `iceberg-datafusion` crate is then published to crates.io from the approved source. + +## Release Prerequisites + +### Add git remote for `apache` repo + +The instructions below assume the upstream git repo `git@github.com:apache/datafusion-iceberg.git` in remote `apache`. + +```shell +git remote add apache git@github.com:apache/datafusion-iceberg.git +``` + +### Install tools + +- `gpg` and `svn` to sign and upload release candidates. +- `java` and `python3` for the license check that `create-tarball.sh` runs. +- [`uv`](https://docs.astral.sh/uv/), or the `PyGithub` Python package, for the changelog script. + +### Create GitHub Personal Access Token (PAT) + +A personal access token (PAT) is needed for the changelog script. If you do not already have one, create a token with `repo` access by navigating to the [GitHub Developer Settings] page, and [follow these steps]. + +[github developer settings]: https://github.com/settings/developers +[follow these steps]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token + +### Add GPG Public Key to SVN `KEYS` file + +DataFusion subprojects share one `KEYS` file. If you will be releasing the final tarball, your GPG public key must be present in: + +- https://dist.apache.org/repos/dist/dev/datafusion/KEYS +- https://dist.apache.org/repos/dist/release/datafusion/KEYS + +See https://infra.apache.org/release-signing.html#generate for instructions on generating keys. + +Committers can add signing keys using the Subversion client and their ASF account: + +```shell +$ svn co https://dist.apache.org/repos/dist/dev/datafusion +$ cd datafusion +$ editor KEYS # add your key here +$ svn ci KEYS # commit changes +``` + +Follow the instructions in the header of the KEYS file to append your key. Here is an example: + +```shell +(gpg --list-sigs "John Doe" && gpg --armor --export "John Doe") >> KEYS +svn commit KEYS -m "Add key for John Doe" +``` + +## Release Process: Step by Step + +The examples below release version `0.11.0`. + +### 1. Check that the crate can be published + +crates.io rejects crates with git dependencies, so every dependency of `iceberg-datafusion` must come from crates.io. In particular, `iceberg` and `iceberg-catalog-rest` in the root `Cargo.toml` must point to a released iceberg-rust version, not a git revision. Check with: + +```shell +cargo publish --dry-run -p iceberg-datafusion +``` + +### 2. Update the version + +Update `version` under `[workspace.package]` in the root `Cargo.toml`. It must be higher than the latest version of [`iceberg-datafusion` on crates.io](https://crates.io/crates/iceberg-datafusion), which includes releases made from apache/iceberg-rust. Then update `Cargo.lock`: + +```shell +cargo check --workspace +``` + +Commit the changes and create a PR against `main`. + +### 3. Update the changelog + +Each release has its own changelog file in `dev/changelog/`, such as `dev/changelog/0.11.0.md`, listing all changes since the previous release. + +Generate it with `generate-changelog.py`. Pass the previous release tag, the commit to release, and the new version. For the first release from this repository, which has no earlier tag, pass the commit to start from instead of a tag. Run the script from the repository root with the `GITHUB_TOKEN` environment variable set; without a token, GitHub's API rate limit causes `403` errors. + +```shell +export GITHUB_TOKEN= +uv run dev/release/generate-changelog.py apache/main 0.11.0 > dev/changelog/0.11.0.md +``` + +Without `uv`, install `PyGithub` and run the script with `python3` instead. + +The script groups PRs using their labels and [Conventional Commits](https://www.conventionalcommits.org/) title prefixes (`feat:`, `fix:`, `docs:`, `perf:`, and `!` for breaking changes). + +Commit the changelog and create a PR against `main`. + +### 4. Create the release candidate + +You must be a committer to run these steps because they upload to the Apache SVN distribution servers. + +#### Pick a release candidate (RC) number + +Pick numbers in sequential order, with `1` for `rc1`, `2` for `rc2`, etc. + +#### Create a git tag for the release candidate + +The official release artifacts are signed tarballs, but we also tag the commit they were created from. Release tags look like `0.11.0`, and release candidate tags look like `0.11.0-rc1`. + +```shell +git fetch apache +git tag 0.11.0-rc1 apache/main +git push apache 0.11.0-rc1 +``` + +#### Create, sign, and upload artifacts + +```shell +./dev/release/create-tarball.sh 0.11.0 1 +``` + +The script: + +1. Checks that the tag exists and that its `Cargo.toml` has the release version. +2. Prints an email template for the vote. +3. Creates the source tarball from the tag with `git archive` and runs the RAT license check on it. +4. Signs the tarball, writes SHA-256 and SHA-512 checksums, and uploads everything to https://dist.apache.org/repos/dist/dev/datafusion/. + +### 5. Vote on the release candidate + +Send the email printed by `create-tarball.sh` to `dev@datafusion.apache.org`. + +To become official, the release needs at least three +1 votes from PMC members and no -1 votes. The vote must stay open for at least 72 hours so everyone has a chance to review the release candidate. + +#### Verifying release candidates + +`verify-release-candidate.sh` downloads a release candidate and checks its signature and checksums. It then builds and tests it in a sandbox, using the toolchain pinned in `rust-toolchain.toml`, and runs `cargo publish --dry-run`: + +```shell +./dev/release/verify-release-candidate.sh 0.11.0 1 +``` + +To check license headers as well, run `./dev/release/run-rat.sh` on the downloaded tarball. + +#### If changes are requested + +Merge the fixes to `main`, update the changelog, and start again from step 4 with the next RC number. + +#### If the vote passes: announce the result + +Reply to the vote thread with the `[RESULT]` prefix added to the subject line. For example: + +``` +The vote has passed with +1 votes. Thank you to all who helped +with the release verification. +``` + +### 6. Finalize the release + +Only PMC members can do this step, after the release is approved. + +Move the artifacts to the release location in SVN, e.g. https://dist.apache.org/repos/dist/release/datafusion/datafusion-iceberg-0.11.0/, using `release-tarball.sh`: + +```shell +./dev/release/release-tarball.sh 0.11.0 1 +``` + +Congratulations! The release is now official! + +### 7. Create the release git tag + +Tag the same commit as the approved release candidate: + +```shell +git checkout 0.11.0-rc1 +git tag 0.11.0 +git push apache 0.11.0 +``` + +### 8. Publish on crates.io + +Only approved releases of the tarball may be published to crates.io, in order to conform to Apache Software Foundation governance standards. + +Follow [these instructions](https://doc.rust-lang.org/cargo/reference/publishing.html) to create an account and log in to crates.io. Then ask an existing owner of `iceberg-datafusion` to add you; `cargo owner --list iceberg-datafusion` shows the current owners. + +Download and unpack the official release tarball, check that its `Cargo.toml` has the release version, and publish from the unpacked directory: + +```shell +cargo publish -p iceberg-datafusion +``` + +`iceberg-datafusion` is the only published crate. `iceberg-sqllogictest` and `iceberg-playground` set `publish = false`. + +### 9. Add the release to Apache Reporter + +Add the release to [Apache Reporter](https://reporter.apache.org/addrelease.html?datafusion), following the examples from previous releases. The reporter system should send you a reminder email. The release information is used to generate a template for the DataFusion board report. + +### 10. Delete old RCs and releases + +See the ASF documentation on [when to archive](https://www.apache.org/legal/release-policy.html#when-to-archive) for more information. + +Release candidates should be deleted once the release is published. + +To list DataFusion Iceberg release candidates: + +```shell +svn ls https://dist.apache.org/repos/dist/dev/datafusion | grep datafusion-iceberg +``` + +To delete a release candidate: + +```shell +svn delete -m "delete old DataFusion Iceberg RC" https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-iceberg-0.11.0-rc1/ +``` + +Only the latest release should be available from the `release` area. Delete old releases after publishing a new one. + +To list DataFusion Iceberg releases: + +```shell +svn ls https://dist.apache.org/repos/dist/release/datafusion | grep datafusion-iceberg +``` + +To delete a release: + +```shell +svn delete -m "delete old DataFusion Iceberg release" https://dist.apache.org/repos/dist/release/datafusion/datafusion-iceberg-0.11.0 +``` + +## Checking License Headers + +CI runs the Apache Release Audit Tool (RAT) on every pull request. To run it locally, which requires `java` and `python3`: + +```shell +./dev/release/run-rat.sh . +``` + +Files that cannot carry a license header are listed in `dev/release/rat_exclude_files.txt`. diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh new file mode 100755 index 0000000..e4fce98 --- /dev/null +++ b/dev/release/create-tarball.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Adapted from https://github.com/apache/datafusion/tree/main/dev/release/create-tarball.sh + +# This script creates a signed tarball in +# dev/dist/apache-datafusion-iceberg--rc/apache-datafusion-iceberg-.tar.gz, +# uploads it to the "dev" area of the dist.apache.org datafusion repository and +# prints an email for the dev@datafusion.apache.org list to start the vote. +# +# See dev/release/README.md for full release instructions. +# +# Requirements: +# +# 1. gpg set up for signing, with your public key in the DataFusion KEYS file +# 2. svn, logged into the Apache SVN server with your ASF credentials +# 3. java and python3 (for the RAT license check) + +set -euo pipefail + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo "ex. $0 0.11.0 1" + exit 1 +fi + +version=$1 +rc=$2 +tag="${version}-rc${rc}" + +release=apache-datafusion-iceberg-${version} +distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc} +tarname=${release}.tar.gz +tarball=${distdir}/${tarname} +url="https://dist.apache.org/repos/dist/dev/datafusion/${release}-rc${rc}" + +echo "Attempting to create ${tarball} from tag ${tag}" +if ! release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 "${tag}" 2> /dev/null); then + echo "Cannot continue: unknown git tag: ${tag}" + exit 1 +fi + +if ! (cd "${SOURCE_TOP_DIR}" && git show "${release_hash}:Cargo.toml") \ + | grep "^version = \"${version}\"$" > /dev/null; then + echo "Cannot continue: Cargo.toml at ${tag} does not set version = \"${version}\"" + exit 1 +fi + +echo "Draft email for dev@datafusion.apache.org mailing list" +echo "" +echo "---------------------------------------------------------" +cat < containing the files in git at $release_hash +# the files in the tarball are prefixed with ${release} (e.g. apache-datafusion-iceberg-0.11.0) +mkdir -p "${distdir}" +(cd "${SOURCE_TOP_DIR}" && git archive "${release_hash}" --prefix "${release}/" | gzip > "${tarball}") + +echo "Running rat license checker on ${tarball}" +"${SOURCE_DIR}/run-rat.sh" "${tarball}" + +echo "Signing tarball and creating checksums" +gpg --armor --output "${tarball}.asc" --detach-sig "${tarball}" +# create signing with relative path of tarball +# so that they can be verified with a command such as +# shasum --check apache-datafusion-iceberg-0.11.0.tar.gz.sha512 +(cd "${distdir}" && shasum -a 256 "${tarname}") > "${tarball}.sha256" +(cd "${distdir}" && shasum -a 512 "${tarname}") > "${tarball}.sha512" + +echo "Uploading to datafusion dist/dev to ${url}" +svn co --depth=empty https://dist.apache.org/repos/dist/dev/datafusion "${SOURCE_TOP_DIR}/dev/dist" +svn add "${distdir}" +svn ci -m "Apache DataFusion Iceberg ${version} ${rc}" "${distdir}" diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py new file mode 100755 index 0000000..d8b9632 --- /dev/null +++ b/dev/release/generate-changelog.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# /// script +# requires-python = ">=3.9" +# dependencies = ["PyGithub"] +# /// + +# Adapted from https://github.com/apache/datafusion/tree/main/dev/release/generate-changelog.py + +import argparse +import sys +from github import Auth, Github +import os +import re +import subprocess + +def print_pulls(repo_name, title, pulls): + if len(pulls) > 0: + print("**{}:**".format(title)) + print() + for (pull, commit) in pulls: + url = "https://github.com/{}/pull/{}".format(repo_name, pull.number) + print("- {} [#{}]({}) ({})".format(pull.title, pull.number, url, commit.author.login)) + print() + + +def generate_changelog(repo, repo_name, tag1, tag2, version): + + # get a list of commits between two tags + print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr) + comparison = repo.compare(tag1, tag2) + + # get the pull requests for these commits + print("Fetching pull requests", file=sys.stderr) + unique_pulls = [] + all_pulls = [] + for commit in comparison.commits: + print(".", file=sys.stderr, end=" ", flush=True) # progress indicator + pulls = commit.get_pulls() + for pull in pulls: + # there can be multiple commits per PR if squash merge is not being used and + # in this case we should get all the author names, but for now just pick one + if pull.number not in unique_pulls: + unique_pulls.append(pull.number) + all_pulls.append((pull, commit)) + + # we split the pulls into categories + breaking = [] + bugs = [] + docs = [] + enhancements = [] + performance = [] + other = [] + + # categorize the pull requests based on GitHub labels + print("Categorizing pull requests", file=sys.stderr) + for (pull, commit) in all_pulls: + + # see if PR title uses Conventional Commits + cc_type = '' + cc_scope = '' + cc_breaking = '' + parts = re.findall(r'^([a-z]+)(\([a-z]+\))?(!)?:', pull.title) + if len(parts) == 1: + parts_tuple = parts[0] + cc_type = parts_tuple[0] # fix, feat, docs, chore + cc_scope = parts_tuple[1] # component within project + cc_breaking = parts_tuple[2] == '!' + + labels = [label.name for label in pull.labels] + if 'api change' in labels or cc_breaking: + breaking.append((pull, commit)) + elif 'bug' in labels or cc_type == 'fix': + bugs.append((pull, commit)) + elif 'performance' in labels or cc_type == 'perf': + performance.append((pull, commit)) + elif 'enhancement' in labels or cc_type == 'feat': + enhancements.append((pull, commit)) + elif 'documentation' in labels or cc_type == 'docs' or cc_type == 'doc': + docs.append((pull, commit)) + else: + other.append((pull, commit)) + + # produce the changelog content + print("Generating changelog content", file=sys.stderr) + + # ASF header + print("""\n""") + + print(f"# Apache DataFusion Iceberg {version} Changelog\n") + + # get the number of commits + commit_count = subprocess.check_output(f"git log --pretty=oneline {tag1}..{tag2} | wc -l", shell=True, text=True).strip() + + # get number of contributors + contributor_count = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2} | wc -l", shell=True, text=True).strip() + + print(f"This release consists of {commit_count} commits from {contributor_count} contributors. " + f"See credits at the end of this changelog for more information.\n") + + print_pulls(repo_name, "Breaking changes", breaking) + print_pulls(repo_name, "Performance related", performance) + print_pulls(repo_name, "Implemented enhancements", enhancements) + print_pulls(repo_name, "Fixed bugs", bugs) + print_pulls(repo_name, "Documentation updates", docs) + print_pulls(repo_name, "Other", other) + + # show code contributions + credits = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2}", shell=True, text=True).rstrip() + + print("## Credits\n") + print("Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) " + "per contributor.\n") + print("```") + print(credits) + print("```\n") + + print("Thank you also to everyone who contributed in other ways such as filing issues, reviewing " + "PRs, and providing feedback on this release.\n") + +def cli(args=None): + """Process command line arguments.""" + if not args: + args = sys.argv[1:] + + parser = argparse.ArgumentParser() + parser.add_argument("tag1", help="The previous commit or tag (e.g. 0.11.0)") + parser.add_argument("tag2", help="The current commit or tag (e.g. HEAD)") + parser.add_argument("version", help="The version number to include in the changelog") + args = parser.parse_args() + + token = os.getenv("GITHUB_TOKEN") + project = "apache/datafusion-iceberg" + + g = Github(auth=Auth.Token(token) if token else None) + repo = g.get_repo(project) + generate_changelog(repo, project, args.tag1, args.tag2, args.version) + +if __name__ == "__main__": + cli() diff --git a/dev/release/release-tarball.sh b/dev/release/release-tarball.sh new file mode 100755 index 0000000..3b95d89 --- /dev/null +++ b/dev/release/release-tarball.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Adapted from https://github.com/apache/datafusion/tree/main/dev/release/release-tarball.sh + +# This script copies a tarball from the "dev" area of the +# dist.apache.org datafusion repository to the "release" area +# +# This script should only be run after the release has been approved +# by the Apache DataFusion PMC. +# +# See dev/release/README.md for full release instructions + +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo "ex. $0 0.11.0 1" + exit 1 +fi + +version=$1 +rc=$2 + +read -r -p "Proceed to release tarball for ${version}-rc${rc}? [y/N]: " answer +answer=${answer:-no} +if [ "${answer}" != "y" ]; then + echo "Cancelled tarball release!" + exit 1 +fi + +tmp_dir=tmp-apache-datafusion-iceberg-dist + +echo "Recreate temporary directory: ${tmp_dir}" +rm -rf "${tmp_dir}" +mkdir -p "${tmp_dir}" + +echo "Clone dev dist repository" +svn \ + co \ + "https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-iceberg-${version}-rc${rc}" \ + "${tmp_dir}/dev" + +echo "Clone release dist repository" +svn co --depth=empty https://dist.apache.org/repos/dist/release/datafusion "${tmp_dir}/release" + +echo "Copy ${version}-rc${rc} to release working copy" +release_version=datafusion-iceberg-${version} +mkdir -p "${tmp_dir}/release/${release_version}" +cp -r "${tmp_dir}"/dev/* "${tmp_dir}/release/${release_version}/" +svn add "${tmp_dir}/release/${release_version}" + +echo "Commit release" +svn ci -m "Apache DataFusion Iceberg ${version}" "${tmp_dir}/release" + +echo "Clean up" +rm -rf "${tmp_dir}" + +echo "Success! The release is available here:" +echo " https://dist.apache.org/repos/dist/release/datafusion/${release_version}" diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh new file mode 100755 index 0000000..c634a43 --- /dev/null +++ b/dev/release/verify-release-candidate.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Adapted from https://github.com/apache/datafusion/tree/main/dev/release/verify-release-candidate.sh + +# Downloads a release candidate, verifies its signature and checksums, then +# builds and tests it in a sandbox with the toolchain pinned in +# rust-toolchain.toml. Set VERIFY_TMPDIR to keep the sandbox afterwards. + +# Check that required dependencies are installed +check_dependencies() { + local missing_deps=0 + local required_deps=("curl" "git" "gpg" "cc") + + for dep in "${required_deps[@]}"; do + if ! command -v "$dep" &> /dev/null; then + echo "Error: $dep is not installed or not in PATH" + missing_deps=1 + fi + done + + # Either shasum or sha256sum/sha512sum are required + if ! command -v shasum &> /dev/null \ + && ! { command -v sha256sum &> /dev/null && command -v sha512sum &> /dev/null; }; then + echo "Error: Neither shasum nor sha256sum/sha512sum are installed or in PATH" + missing_deps=1 + fi + + if [ $missing_deps -ne 0 ]; then + echo "Please install missing dependencies and try again" + exit 1 + fi +} + +case $# in + 2) VERSION="$1" + RC_NUMBER="$2" + ;; + *) echo "Usage: $0 X.Y.Z RC_NUMBER" + exit 1 + ;; +esac + +set -e +set -x +set -o pipefail + +check_dependencies + +DIST_URL='https://dist.apache.org/repos/dist/dev/datafusion' + +download_dist_file() { + curl \ + --silent \ + --show-error \ + --fail \ + --location \ + --remote-name "${DIST_URL}/$1" +} + +download_rc_file() { + download_dist_file "apache-datafusion-iceberg-${VERSION}-rc${RC_NUMBER}/$1" +} + +import_gpg_keys() { + download_dist_file KEYS + gpg --import KEYS +} + +if type shasum >/dev/null 2>&1; then + sha256_verify="shasum -a 256 -c" + sha512_verify="shasum -a 512 -c" +else + sha256_verify="sha256sum -c" + sha512_verify="sha512sum -c" +fi + +fetch_archive() { + local dist_name=$1 + download_rc_file "${dist_name}.tar.gz" + download_rc_file "${dist_name}.tar.gz.asc" + download_rc_file "${dist_name}.tar.gz.sha256" + download_rc_file "${dist_name}.tar.gz.sha512" + verify_dir_artifact_signatures +} + +verify_dir_artifact_signatures() { + # verify the signature and the checksums of each artifact + find . -name '*.asc' | while read -r sigfile; do + artifact=${sigfile/.asc/} + gpg --verify "$sigfile" "$artifact" || exit 1 + + # go into the directory because the checksum files contain only the + # basename of the artifact + pushd "$(dirname "$artifact")" + base_artifact=$(basename "$artifact") + ${sha256_verify} "$base_artifact.sha256" || exit 1 + ${sha512_verify} "$base_artifact.sha512" || exit 1 + popd + done +} + +setup_tempdir() { + # shellcheck disable=SC2329 # invoked by the EXIT trap below + cleanup() { + if [ "${TEST_SUCCESS}" = "yes" ]; then + rm -fr "${VERIFY_TMPDIR}" + else + echo "Failed to verify release candidate. See ${VERIFY_TMPDIR} for details." + fi + } + + if [ -z "${VERIFY_TMPDIR}" ]; then + # clean up automatically if VERIFY_TMPDIR is not defined + VERIFY_TMPDIR=$(mktemp -d -t "$1.XXXXX") + trap cleanup EXIT + else + # don't clean up automatically + mkdir -p "${VERIFY_TMPDIR}" + fi +} + +test_source_distribution() { + # install a sandboxed rust toolchain that doesn't touch the user's own + export RUSTUP_HOME=$PWD/test-rustup + export CARGO_HOME=$PWD/test-rustup + + curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path + + export PATH=$RUSTUP_HOME/bin:$PATH + # shellcheck disable=SC1091 + source "$RUSTUP_HOME/env" + + # install the version of rust pinned in rust-toolchain.toml + rustup toolchain install + + # raises on any formatting errors + cargo fmt --all -- --check + + cargo test --workspace --locked + + # the published crate must build from its packaged contents, and every + # dependency must come from crates.io + cargo publish --dry-run --locked -p iceberg-datafusion +} + +TEST_SUCCESS=no + +setup_tempdir "datafusion-iceberg-${VERSION}" +echo "Working in sandbox ${VERIFY_TMPDIR}" +cd "${VERIFY_TMPDIR}" + +dist_name="apache-datafusion-iceberg-${VERSION}" +import_gpg_keys +fetch_archive "${dist_name}" +tar xf "${dist_name}.tar.gz" +pushd "${dist_name}" + test_source_distribution +popd + +TEST_SUCCESS=yes +echo 'Release candidate looks good!' +exit 0