Skip to content

Commit 145a009

Browse files
mtsokoliindyk
mtsokol
and
iindyk
committed
GitHub Actions build refactor
Co-authored-by: Ihor Indyk <iindyk@google.com>
1 parent 85ceac5 commit 145a009

10 files changed

Lines changed: 188 additions & 119 deletions

‎.bazelrc‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎.github/workflows/build_and_publish_template.yml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# This workflow builds ArrayRecord wheels and uploads them as artifacts.
2+
3+
name: Build & Publish Template
4+
5+
on:
6+
workflow_call:
7+
inputs:
8+
pypi_project_url:
9+
required: true
10+
type: string
11+
is_nightly:
12+
required: true
13+
type: boolean
14+
15+
defaults:
16+
run:
17+
shell: bash
18+
19+
permissions:
20+
contents: read
21+
22+
jobs:
23+
build-and-test:
24+
name: "Python ${{ matrix.python-version }} on ${{ matrix.os }}"
25+
runs-on: "${{ matrix.os }}"
26+
27+
strategy:
28+
fail-fast: false
29+
matrix:
30+
python-version: ["3.10", "3.11", "3.12", "3.13"]
31+
os: [ubuntu-22.04, ubuntu-22.04-arm, macos-14]
32+
33+
env:
34+
USE_BAZEL_VERSION: "7.2.1"
35+
steps:
36+
- name: Set up Bazel
37+
uses: bazel-contrib/setup-bazel@0.15.0
38+
- name: Check Bazel installation
39+
run: |
40+
which bazel
41+
bazel version
42+
- name: Set up Python
43+
uses: actions/setup-python@v5
44+
with:
45+
python-version: ${{ matrix.python-version }}
46+
- uses: "actions/checkout@v3"
47+
- name: Create directory
48+
run: |
49+
mkdir -p /tmp/array_record
50+
cp -r . /tmp/array_record
51+
- name: Build package
52+
run: |
53+
set -xe
54+
export PYTHON_VERSION=${{ matrix.python-version }}
55+
export PYTHON_MAJOR_VERSION=$(echo $PYTHON_VERSION | cut -d. -f1)
56+
export PYTHON_MINOR_VERSION=$(echo $PYTHON_VERSION | cut -d. -f2)
57+
export BAZEL_VERSION="7.2.1"
58+
export OUTPUT_DIR="/tmp/array_record"
59+
export SOURCE_DIR="/tmp/array_record"
60+
. "${SOURCE_DIR}"'/oss/runner_common.sh'
61+
build_and_test_array_record
62+
- name: Upload ArrayRecord artifacts
63+
uses: actions/upload-artifact@v4
64+
with:
65+
name: built-array-record-wheels-${{ matrix.os }}-${{ matrix.python-version }}
66+
path: /tmp/grain/all_dist/*.whl
67+
68+
publish-wheel:
69+
runs-on: ubuntu-22.04
70+
needs: build-and-test
71+
permissions:
72+
id-token: write
73+
environment:
74+
name: pypi
75+
url: ${{ inputs.pypi_project_url }}
76+
steps:
77+
- name: Download ArrayRecord artifacts
78+
uses: actions/download-artifact@v4
79+
with:
80+
pattern: built-array-record-wheels-*
81+
path: dist/
82+
merge-multiple: true
83+
- name: Publish package distributions to PyPI
84+
uses: pypa/gh-action-pypi-publish@release/v1
85+
with:
86+
attestations: false
87+
verbose: true

‎.github/workflows/publish_release.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
name: Build and Publish Release
2+
3+
on: workflow_dispatch
4+
5+
jobs:
6+
call-workflow:
7+
uses: ./.github/workflows/build_and_publish_template.yml
8+
permissions:
9+
contents: read
10+
id-token: write
11+
with:
12+
pypi_project_url: https://pypi.org/project/array-record
13+
is_nightly: false

‎.gitignore‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Byte-compiled / optimized / DLL files
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
6+
# Bazel outputs
7+
bazel-array_record
8+
bazel-bin
9+
bazel-out
10+
bazel-testlogs
11+
12+
MODULE.bazel.lock

‎oss/build.Dockerfile‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,24 @@ ARG AUDITWHEEL_PLATFORM
66
FROM quay.io/pypa/${AUDITWHEEL_PLATFORM}
77

88
ARG PYTHON_VERSION
9-
ARG PYTHON_BIN
109
ARG BAZEL_VERSION
1110

1211
ENV DEBIAN_FRONTEND=noninteractive
1312

1413
RUN ulimit -n 1024 && yum install -y rsync
14+
ENV PYTHON_BIN_PATH=/opt/python/cp${PYTHON_VERSION}-cp${PYTHON_VERSION}/bin
1515
ENV PATH="${PYTHON_BIN}:${PATH}"
1616

17+
ENV PYTHON_BIN=${PYTHON_BIN_PATH}/python
18+
1719
# Download the correct bazel version and make sure it's on path.
1820
RUN BAZEL_ARCH_SUFFIX="$(uname -m | sed s/aarch64/arm64/)" \
1921
&& curl -sSL --fail -o /usr/local/bin/bazel "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-linux-$BAZEL_ARCH_SUFFIX" \
2022
&& chmod a+x /usr/local/bin/bazel
2123

2224
# Install dependencies needed for array_record.
2325
RUN --mount=type=cache,target=/root/.cache \
24-
${PYTHON_BIN}/python -m pip install -U \
26+
$PYTHON_BIN -m pip install -U \
2527
absl-py \
2628
auditwheel \
2729
etils[epath] \
@@ -30,4 +32,4 @@ RUN --mount=type=cache,target=/root/.cache \
3032
twine \
3133
wheel;
3234

33-
WORKDIR "/tmp/array_record"
35+
WORKDIR "/tmp/array_record"

‎oss/build_whl.sh‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,7 @@
55

66
set -e -x
77

8-
if [ -z ${PYTHON_BIN} ]; then
9-
if [ -z ${PYTHON_VERSION} ]; then
10-
PYTHON_BIN=$(which python3)
11-
else
12-
PYTHON_BIN=$(which python${PYTHON_VERSION})
13-
fi
14-
fi
15-
16-
PYTHON_MAJOR_VERSION=$(${PYTHON_BIN} -c 'import sys; print(sys.version_info.major)')
17-
PYTHON_MINOR_VERSION=$(${PYTHON_BIN} -c 'import sys; print(sys.version_info.minor)')
18-
PYTHON_VERSION="${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}"
19-
export PYTHON_VERSION="${PYTHON_VERSION}"
8+
OUTPUT_DIR="${OUTPUT_DIR:-/tmp/array_record}"
209

2110
function write_to_bazelrc() {
2211
echo "$1" >> .bazelrc
@@ -31,6 +20,11 @@ function main() {
3120
write_to_bazelrc "build --host_cxxopt=-std=c++17"
3221
write_to_bazelrc "build --experimental_repo_remote_exec"
3322
write_to_bazelrc "build --python_path=\"${PYTHON_BIN}\""
23+
write_to_bazelrc "test --python_path=\"${PYTHON_BIN}\""
24+
PLATFORM="$(uname)"
25+
if [[ "$PLATFORM" != "Darwin" ]]; then
26+
write_to_bazelrc "build --linkopt=\"-lrt -lm\""
27+
fi
3428

3529
if [ -n "${CROSSTOOL_TOP}" ]; then
3630
write_to_bazelrc "build --crosstool_top=${CROSSTOOL_TOP}"
@@ -39,8 +33,8 @@ function main() {
3933

4034
export USE_BAZEL_VERSION="${BAZEL_VERSION}"
4135
bazel clean
42-
bazel build ...
43-
bazel test --verbose_failures --test_output=errors ...
36+
bazel build ... --action_env MACOSX_DEPLOYMENT_TARGET='11.0' --action_env PYTHON_BIN_PATH="${PYTHON_BIN}"
37+
bazel test --verbose_failures --test_output=errors ... --action_env PYTHON_BIN_PATH="${PYTHON_BIN}"
4438

4539
DEST="/tmp/array_record/all_dist"
4640
# Create the directory, then do dirname on a non-existent file inside it to
@@ -68,7 +62,11 @@ function main() {
6862

6963
pushd ${TMPDIR}
7064
echo $(date) : "=== Building wheel"
71-
${PYTHON_BIN} setup.py bdist_wheel --python-tag py3${PYTHON_MINOR_VERSION}
65+
if [ "$(uname)" = "Darwin" ]; then
66+
"$PYTHON_BIN" setup.py bdist_wheel --python-tag py3"${PYTHON_MINOR_VERSION}" --plat-name macosx_11_0_"$(uname -m)"
67+
else
68+
"$PYTHON_BIN" setup.py bdist_wheel --python-tag py3"${PYTHON_MINOR_VERSION}"
69+
fi
7270

7371
if [ -n "${AUDITWHEEL_PLATFORM}" ]; then
7472
echo $(date) : "=== Auditing wheel"
@@ -81,6 +79,15 @@ function main() {
8179
popd
8280

8381
echo $(date) : "=== Output wheel file is in: ${DEST}"
82+
83+
# Install ArrayRecord from the wheel and run smoke tests.
84+
# TF is not available on Python 3.13 and above.
85+
if (( "${PYTHON_MINOR_VERSION}" < 13 )); then
86+
$PYTHON_BIN -m pip install --find-links=/tmp/grain/all_dist --pre array-record
87+
$PYTHON_BIN -m pip install jax tensorflow grain
88+
$PYTHON_BIN oss/test_with_grain.py
89+
$PYTHON_BIN oss/test_with_tf.py
90+
fi
8491
}
8592

8693
main

‎oss/runner_common.sh‎

Lines changed: 36 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -1,102 +1,48 @@
11
#!/bin/bash
22

3-
# Builds ArrayRecord from source code located in SOURCE_DIR producing wheels
4-
# under $SOURCE_DIR/all_dist.
5-
function build_and_test_array_record_linux() {
6-
SOURCE_DIR=$1
7-
8-
# Automatically decide which platform to build for by checking on which
9-
# platform this runs.
10-
AUDITWHEEL_PLATFORM="manylinux2014_$(uname -m)"
3+
set -e -x
114

12-
# Using a previous version of Blaze to avoid:
13-
# https://github.com/bazelbuild/bazel/issues/8622
14-
export BAZEL_VERSION="8.0.0"
5+
OUTPUT_DIR="${OUTPUT_DIR:-/tmp/array_record}"
156

16-
# Build wheels for multiple Python minor versions.
17-
PYTHON_MAJOR_VERSION=3
18-
for PYTHON_MINOR_VERSION in 10 11 12
19-
do
20-
PYTHON_VERSION=${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}
21-
PYTHON_BIN=/opt/python/cp${PYTHON_MAJOR_VERSION}${PYTHON_MINOR_VERSION}-cp${PYTHON_MAJOR_VERSION}${PYTHON_MINOR_VERSION}/bin
7+
setup_env_vars_py() {
8+
# This controls the python binary to use.
9+
PYTHON_MAJOR_VERSION="$1"
10+
PYTHON_MINOR_VERSION="$2"
11+
PYENV_PYTHON_VERSION="${PYTHON_MAJOR_VERSION}"'.'"${PYTHON_MINOR_VERSION}"
12+
PYTHON='python'"${PYENV_PYTHON_VERSION}"
13+
export PYTHON
14+
PYTHON_BIN="$(which python)"
15+
export PYTHON_BIN
16+
}
2217

23-
# Cleanup older images.
18+
# Builds ArrayRecord from source code located in SOURCE_DIR producing wheels
19+
# under $SOURCE_DIR/all_dist.
20+
build_and_test_array_record() {
21+
printf 'Creating ArrayRecord wheel for Python Version %s\n' "$PYTHON_VERSION"
22+
if [ "$(uname)" = "Darwin" ]; then
23+
setup_env_vars_py "$PYTHON_MAJOR_VERSION" "$PYTHON_MINOR_VERSION"
24+
"$PYTHON_BIN" -m pip install -U setuptools wheel etils[epath]
25+
sh "${SOURCE_DIR}"'/oss/build_whl.sh'
26+
else
27+
# Automatically decide which platform to build for by checking on which
28+
# platform this runs.
29+
AUDITWHEEL_PLATFORM='manylinux2014_'"$(uname -m)"
2430
docker rmi -f array_record:${PYTHON_VERSION}
2531
docker rm -f array_record
26-
2732
DOCKER_BUILDKIT=1 docker build --progress=plain --no-cache \
28-
--build-arg AUDITWHEEL_PLATFORM=${AUDITWHEEL_PLATFORM} \
29-
--build-arg PYTHON_VERSION=${PYTHON_VERSION} \
30-
--build-arg PYTHON_BIN=${PYTHON_BIN} \
31-
--build-arg BAZEL_VERSION=${BAZEL_VERSION} \
32-
-t array_record:${PYTHON_VERSION} - < ${SOURCE_DIR}/oss/build.Dockerfile
33+
--build-arg AUDITWHEEL_PLATFORM="${AUDITWHEEL_PLATFORM}" \
34+
--build-arg PYTHON_VERSION="${PYTHON_MAJOR_VERSION}""${PYTHON_MINOR_VERSION}" \
35+
--build-arg BAZEL_VERSION="${BAZEL_VERSION}" \
36+
-t array_record:"${PYTHON_VERSION}" "${SOURCE_DIR}"'/oss'
3337

3438
docker run --rm -a stdin -a stdout -a stderr \
35-
--env PYTHON_BIN="${PYTHON_BIN}/python" \
36-
--env BAZEL_VERSION=${BAZEL_VERSION} \
37-
--env AUDITWHEEL_PLATFORM=${AUDITWHEEL_PLATFORM} \
38-
-v $SOURCE_DIR:/tmp/array_record \
39-
--name array_record array_record:${PYTHON_VERSION} \
40-
bash oss/build_whl.sh
41-
done
42-
43-
ls ${SOURCE_DIR}/all_dist/*.whl
44-
}
45-
46-
function install_and_init_pyenv {
47-
pyenv_root=${1:-$HOME/.pyenv}
48-
export PYENV_ROOT=$pyenv_root
49-
if [[ ! -d $PYENV_ROOT ]]; then
50-
echo "Installing pyenv.."
51-
git clone https://github.com/pyenv/pyenv.git "$PYENV_ROOT"
52-
export PATH="/home/kbuilder/.local/bin:$PYENV_ROOT/bin:$PATH"
53-
eval "$(pyenv init --path)"
39+
--env PYTHON_VERSION="${PYTHON_MAJOR_VERSION}"'.'"${PYTHON_MINOR_VERSION}" \
40+
--env PYTHON_MAJOR_VERSION="${PYTHON_MAJOR_VERSION}" \
41+
--env PYTHON_MINOR_VERSION="${PYTHON_MINOR_VERSION}" \
42+
--env BAZEL_VERSION="${BAZEL_VERSION}" \
43+
--env AUDITWHEEL_PLATFORM="${AUDITWHEEL_PLATFORM}" \
44+
-v "${SOURCE_DIR}":"${OUTPUT_DIR}" \
45+
--name array_record array_record:"${PYTHON_VERSION}" \
46+
sh oss/build_whl.sh
5447
fi
55-
56-
echo "Python setup..."
57-
pyenv install -s "$PYENV_PYTHON_VERSION"
58-
pyenv global "$PYENV_PYTHON_VERSION"
59-
PYTHON=$(pyenv which python)
60-
}
61-
62-
function setup_env_vars_py310 {
63-
# This controls the python binary to use.
64-
PYTHON=python3.10
65-
PYTHON_STR=python3.10
66-
PYTHON_MAJOR_VERSION=3
67-
PYTHON_MINOR_VERSION=10
68-
# This is for pyenv install.
69-
PYENV_PYTHON_VERSION=3.10.13
70-
}
71-
72-
function update_bazel_macos {
73-
BAZEL_VERSION=$1
74-
ARCH="$(uname -m)"
75-
curl -L https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-darwin-${ARCH}.sh -O
76-
ls
77-
chmod +x bazel-*.sh
78-
./bazel-${BAZEL_VERSION}-installer-darwin-${ARCH}.sh --user
79-
rm -f ./bazel-${BAZEL_VERSION}-installer-darwin-${ARCH}.sh
80-
# Add new bazel installation to path
81-
PATH="/Users/kbuilder/bin:$PATH"
82-
}
83-
84-
function build_and_test_array_record_macos() {
85-
SOURCE_DIR=$1
86-
# Set up Bazel.
87-
# Using a previous version of Bazel to avoid:
88-
# https://github.com/bazelbuild/bazel/issues/8622
89-
export BAZEL_VERSION="8.0.0"
90-
update_bazel_macos ${BAZEL_VERSION}
91-
bazel --version
92-
93-
# Set up Pyenv.
94-
setup_env_vars_py310
95-
install_and_init_pyenv
96-
97-
# Build and test ArrayRecord.
98-
cd ${SOURCE_DIR}
99-
bash ${SOURCE_DIR}/oss/build_whl.sh
100-
101-
ls ${SOURCE_DIR}/all_dist/*.whl
10248
}

‎oss/test_with_grain.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import grain.python as grain
2+
print("imported grain")
3+
4+
import tensorflow as tf
5+
print("imported TF", flush=True)

‎oss/test_with_tf.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import tensorflow as tf
2+
print(f"iindyk: imported TF", flush=True)
3+
from array_record.python import array_record_module
4+
5+
print("tf", flush=True)
6+
print(f"iindyk: Done {tf.Tensor}", flush=True)

0 commit comments

Comments
 (0)