From 2ed73e658eac825b2d88cd2ecea5ba1c2fd2a0ec Mon Sep 17 00:00:00 2001 From: Nathan Probert Date: Fri, 28 Aug 2026 23:45:20 -0400 Subject: [PATCH] Combine C/Rust compile into one cached Docker build image --- .github/workflows/deploy.yml | 26 +++++++++++++++--------- build_scripts/Dockerfile | 18 +++++++++++++++++ build_scripts/build.sh | 38 +++++++++++++++++++++++++++++++++++ build_scripts/compile.sh | 30 --------------------------- build_scripts/deploy.sh | 14 +++---------- build_scripts/rust_compile.sh | 33 ------------------------------ 6 files changed, 76 insertions(+), 83 deletions(-) create mode 100644 build_scripts/Dockerfile create mode 100644 build_scripts/build.sh delete mode 100755 build_scripts/compile.sh delete mode 100644 build_scripts/rust_compile.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 0d15a3f..7281016 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -17,11 +17,6 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - - name: Install build tools - run: | - sudo apt-get update - sudo apt-get install -y build-essential - - name: Set up Python id: setup-python uses: actions/setup-python@v6 @@ -51,11 +46,24 @@ jobs: restore-keys: | rust-build-${{ runner.os }}- - - name: Compile C code - run: make compile_c + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container - - name: Compile Rust code (Linux Lambda target) - run: cargo build --release --target x86_64-unknown-linux-gnu --manifest-path smartscore/Rust/make_predictions/Cargo.toml + - name: Build compiler environment image + uses: docker/build-push-action@v6 + with: + context: . + file: build_scripts/Dockerfile + push: false + load: true + tags: smartscore-build-env:latest + cache-from: type=gha,scope=smartscore-build-env + cache-to: type=gha,mode=max,scope=smartscore-build-env + + - name: Compile C and Rust code (Linux Lambda target) + run: sh build_scripts/build.sh - name: Install compiled Rust module into venv run: | diff --git a/build_scripts/Dockerfile b/build_scripts/Dockerfile new file mode 100644 index 0000000..3370c1a --- /dev/null +++ b/build_scripts/Dockerfile @@ -0,0 +1,18 @@ +# Compiler environment for SmartScore's C and Rust extensions. +# +# Built on manylinux_2_28 so the produced artifacts link against glibc 2.28, +# making them safe to run on the Amazon Linux 2023 Lambda runtime. The image +# provides gcc/g++/make (for the C extension) plus a pinned Rust toolchain +# (for the pyo3 extension). +# +# The image layers (base + Rust toolchain) are cached in CI via Docker BuildKit +# so the toolchain install only happens once instead of on every run. +FROM quay.io/pypa/manylinux_2_28_x86_64 + +# Install a pinned Rust toolchain via rustup. "minimal" profile keeps it small. +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \ + -y --profile minimal --default-toolchain 1.85.0 + +ENV PATH="/root/.cargo/bin:${PATH}" + +WORKDIR /project \ No newline at end of file diff --git a/build_scripts/build.sh b/build_scripts/build.sh new file mode 100644 index 0000000..9c19d7f --- /dev/null +++ b/build_scripts/build.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Compiles SmartScore's C and Rust extensions inside a single manylinux +# container (see build_scripts/Dockerfile). +# +# In CI the compiler image is built and cached with Docker BuildKit before +# this script runs; if it is missing (e.g. first local use), it is built here. +# +# Outputs (written into the mounted workspace, matching the layout the rest of +# the build/deploy expects): +# smartscore/compiled_code.so +# smartscore/Rust/make_predictions/target/x86_64-unknown-linux-gnu/release/libmake_predictions_rust.so + +SCRIPT_DIR=$(dirname "$(realpath "$0")") +PROJECT_PATH=$(dirname "$SCRIPT_DIR") + +# Convert to Windows path only when cygpath is available (Git Bash on Windows). +if command -v cygpath.exe >/dev/null 2>&1; then + PROJECT_PATH="$(cygpath.exe -C ANSI -w -p "${PROJECT_PATH}")" +fi + +IMAGE=${BUILD_IMAGE:-smartscore-build-env:latest} + +# Build the compiler image on first use (local machines). +if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then + echo "Building compiler environment image $IMAGE..." + docker build -t "$IMAGE" -f "$SCRIPT_DIR/Dockerfile" "$PROJECT_PATH" +fi + +echo "Compiling C and Rust code in $IMAGE..." +docker run --rm -v "$PROJECT_PATH:/project" "$IMAGE" sh -c " + cd /project + make compile_c + cd /project/smartscore/Rust/make_predictions + cargo build --release --target x86_64-unknown-linux-gnu +" + +echo "Compilation completed. Check the artifacts in $PROJECT_PATH/smartscore/" \ No newline at end of file diff --git a/build_scripts/compile.sh b/build_scripts/compile.sh deleted file mode 100755 index c09860e..0000000 --- a/build_scripts/compile.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$(dirname "$(realpath "$0")") -PROJECT_PATH=$(dirname "$SCRIPT_DIR") - -# Convert to Windows path only when cygpath is available (Git Bash on Windows). -if command -v cygpath.exe >/dev/null 2>&1; then - PROJECT_PATH="$(cygpath.exe -C ANSI -w -p "${PROJECT_PATH}")" -fi - -# Run the Docker container with volume mounting -if ! docker run --rm -v "$PROJECT_PATH:/project" amazonlinux:2 sh -c " - # Install required tools - yum update -y - yum install -y gcc gcc-c++ make - - # Navigate to the C source directory - cd /project - - # Compile the C code into a shared object - make compile -"; then - echo "Running for linux environment." - echo "If you are on windows, ensure docker is running." - - make compile -fi - -# Print a message upon successful completion -echo "Compilation completed. Check the compiled_code.so in $PROJECT_PATH/smartscore/" diff --git a/build_scripts/deploy.sh b/build_scripts/deploy.sh index 14d7ce7..5217ac1 100755 --- a/build_scripts/deploy.sh +++ b/build_scripts/deploy.sh @@ -290,17 +290,9 @@ if [ "${DEPLOY_SKIP_BUILD:-0}" = "1" ]; then fi echo "Skipping C/Rust compilation (reusing artifacts from CI setup job)..." else - # compile C code - echo "Compiling C code..." - sh build_scripts/compile.sh - if [ $? -ne 0 ]; then - echo "Error: Compilation failed. Ensure docker is running." - exit 1 - fi - - # compile Rust code - echo "Compiling Rust code..." - sh build_scripts/rust_compile.sh + # compile C and Rust code + echo "Compiling C and Rust code..." + sh build_scripts/build.sh if [ $? -ne 0 ]; then echo "Error: Compilation failed. Ensure docker is running." exit 1 diff --git a/build_scripts/rust_compile.sh b/build_scripts/rust_compile.sh deleted file mode 100644 index 7337b6f..0000000 --- a/build_scripts/rust_compile.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$(dirname "$(realpath "$0")") -PROJECT_PATH=$(dirname "$SCRIPT_DIR") - -# Convert to Windows path only when cygpath is available (Git Bash on Windows). -if command -v cygpath.exe >/dev/null 2>&1; then - PROJECT_PATH="$(cygpath.exe -C ANSI -w -p "${PROJECT_PATH}")" -fi - -# Run the Docker container with volume mounting -if ! docker run --rm -v "$PROJECT_PATH:/project" quay.io/pypa/manylinux_2_28_x86_64 sh -c " - # Install required tools for Rust - yum update -y - yum groupinstall -y 'Development Tools' - yum install -y rust-toolset - - # Navigate to the Rust project directory - cd /project - cd /smartscore/Rust/make_predictions - - # Compile the Rust code (assuming it's using Cargo) - cargo build --release --target x86_64-unknown-linux-gnu -"; then - echo "Running for linux environment." - echo "If you are on windows, ensure docker is running." - cd "$PROJECT_PATH/smartscore/Rust/make_predictions" - - cargo build --release --target x86_64-unknown-linux-gnu -fi - -# Print a message upon successful completion -echo "Compilation completed. Check the target/release directory in $PROJECT_PATH/smartscore/"