Skip to content

add support for multi arch #128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
haobibo wants to merge 1 commit into
base: main
Choose a base branch
from
Open

add support for multi arch #128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions .github/workflows/build-docker-db.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
name: build-docker-images-dbs

on:
push:
branches: [ "main" ]
paths-ignore: [ "*.md" ]

pull_request:
branches: [ "main" ]
paths-ignore: [ "*.md" ]

workflow_dispatch: # Allows you to run this workflow manually from the Actions tab

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

env:
BUILDKIT_PROGRESS: "plain" # Full logs for CI build.
REGISTRY_SRC: ${{ vars.REGISTRY_SRC || 'quay.io' }} # For BASE_NAMESPACE of images: where to pull base images from, docker.io or other source registry URL.
REGISTRY_DST: ${{ vars.REGISTRY_DST || 'quay.io' }} # For tags of built images: where to push images to, docker.io or other destination registry URL.
# DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets.
DOCKER_REGISTRY_USERNAME: ${{ vars.DOCKER_REGISTRY_USERNAME }}
DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
# used to sync image to mirror registry
DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }}
DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }}
CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-foundation' }}


jobs:
## Postgres
job-postgres:
name: 'postgres-17'
strategy:
matrix:
include: [{arch: amd64, runner: ubuntu-latest}, {arch: arm64, runner: ubuntu-24.04-arm}]
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v4
- run: |
source ./tool.sh
PUSH='false' TAG_VER='false' IMG_PREFIX_DST='' build_image postgres-17 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:17"
docker images
IMG_PREFIX_SRC='' build_image postgres-17 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-17" --build-arg "PYTHON_VERSION=3.12" --build-context docker.io/library/postgres-17=docker-image://postgres-17:latest


## Sync all images in this build (listed by "names") to mirror registry.
sync_images:
needs: ["job-postgres"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- env:
AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }}
DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }}
run: |
source ./tool.sh
printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows
printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env
docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \
python /opt/utils/image-syncer/run_jobs.py --auth-file=/tmp/.github/workflows/auth.json
47 changes: 26 additions & 21 deletions .github/workflows/build-docker-gpu.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,30 +37,34 @@ jobs:
# reserved for vllm: https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile
job-cuda_128:
name: 'cuda_12.8,cuda,nvidia-cuda'
runs-on: ubuntu-latest
strategy:
matrix:
include: [{arch: amd64, runner: ubuntu-latest}] # , {arch: arm64, runner: ubuntu-24.04-arm}
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v4
- run: |
source ./tool.sh
build_image_no_tag nvidia-cuda 12.8.1-cudnn-devel-ubuntu24.04 docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04" && clear_images nvidia/cuda
export IMG_PREFIX_SRC="${IMG_PREFIX_DST}"
build_image_no_tag tmp latest docker_base/Dockerfile --build-arg "BASE_IMG=nvidia-cuda:12.8.1-cudnn-devel-ubuntu24.04"
build_image cuda_12.8 latest docker_cuda/nvidia-cuda.Dockerfile --build-arg "BASE_IMG=tmp"
push_image cuda
PUSH=false TAG_VER=false build_image tmp latest docker_base/Dockerfile --build-arg "BASE_IMG=nvidia-cuda:12.8.1-cudnn-devel-ubuntu24.04"
build_image cuda_12.8 latest docker_cuda/nvidia-cuda.Dockerfile --build-arg "BASE_IMG=tmp" --build-context ${IMG_PREFIX_SRC}/tmp:latest=docker-image://${IMG_PREFIX_SRC}/tmp:latest

job-cuda_126:
name: 'cuda_12.6'
runs-on: ubuntu-latest
strategy:
matrix:
include: [{arch: amd64, runner: ubuntu-latest}] # , {arch: arm64, runner: ubuntu-24.04-arm}
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v4
- run: |
source ./tool.sh
build_image_no_tag nvidia-cuda 12.6.3-cudnn-devel-ubuntu24.04 docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:12.6.3-cudnn-devel-ubuntu24.04" && clear_images nvidia/cuda
export IMG_PREFIX_SRC="${IMG_PREFIX_DST}"
build_image_no_tag tmp latest docker_base/Dockerfile --build-arg "BASE_IMG=nvidia-cuda:12.6.3-cudnn-devel-ubuntu24.04"
build_image cuda_12.6 latest docker_cuda/nvidia-cuda.Dockerfile --build-arg "BASE_IMG=tmp"
alias_image cuda_12.6 latest cuda latest
push_image cuda
PUSH=false TAG_VER=false build_image tmp latest docker_base/Dockerfile --build-arg "BASE_IMG=nvidia-cuda:12.6.3-cudnn-devel-ubuntu24.04"
build_image cuda_12.6 latest docker_cuda/nvidia-cuda.Dockerfile --build-arg "BASE_IMG=tmp" --build-context ${IMG_PREFIX_SRC}/tmp:latest=docker-image://${IMG_PREFIX_SRC}/tmp:latest
build_image cuda latest docker_cuda/nvidia-cuda.Dockerfile --build-arg "BASE_IMG=tmp" --build-context ${IMG_PREFIX_SRC}/tmp:latest=docker-image://${IMG_PREFIX_SRC}/tmp:latest

job-tf2:
name: 'tf2,tf2-cuda126'
Expand All @@ -71,8 +75,7 @@ jobs:
- run: |
source ./tool.sh
build_image tf2-cuda126 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.6" --build-arg "ARG_PROFILE_PYTHON=tf2"
alias_image tf2-cuda126 latest tf2 latest
push_image
build_image tf2 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.6" --build-arg "ARG_PROFILE_PYTHON=tf2"

job-torch_cuda128:
name: 'torch,torch-cuda128'
Expand All @@ -83,8 +86,7 @@ jobs:
- run: |
source ./tool.sh
build_image torch-cuda128 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.8" --build-arg "ARG_PROFILE_PYTHON=torch"
alias_image torch-cuda128 latest torch latest
push_image
build_image torch latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.8" --build-arg "ARG_PROFILE_PYTHON=torch"

job-paddle_cuda126:
name: 'paddle-cuda126,paddle-3.0'
Expand All @@ -95,8 +97,7 @@ jobs:
- run: |
source ./tool.sh
build_image paddle-cuda126 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.6" --build-arg "ARG_PROFILE_PYTHON=paddle,mkl"
alias_image paddle-cuda126 latest paddle-3.0 latest
push_image
build_image paddle-3.0 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.6" --build-arg "ARG_PROFILE_PYTHON=paddle,mkl"


job-py-nlp:
Expand All @@ -108,8 +109,7 @@ jobs:
- run: |
source ./tool.sh && export IMG_PREFIX_SRC="${IMG_PREFIX_DST}"
build_image py-nlp-cuda128 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.8" --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,torch,nlp"
alias_image py-nlp-cuda128 latest py-nlp latest
push_image
build_image py-nlp latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.8" --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,torch,nlp"

job-py-cv:
name: 'py-cv'
Expand All @@ -120,7 +120,6 @@ jobs:
- run: |
source ./tool.sh && export IMG_PREFIX_SRC="${IMG_PREFIX_DST}"
build_image py-cv latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.8" --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,torch,cv"
push_image


job-core-cuda:
Expand All @@ -131,14 +130,20 @@ jobs:
- uses: actions/checkout@v4
- run: |
source ./tool.sh && free_diskspace && export IMG_PREFIX_SRC="${IMG_PREFIX_DST}"
build_image full-cuda-12.8 latest docker_core/Dockerfile \
build_image full-cuda-12.8 latest docker_core/Dockerfile \
--build-arg "BASE_IMG=cuda_12.8" \
--build-arg "ARG_PROFILE_PYTHON=base,datascience,mkl,database,nlp,cv,chem,tf2,torch" \
--build-arg "ARG_PROFILE_R=base,datascience" \
--build-arg "ARG_PROFILE_NODEJS=base" \
--build-arg "ARG_PROFILE_JAVA=base,maven" \
--build-arg "ARG_PROFILE_LATEX=base,cjk"
build_image core-cuda latest docker_core/Dockerfile \
--build-arg "BASE_IMG=cuda_12.8" \
--build-arg "ARG_PROFILE_PYTHON=base,datascience,mkl,database,nlp,cv,chem,tf2,torch" \
--build-arg "ARG_PROFILE_R=base,datascience" \
--build-arg "ARG_PROFILE_NODEJS=base" \
--build-arg "ARG_PROFILE_JAVA=base,maven" \
--build-arg "ARG_PROFILE_LATEX=base,cjk"
alias_image full-cuda-12.8 latest core-cuda latest && push_image cuda

job-nvidia-ctk:
name: 'nvidia-ctk'
Expand All @@ -147,7 +152,7 @@ jobs:
- uses: actions/checkout@v4
- run: |
source ./tool.sh && export IMG_PREFIX_SRC="docker.io/library"
build_image nvidia-ctk latest docker_cuda/nvidia-ctk.Dockerfile && push_image
build_image nvidia-ctk latest docker_cuda/nvidia-ctk.Dockerfile


## Sync all images in this build (listed by "names") to mirror registry.
Expand Down
57 changes: 17 additions & 40 deletions .github/workflows/build-docker.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ env:
DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }}
DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }}
CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-foundation' }}
DOCKER_PLATFORM: amd64,arm64


jobs:
job-base:
Expand All @@ -35,10 +37,10 @@ jobs:
- uses: actions/checkout@v4
- run: |
source ./tool.sh
build_image atom latest docker_atom/Dockerfile && push_image
build_image atom latest docker_atom/Dockerfile
export IMG_PREFIX_SRC="${IMG_PREFIX_DST}"
build_image python-3.12 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.12"
alias_image python-3.12 latest base latest && push_image
build_image base latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.12"

job-python:
name: 'python-3.7,python-3.13'
Expand All @@ -49,8 +51,8 @@ jobs:
- run: |
source ./tool.sh
export IMG_PREFIX_SRC="${IMG_PREFIX_DST}"
build_image python-3.13 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.13" && push_image
build_image python-3.7 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.7" && push_image
build_image python-3.13 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.13"
build_image python-3.7 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.7"


job-core:
Expand All @@ -67,31 +69,30 @@ jobs:
--build-arg "ARG_PROFILE_NODEJS=base" \
--build-arg "ARG_PROFILE_JAVA=base,maven" \
--build-arg "ARG_PROFILE_LATEX=base,cjk"
push_image core

job-py-data:
name: 'py-data'
needs: job-base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: source ./tool.sh && build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database" && push_image
- run: source ./tool.sh && build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database"

job-py-chem:
name: 'py-chem'
needs: job-base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: source ./tool.sh && build_image py-chem latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,chem,torch" && push_image
- run: source ./tool.sh && build_image py-chem latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,chem,torch"

job-py-std:
name: 'py-std'
needs: job-base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: source ./tool.sh && build_image py-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database,nlp,cv,chem" && push_image
- run: source ./tool.sh && build_image py-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database,nlp,cv,chem"

job-py-jdk:
name: 'py-jdk'
Expand All @@ -104,39 +105,38 @@ jobs:
build_image py-jdk latest docker_core/Dockerfile \
--build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database,nlp,cv,chem" \
--build-arg "ARG_PROFILE_JAVA=base"
push_image

job-r-base:
name: 'r-base'
needs: job-base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: source ./tool.sh && build_image r-base latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base" && push_image
- run: source ./tool.sh && build_image r-base latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base"

job-r-std:
name: 'r-std'
needs: job-base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: source ./tool.sh && build_image r-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base,datascience" --build-arg "ARG_PROFILE_JAVA=base" && push_image
- run: source ./tool.sh && build_image r-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base,datascience" --build-arg "ARG_PROFILE_JAVA=base"

job-node:
name: 'node'
needs: job-base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: source ./tool.sh && build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" && push_image
- run: source ./tool.sh && build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base"

job-bun:
name: 'bun'
needs: job-base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: source ./tool.sh && build_image bun latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=bun" && push_image
- run: source ./tool.sh && build_image bun latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=bun"


job-jdk:
Expand All @@ -151,15 +151,14 @@ jobs:
build_image jdk-17 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" --build-arg "VERSION_JDK=17"
build_image jdk-11 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" --build-arg "VERSION_JDK=11"
build_image jdk-8 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" --build-arg "VERSION_JDK=8"
alias_image jdk-11 latest jdk latest && push_image jdk

job-julia:
name: 'julia'
needs: job-base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: source ./tool.sh && build_image julia latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base" && push_image
- run: source ./tool.sh && build_image julia latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base"

job-go:
name: 'go,go-stack'
Expand All @@ -172,15 +171,14 @@ jobs:
build_image go latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base"
export IMG_PREFIX_SRC="${IMG_PREFIX_DST}"
build_image go-stack latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base" --build-arg "ARG_PROFILE_NODEJS=base"
push_image go

job-rust:
name: 'rust'
needs: job-base
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: source ./tool.sh && build_image rust latest docker_core/Dockerfile --build-arg "ARG_PROFILE_RUST=base" && push_image
- run: source ./tool.sh && build_image rust latest docker_core/Dockerfile --build-arg "ARG_PROFILE_RUST=base"

job-data-science-stack:
name: 'data-science-stack'
Expand All @@ -195,7 +193,6 @@ jobs:
--build-arg "ARG_PROFILE_R=base,datascience" \
--build-arg "ARG_PROFILE_NODEJS=base" \
--build-arg "ARG_PROFILE_JULIA=base"
push_image data-science-stack

job-full-stack:
name: 'full-stack'
Expand All @@ -213,26 +210,6 @@ jobs:
--build-arg "ARG_PROFILE_JULIA=base" \
--build-arg "ARG_PROFILE_GO=base" \
--build-arg "ARG_PROFILE_RUST=base"
push_image full-stack

## Postgres
job-postgres:
name: 'postgres-17,postgres-16,postgres-15'
needs: ["job-base"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: |
source ./tool.sh
build_image_no_tag postgres-17 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:17"
build_image_no_tag postgres-16 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:16"
build_image_no_tag postgres-15 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:15"

export IMG_PREFIX_SRC="${IMG_PREFIX_DST}"
build_image postgres-17 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-17" --build-arg "PYTHON_VERSION=3.12"
build_image postgres-16 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-16" --build-arg "PYTHON_VERSION=3.12"
build_image postgres-15 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-15" --build-arg "PYTHON_VERSION=3.12"
push_image postgres-

## DockerKit - including image-syncer and docker-compsoe
job-docker_kit:
Expand All @@ -245,7 +222,7 @@ jobs:
AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }}
DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }}
run: |
source ./tool.sh && build_image docker-kit latest docker_docker_kit/Dockerfile && push_image docker-kit
source ./tool.sh && build_image docker-kit latest docker_docker_kit/Dockerfile
echo "jobs: { job-docker_kit: { name: 'docker-kit' } }" | tee ".github/workflows/build-docker.yml" > /dev/null
printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows
printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env
Expand All @@ -254,7 +231,7 @@ jobs:

## Sync all images in this build (listed by "names") to mirror registry.
sync_images:
needs: ["job-docker_kit", "job-postgres", "job-full-stack", "job-core"]
needs: ["job-docker_kit", "job-full-stack", "job-core"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
Expand Down
Loading