From 11844e1c7464ccc4bdad6424831c78627c67c168 Mon Sep 17 00:00:00 2001 From: haobibo Date: Tue, 4 Nov 2025 03:50:22 +0800 Subject: [PATCH] debug --- .github/workflows/build-docker-db.yml | 62 +++++++++++++++++++++ .github/workflows/build-docker-gpu.yml | 47 ++++++++-------- .github/workflows/build-docker.yml | 57 ++++++-------------- tool.sh | 74 +++++++++++++------------- 4 files changed, 143 insertions(+), 97 deletions(-) create mode 100644 .github/workflows/build-docker-db.yml diff --git a/.github/workflows/build-docker-db.yml b/.github/workflows/build-docker-db.yml new file mode 100644 index 00000000..a31fec73 --- /dev/null +++ b/.github/workflows/build-docker-db.yml @@ -0,0 +1,62 @@ +name: build-docker-images-dbs + +on: + push: + branches: [ "main" ] + paths-ignore: [ "*.md" ] + + pull_request: + branches: [ "main" ] + paths-ignore: [ "*.md" ] + + workflow_dispatch: # Allows you to run this workflow manually from the Actions tab + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + BUILDKIT_PROGRESS: "plain" # Full logs for CI build. + REGISTRY_SRC: ${{ vars.REGISTRY_SRC || 'quay.io' }} # For BASE_NAMESPACE of images: where to pull base images from, docker.io or other source registry URL. + REGISTRY_DST: ${{ vars.REGISTRY_DST || 'quay.io' }} # For tags of built images: where to push images to, docker.io or other destination registry URL. + # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. + DOCKER_REGISTRY_USERNAME: ${{ vars.DOCKER_REGISTRY_USERNAME }} + DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + # used to sync image to mirror registry + DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }} + DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} + CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-foundation' }} + + +jobs: + ## Postgres + job-postgres: + name: 'postgres-17' + strategy: + matrix: + include: [{arch: amd64, runner: ubuntu-latest}, {arch: arm64, runner: ubuntu-24.04-arm}] + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - run: | + source ./tool.sh + PUSH='false' TAG_VER='false' IMG_PREFIX_DST='' build_image postgres-17 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:17" + docker images + IMG_PREFIX_SRC='' build_image postgres-17 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-17" --build-arg "PYTHON_VERSION=3.12" --build-context docker.io/library/postgres-17=docker-image://postgres-17:latest + + + ## Sync all images in this build (listed by "names") to mirror registry. + sync_images: + needs: ["job-postgres"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - env: + AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} + DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }} + run: | + source ./tool.sh + printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows + printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env + docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-labnow}/docker-kit \ + python /opt/utils/image-syncer/run_jobs.py --auth-file=/tmp/.github/workflows/auth.json diff --git a/.github/workflows/build-docker-gpu.yml b/.github/workflows/build-docker-gpu.yml index 8837b5fb..d91c1630 100644 --- a/.github/workflows/build-docker-gpu.yml +++ b/.github/workflows/build-docker-gpu.yml @@ -37,30 +37,34 @@ jobs: # reserved for vllm: https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile job-cuda_128: name: 'cuda_12.8,cuda,nvidia-cuda' - runs-on: ubuntu-latest + strategy: + matrix: + include: [{arch: amd64, runner: ubuntu-latest}] # , {arch: arm64, runner: ubuntu-24.04-arm} + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 - run: | source ./tool.sh build_image_no_tag nvidia-cuda 12.8.1-cudnn-devel-ubuntu24.04 docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04" && clear_images nvidia/cuda export IMG_PREFIX_SRC="${IMG_PREFIX_DST}" - build_image_no_tag tmp latest docker_base/Dockerfile --build-arg "BASE_IMG=nvidia-cuda:12.8.1-cudnn-devel-ubuntu24.04" - build_image cuda_12.8 latest docker_cuda/nvidia-cuda.Dockerfile --build-arg "BASE_IMG=tmp" - push_image cuda + PUSH=false TAG_VER=false build_image tmp latest docker_base/Dockerfile --build-arg "BASE_IMG=nvidia-cuda:12.8.1-cudnn-devel-ubuntu24.04" + build_image cuda_12.8 latest docker_cuda/nvidia-cuda.Dockerfile --build-arg "BASE_IMG=tmp" --build-context ${IMG_PREFIX_SRC}/tmp:latest=docker-image://${IMG_PREFIX_SRC}/tmp:latest job-cuda_126: name: 'cuda_12.6' - runs-on: ubuntu-latest + strategy: + matrix: + include: [{arch: amd64, runner: ubuntu-latest}] # , {arch: arm64, runner: ubuntu-24.04-arm} + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 - run: | source ./tool.sh build_image_no_tag nvidia-cuda 12.6.3-cudnn-devel-ubuntu24.04 docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:12.6.3-cudnn-devel-ubuntu24.04" && clear_images nvidia/cuda export IMG_PREFIX_SRC="${IMG_PREFIX_DST}" - build_image_no_tag tmp latest docker_base/Dockerfile --build-arg "BASE_IMG=nvidia-cuda:12.6.3-cudnn-devel-ubuntu24.04" - build_image cuda_12.6 latest docker_cuda/nvidia-cuda.Dockerfile --build-arg "BASE_IMG=tmp" - alias_image cuda_12.6 latest cuda latest - push_image cuda + PUSH=false TAG_VER=false build_image tmp latest docker_base/Dockerfile --build-arg "BASE_IMG=nvidia-cuda:12.6.3-cudnn-devel-ubuntu24.04" + build_image cuda_12.6 latest docker_cuda/nvidia-cuda.Dockerfile --build-arg "BASE_IMG=tmp" --build-context ${IMG_PREFIX_SRC}/tmp:latest=docker-image://${IMG_PREFIX_SRC}/tmp:latest + build_image cuda latest docker_cuda/nvidia-cuda.Dockerfile --build-arg "BASE_IMG=tmp" --build-context ${IMG_PREFIX_SRC}/tmp:latest=docker-image://${IMG_PREFIX_SRC}/tmp:latest job-tf2: name: 'tf2,tf2-cuda126' @@ -71,8 +75,7 @@ jobs: - run: | source ./tool.sh build_image tf2-cuda126 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.6" --build-arg "ARG_PROFILE_PYTHON=tf2" - alias_image tf2-cuda126 latest tf2 latest - push_image + build_image tf2 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.6" --build-arg "ARG_PROFILE_PYTHON=tf2" job-torch_cuda128: name: 'torch,torch-cuda128' @@ -83,8 +86,7 @@ jobs: - run: | source ./tool.sh build_image torch-cuda128 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.8" --build-arg "ARG_PROFILE_PYTHON=torch" - alias_image torch-cuda128 latest torch latest - push_image + build_image torch latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.8" --build-arg "ARG_PROFILE_PYTHON=torch" job-paddle_cuda126: name: 'paddle-cuda126,paddle-3.0' @@ -95,8 +97,7 @@ jobs: - run: | source ./tool.sh build_image paddle-cuda126 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.6" --build-arg "ARG_PROFILE_PYTHON=paddle,mkl" - alias_image paddle-cuda126 latest paddle-3.0 latest - push_image + build_image paddle-3.0 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.6" --build-arg "ARG_PROFILE_PYTHON=paddle,mkl" job-py-nlp: @@ -108,8 +109,7 @@ jobs: - run: | source ./tool.sh && export IMG_PREFIX_SRC="${IMG_PREFIX_DST}" build_image py-nlp-cuda128 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.8" --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,torch,nlp" - alias_image py-nlp-cuda128 latest py-nlp latest - push_image + build_image py-nlp latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.8" --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,torch,nlp" job-py-cv: name: 'py-cv' @@ -120,7 +120,6 @@ jobs: - run: | source ./tool.sh && export IMG_PREFIX_SRC="${IMG_PREFIX_DST}" build_image py-cv latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_12.8" --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,torch,cv" - push_image job-core-cuda: @@ -131,14 +130,20 @@ jobs: - uses: actions/checkout@v4 - run: | source ./tool.sh && free_diskspace && export IMG_PREFIX_SRC="${IMG_PREFIX_DST}" - build_image full-cuda-12.8 latest docker_core/Dockerfile \ + build_image full-cuda-12.8 latest docker_core/Dockerfile \ + --build-arg "BASE_IMG=cuda_12.8" \ + --build-arg "ARG_PROFILE_PYTHON=base,datascience,mkl,database,nlp,cv,chem,tf2,torch" \ + --build-arg "ARG_PROFILE_R=base,datascience" \ + --build-arg "ARG_PROFILE_NODEJS=base" \ + --build-arg "ARG_PROFILE_JAVA=base,maven" \ + --build-arg "ARG_PROFILE_LATEX=base,cjk" + build_image core-cuda latest docker_core/Dockerfile \ --build-arg "BASE_IMG=cuda_12.8" \ --build-arg "ARG_PROFILE_PYTHON=base,datascience,mkl,database,nlp,cv,chem,tf2,torch" \ --build-arg "ARG_PROFILE_R=base,datascience" \ --build-arg "ARG_PROFILE_NODEJS=base" \ --build-arg "ARG_PROFILE_JAVA=base,maven" \ --build-arg "ARG_PROFILE_LATEX=base,cjk" - alias_image full-cuda-12.8 latest core-cuda latest && push_image cuda job-nvidia-ctk: name: 'nvidia-ctk' @@ -147,7 +152,7 @@ jobs: - uses: actions/checkout@v4 - run: | source ./tool.sh && export IMG_PREFIX_SRC="docker.io/library" - build_image nvidia-ctk latest docker_cuda/nvidia-ctk.Dockerfile && push_image + build_image nvidia-ctk latest docker_cuda/nvidia-ctk.Dockerfile ## Sync all images in this build (listed by "names") to mirror registry. diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 85e656c3..3f959a83 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -26,6 +26,8 @@ env: DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }} DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} CI_PROJECT_NAME: ${{ vars.CI_PROJECT_NAME || 'LabNow/lab-foundation' }} + DOCKER_PLATFORM: amd64,arm64 + jobs: job-base: @@ -35,10 +37,10 @@ jobs: - uses: actions/checkout@v4 - run: | source ./tool.sh - build_image atom latest docker_atom/Dockerfile && push_image + build_image atom latest docker_atom/Dockerfile export IMG_PREFIX_SRC="${IMG_PREFIX_DST}" build_image python-3.12 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.12" - alias_image python-3.12 latest base latest && push_image + build_image base latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.12" job-python: name: 'python-3.7,python-3.13' @@ -49,8 +51,8 @@ jobs: - run: | source ./tool.sh export IMG_PREFIX_SRC="${IMG_PREFIX_DST}" - build_image python-3.13 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.13" && push_image - build_image python-3.7 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.7" && push_image + build_image python-3.13 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.13" + build_image python-3.7 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.7" job-core: @@ -67,7 +69,6 @@ jobs: --build-arg "ARG_PROFILE_NODEJS=base" \ --build-arg "ARG_PROFILE_JAVA=base,maven" \ --build-arg "ARG_PROFILE_LATEX=base,cjk" - push_image core job-py-data: name: 'py-data' @@ -75,7 +76,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: source ./tool.sh && build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database" && push_image + - run: source ./tool.sh && build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database" job-py-chem: name: 'py-chem' @@ -83,7 +84,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: source ./tool.sh && build_image py-chem latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,chem,torch" && push_image + - run: source ./tool.sh && build_image py-chem latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,chem,torch" job-py-std: name: 'py-std' @@ -91,7 +92,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: source ./tool.sh && build_image py-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database,nlp,cv,chem" && push_image + - run: source ./tool.sh && build_image py-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database,nlp,cv,chem" job-py-jdk: name: 'py-jdk' @@ -104,7 +105,6 @@ jobs: build_image py-jdk latest docker_core/Dockerfile \ --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database,nlp,cv,chem" \ --build-arg "ARG_PROFILE_JAVA=base" - push_image job-r-base: name: 'r-base' @@ -112,7 +112,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: source ./tool.sh && build_image r-base latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base" && push_image + - run: source ./tool.sh && build_image r-base latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base" job-r-std: name: 'r-std' @@ -120,7 +120,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: source ./tool.sh && build_image r-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base,datascience" --build-arg "ARG_PROFILE_JAVA=base" && push_image + - run: source ./tool.sh && build_image r-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base,datascience" --build-arg "ARG_PROFILE_JAVA=base" job-node: name: 'node' @@ -128,7 +128,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: source ./tool.sh && build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" && push_image + - run: source ./tool.sh && build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" job-bun: name: 'bun' @@ -136,7 +136,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: source ./tool.sh && build_image bun latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=bun" && push_image + - run: source ./tool.sh && build_image bun latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=bun" job-jdk: @@ -151,7 +151,6 @@ jobs: build_image jdk-17 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" --build-arg "VERSION_JDK=17" build_image jdk-11 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" --build-arg "VERSION_JDK=11" build_image jdk-8 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" --build-arg "VERSION_JDK=8" - alias_image jdk-11 latest jdk latest && push_image jdk job-julia: name: 'julia' @@ -159,7 +158,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: source ./tool.sh && build_image julia latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base" && push_image + - run: source ./tool.sh && build_image julia latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base" job-go: name: 'go,go-stack' @@ -172,7 +171,6 @@ jobs: build_image go latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base" export IMG_PREFIX_SRC="${IMG_PREFIX_DST}" build_image go-stack latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base" --build-arg "ARG_PROFILE_NODEJS=base" - push_image go job-rust: name: 'rust' @@ -180,7 +178,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: source ./tool.sh && build_image rust latest docker_core/Dockerfile --build-arg "ARG_PROFILE_RUST=base" && push_image + - run: source ./tool.sh && build_image rust latest docker_core/Dockerfile --build-arg "ARG_PROFILE_RUST=base" job-data-science-stack: name: 'data-science-stack' @@ -195,7 +193,6 @@ jobs: --build-arg "ARG_PROFILE_R=base,datascience" \ --build-arg "ARG_PROFILE_NODEJS=base" \ --build-arg "ARG_PROFILE_JULIA=base" - push_image data-science-stack job-full-stack: name: 'full-stack' @@ -213,26 +210,6 @@ jobs: --build-arg "ARG_PROFILE_JULIA=base" \ --build-arg "ARG_PROFILE_GO=base" \ --build-arg "ARG_PROFILE_RUST=base" - push_image full-stack - - ## Postgres - job-postgres: - name: 'postgres-17,postgres-16,postgres-15' - needs: ["job-base"] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: | - source ./tool.sh - build_image_no_tag postgres-17 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:17" - build_image_no_tag postgres-16 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:16" - build_image_no_tag postgres-15 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:15" - - export IMG_PREFIX_SRC="${IMG_PREFIX_DST}" - build_image postgres-17 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-17" --build-arg "PYTHON_VERSION=3.12" - build_image postgres-16 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-16" --build-arg "PYTHON_VERSION=3.12" - build_image postgres-15 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-15" --build-arg "PYTHON_VERSION=3.12" - push_image postgres- ## DockerKit - including image-syncer and docker-compsoe job-docker_kit: @@ -245,7 +222,7 @@ jobs: AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} DOCKER_MIRROR_REGISTRY: ${{ vars.DOCKER_MIRROR_REGISTRY }} run: | - source ./tool.sh && build_image docker-kit latest docker_docker_kit/Dockerfile && push_image docker-kit + source ./tool.sh && build_image docker-kit latest docker_docker_kit/Dockerfile echo "jobs: { job-docker_kit: { name: 'docker-kit' } }" | tee ".github/workflows/build-docker.yml" > /dev/null printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env @@ -254,7 +231,7 @@ jobs: ## Sync all images in this build (listed by "names") to mirror registry. sync_images: - needs: ["job-docker_kit", "job-postgres", "job-full-stack", "job-core"] + needs: ["job-docker_kit", "job-full-stack", "job-core"] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/tool.sh b/tool.sh index 415d41f8..f0287ec4 100755 --- a/tool.sh +++ b/tool.sh @@ -5,13 +5,10 @@ CI_PROJECT_NAME=${CI_PROJECT_NAME:-$GITHUB_REPOSITORY} CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"} CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/') -if [ "${CI_PROJECT_BRANCH}" = "main" ] ; then - # If on the main branch, docker images namespace will be same as CI_PROJECT_NAME's name space - export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})" ; -else - # not main branch, docker namespace = {CI_PROJECT_NAME's name space} + "-" + {1st substr before / in CI_PROJECT_SPACE} - export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})0${CI_PROJECT_SPACE}" ; -fi +# If on the main branch, image namespace will be same as CI_PROJECT_NAME's name space; +# else (not main branch), image namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE}. +[ "${CI_PROJECT_BRANCH}" = "main" ] && NAMESPACE_SUFFIX="" || NAMESPACE_SUFFIX="0${CI_PROJECT_SPACE}" ; +export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})${NAMESPACE_SUFFIX}" ; export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') @@ -24,41 +21,35 @@ echo "--------> DOCKER_IMG_PREFIX_SRC=${IMG_PREFIX_SRC}" echo "--------> DOCKER_IMG_PREFIX_DST=${IMG_PREFIX_DST}" echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}" -[ ! -f /etc/docker/daemon.json ] && sudo tee /etc/docker/daemon.json > /dev/null <<< '{}' -jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ -( sudo service docker restart || true ) && cat /etc/docker/daemon.json && docker info +build_image_dry_run() { + IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)"; + PLATFORM=${DOCKER_PLATFORM:-$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')} ; + echo "[DRY RUN] docker buildx build --compress --force-rm=true --platform ${PLATFORM} -t ${IMG_PREFIX_DST}/${IMG}:${TAG} -f ${FILE} --build-arg BASE_NAMESPACE=${IMG_PREFIX_SRC} $@ ${WORKDIR}" ; +} build_image() { - echo "$@" ; - IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)"; - docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; - docker tag "${IMG_PREFIX_DST}/${IMG}:${TAG}" "${IMG_PREFIX_DST}/${IMG}:${VER}" ; -} + IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)"; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; + PLATFORM=${DOCKER_PLATFORM:-$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')} ; -build_image_no_tag() { - echo "$@" ; - IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)"; - docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; -} + PUSH=${PUSH:-true} ; TAG_VER=${TAG_VER:-true} ; TAGS_EXTRA=${TAGS_EXTRA:-} ; + TAG_ARGS=("-t" "${IMG_PREFIX_DST:+$IMG_PREFIX_DST/}${IMG}:${TAG}"); [ "$TAG_VER" = "true" ] && TAG_ARGS+=("-t" "${IMG_PREFIX_DST}/${IMG}:${VER}"); for t in ${TAGS_EXTRA}; do TAG_ARGS+=("-t" "${IMG_PREFIX_DST}/${IMG}:${t}"); done ; + OPT_OUTPUT=$([ "$PUSH" = "true" ] && echo "--push" || echo "--load") ; -build_image_common() { - echo "$@" ; - IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)"; - docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; - docker tag "${IMG_PREFIX_DST}/${IMG}:${TAG}" "${IMG_PREFIX_DST}/${IMG}:${VER}" ; + docker buildx use multiarch || docker buildx create --name multiarch --driver docker-container --use ; + docker buildx inspect ; + registry_login ; + docker buildx build --compress --force-rm=true --platform "${PLATFORM}" ${OPT_OUTPUT} "${TAG_ARGS[@]}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; } -alias_image() { - IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; - docker tag "${IMG_PREFIX_DST}/${IMG_1}:${TAG_1}" "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" ; - docker tag "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" "${IMG_PREFIX_DST}/${IMG_2}:${VER}" ; +registry_login() { + echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_DST}" -u "$DOCKER_REGISTRY_USERNAME" --password-stdin ; } push_image() { KEYWORD="${1:-second}"; docker image prune --force && docker images | sort; IMAGES=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; - echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_DST}" -u "$DOCKER_REGISTRY_USERNAME" --password-stdin ; + registry_login ; for IMG in $(echo "${IMAGES}" | tr " " "\n") ; do docker push "${IMG}"; @@ -78,15 +69,26 @@ clear_images() { docker image prune --force && docker images ; } - remove_folder() { - sudo du -h -d1 "$1" || true ; - sudo rm -rf "$1" || true ; + for dir in "$@"; do + if [ -d "$dir" ]; then + echo "Removing folder: $dir" ; + sudo du -h -d1 "$dir" || true ; + sudo rm -rf "$dir" || true ; + else + echo "Warn: directory not found: $dir" ; + fi + done } free_diskspace() { - remove_folder /usr/share/dotnet - remove_folder /usr/local/lib/android - # remove_folder /var/lib/docker + remove_folder /usr/share/dotnet ; # /usr/local/lib/android /var/lib/docker df -h } + +setup_github_actions() { + [ ! -f /etc/docker/daemon.json ] && sudo tee /etc/docker/daemon.json > /dev/null <<< '{}' ; + jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ ; + ( sudo service docker restart || true ) && cat /etc/docker/daemon.json && docker info ; +} +[ "$GITHUB_ACTIONS" = "true" ] && echo "Running in GitHub Actions and Setup Env: $(setup_github_actions)"