From 68a1213610d56e728c5776ac061d9b5d206ff42b Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Fri, 18 Apr 2025 20:56:39 +0800 Subject: [PATCH 1/5] code init --- .gitattributes | 25 ++++++++ .github/workflows/build-docker.yml | 43 ++++++++++++++ .gitignore | 67 ++++++++++++++++++++++ README.md | 1 + task_sync_imgs/README.md | 30 ++++++++++ task_sync_imgs/images.yaml | 0 tool.sh | 92 ++++++++++++++++++++++++++++++ 7 files changed, 258 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/workflows/build-docker.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 task_sync_imgs/README.md create mode 100644 task_sync_imgs/images.yaml create mode 100644 tool.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2c9f817 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,25 @@ +# Ignore all differences in line endings +* text=auto eol=lf +*.md text eol=lf +*.py text eol=lf +*.sh text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.json text eol=lf +*.properties text eol=lf +*.conf text eol=lf +*.ipynb text eol=lf +Dockerfile* text eol=lf +.gitattributes text eol=lf +.gitignore text eol=lf +.dockerignore text eol=lf + +# Files using LFS to track +*.tgz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.jsonl filter=lfs diff=lfs merge=lfs -text +*.xlsx filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 0000000..84fa542 --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,43 @@ +name: build-docker-images + +on: + push: + branches: [ "main" ] + paths-ignore: [ "*.md" ] + + pull_request: + branches: [ "main" ] + paths-ignore: [ "*.md" ] + + workflow_dispatch: # Allows you to run this workflow manually from the Actions tab + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + BUILDKIT_PROGRESS: "plain" # Full logs for CI build. + REGISTRY_SRC: ${{ vars.REGISTRY_SRC || 'docker.io' }} # For BASE_NAMESPACE of images: where to pull base images from, docker.io or other source registry URL. + REGISTRY_DST: ${{ vars.REGISTRY_DST || 'docker.io' }} # For tags of built images: where to push images to, docker.io or other destination registry URL. + # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. + DOCKER_REGISTRY_USERNAME: ${{ vars.DOCKER_REGISTRY_USERNAME }} + DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + # used to sync image to mirror registry + DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }} + DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} + +jobs: + ## Sync all images in this build (listed by "names") to mirror registry. + sync_images: + needs: [] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - env: + AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} + run: | + source ./tool.sh + printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows + printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env + docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-qpod}/docker-kit \ + image-syncer \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9adc602 --- /dev/null +++ b/.gitignore @@ -0,0 +1,67 @@ +# Byte-compiled / optimized / DLL files + +*~ + +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# IDE +.vscode/ +.idea/ + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Mac OS X +.DS_Store + +dockerspawner +dockerspawner.tar.gz +*.orig +.ipynb_checkpoints/ +.vscode/ +.pytest_cache/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..bcb2a6c --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# QPod Lab Container Kit diff --git a/task_sync_imgs/README.md b/task_sync_imgs/README.md new file mode 100644 index 0000000..2eb5a1f --- /dev/null +++ b/task_sync_imgs/README.md @@ -0,0 +1,30 @@ +# Task Sync Images + +This task sync docker images from source registries to target registries based on the configuration files. + +```shell +docker run -it --rm -v $(pwd):/root/app -w /root/app docker.io/qpod/docker-kit +``` + +To sync images in batch, two config files (or combine them in one as `--config`) are needed. + +The `auth.yaml` file should look like: + +```yaml +docker.io: + username: "" + password: "" + insecure: true +registry.cn-hangzhou.aliyuncs.com: + username: "" + password: "" + insecure: true +``` + +The `images.yaml` file should look like: + +```yaml +quay.io/qpod/docker-kit: + - docker.io/qpod/docker-kit + - registry.cn-hangzhou.aliyuncs.com/qpod/docker-kit +``` diff --git a/task_sync_imgs/images.yaml b/task_sync_imgs/images.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tool.sh b/tool.sh new file mode 100644 index 0000000..30030ac --- /dev/null +++ b/tool.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -xu + +CI_PROJECT_NAME=${GITHUB_REPOSITORY:-"QPod/lab-foundation"} +CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"} +CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/') + +if [ "${CI_PROJECT_BRANCH}" = "main" ] ; then + # If on the main branch, docker images namespace will be same as CI_PROJECT_NAME's name space + export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})" ; +else + # not main branch, docker namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE} + export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})0${CI_PROJECT_SPACE}" ; +fi + +export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') +export TAG_SUFFIX="-$(git rev-parse --short HEAD)" + +echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" +echo "--------> DOCKER_IMG_NAMESPACE=${IMG_NAMESPACE}" +echo "--------> DOCKER_IMG_PREFIX_SRC=${IMG_PREFIX_SRC}" +echo "--------> DOCKER_IMG_PREFIX_DST=${IMG_PREFIX_DST}" +echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}" + +[ ! -f /etc/docker/daemon.json ] && sudo tee /etc/docker/daemon.json > /dev/null <<< '{}' +jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ +( sudo service docker restart || true ) && cat /etc/docker/daemon.json && docker info + +build_image() { + echo "$@" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; + docker tag "${IMG_PREFIX_DST}/${IMG}:${TAG}" "${IMG_PREFIX_DST}/${IMG}:${VER}" ; +} + +build_image_no_tag() { + echo "$@" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; +} + +build_image_common() { + echo "$@" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; + docker tag "${IMG_PREFIX_DST}/${IMG}:${TAG}" "${IMG_PREFIX_DST}/${IMG}:${VER}" ; +} + +alias_image() { + IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; + docker tag "${IMG_PREFIX_DST}/${IMG_1}:${TAG_1}" "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" ; + docker tag "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" "${IMG_PREFIX_DST}/${IMG_2}:${VER}" ; +} + +push_image() { + KEYWORD="${1:-second}"; + docker image prune --force && docker images | sort; + IMAGES=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; + echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_DST}" -u "$DOCKER_REGISTRY_USERNAME" --password-stdin ; + for IMG in $(echo "${IMAGES}" | tr " " "\n") ; + do + docker push "${IMG}"; + status=$?; + echo "[${status}] Image pushed > ${IMG}"; + done +} + +clear_images() { + KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images build days ago + IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; + IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ; + + for IMG in $(echo "$IMGS_1 $IMGS_2" | tr " " "\n") ; do + docker rmi "${IMG}" || true; status=$?; echo "[${status}] image removed > ${IMG}"; + done + docker image prune --force && docker images ; +} + + +remove_folder() { + sudo du -h -d1 "$1" || true ; + sudo rm -rf "$1" || true ; +} + +free_diskspace() { + remove_folder /usr/share/dotnet + remove_folder /usr/local/lib/android + # remove_folder /var/lib/docker + df -h +} From 11c2a891575e8450f077dcf2da15c280cf3771a6 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Fri, 18 Apr 2025 14:12:54 +0000 Subject: [PATCH 2/5] debug --- .github/workflows/build-docker.yml | 2 +- task_sync_imgs/README.md | 2 ++ task_sync_imgs/images.yaml | 27 +++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 84fa542..9dc51e3 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -40,4 +40,4 @@ jobs: printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-qpod}/docker-kit \ - image-syncer \ No newline at end of file + image-syncer --proc=8 --retries=2 --images ./images.yaml --auth /tmp/.github/workflows/auth.json diff --git a/task_sync_imgs/README.md b/task_sync_imgs/README.md index 2eb5a1f..419dec7 100644 --- a/task_sync_imgs/README.md +++ b/task_sync_imgs/README.md @@ -4,6 +4,8 @@ This task sync docker images from source registries to target registries based o ```shell docker run -it --rm -v $(pwd):/root/app -w /root/app docker.io/qpod/docker-kit + +image-syncer --proc=8 --retries=2 --images ./images.yaml --auth ./auth.json ``` To sync images in batch, two config files (or combine them in one as `--config`) are needed. diff --git a/task_sync_imgs/images.yaml b/task_sync_imgs/images.yaml index e69de29..d6c98c0 100644 --- a/task_sync_imgs/images.yaml +++ b/task_sync_imgs/images.yaml @@ -0,0 +1,27 @@ +# Ref: https://github.com/AliyunContainerService/image-syncer/blob/master/README-zh_CN.md + +# Elasticserach +# - ref1: https://github.com/elastic/elasticsearch/releases +# - ref2: https://www.elastic.co/docs/deploy-manage/deploy/self-managed/install-elasticsearch-with-docker +docker.elastic.co/elasticsearch/elasticsearch:7.17.28,8.17.5,8.18.0,9.0.0: + - quay.io/qpod/elasticsearch + - registry.cn-hangzhou.aliyuncs.com/qpod/elasticsearch + - registry.cn-beijing.aliyuncs.com/qpod/elasticsearch + + +# Minio +# - ref1: https://github.com/minio/minio/releases +# - ref2: https://min.io/docs/minio/container/index.html +docker.io/minio/minio:latest,RELEASE.2023-12-20T01-00-02Z,RELEASE.2025-04-08T15-41-24Z: + - quay.io/qpod/minio + - registry.cn-hangzhou.aliyuncs.com/qpod/minio + - registry.cn-beijing.aliyuncs.com/qpod/minio + + +# valkey +# - ref1: https://github.com/valkey-io/valkey/releases +# - ref2: https://hub.docker.com/r/valkey/valkey +docker.io/valkey/valkey:latest,8,8.1.,8.1.0: + - quay.io/qpod/valkey + - registry.cn-hangzhou.aliyuncs.com/qpod/valkey + - registry.cn-beijing.aliyuncs.com/qpod/valkey From d14edb33faf2e54aeb6e04a36ad40beaec0f70f5 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Fri, 18 Apr 2025 14:19:57 +0000 Subject: [PATCH 3/5] debug path --- .github/workflows/build-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 9dc51e3..24d2d63 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -36,8 +36,8 @@ jobs: - env: AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} run: | - source ./tool.sh + source ./tool.sh && cd task_sync_imgs && ls -alh printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env - docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp ${IMG_PREFIX_DST:-qpod}/docker-kit \ + docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp qpod/docker-kit \ image-syncer --proc=8 --retries=2 --images ./images.yaml --auth /tmp/.github/workflows/auth.json From 41d543fd5586692ed3d4dcfc2855c713ab6af8be Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Fri, 18 Apr 2025 14:23:13 +0000 Subject: [PATCH 4/5] debug folder --- .github/workflows/build-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 24d2d63..494a23c 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -36,8 +36,8 @@ jobs: - env: AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} run: | - source ./tool.sh && cd task_sync_imgs && ls -alh + source ./tool.sh && ls -alh ./ printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp qpod/docker-kit \ - image-syncer --proc=8 --retries=2 --images ./images.yaml --auth /tmp/.github/workflows/auth.json + image-syncer --proc=8 --retries=2 --images /tmp/task_sync_imgs/images.yaml --auth /tmp/.github/workflows/auth.json From 1d4f2c2bdc8c703cc6447cac9140cddf3a71ef3b Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Fri, 18 Apr 2025 14:37:31 +0000 Subject: [PATCH 5/5] update images --- task_sync_imgs/images.yaml | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/task_sync_imgs/images.yaml b/task_sync_imgs/images.yaml index d6c98c0..b8b37e3 100644 --- a/task_sync_imgs/images.yaml +++ b/task_sync_imgs/images.yaml @@ -21,7 +21,37 @@ docker.io/minio/minio:latest,RELEASE.2023-12-20T01-00-02Z,RELEASE.2025-04-08T15- # valkey # - ref1: https://github.com/valkey-io/valkey/releases # - ref2: https://hub.docker.com/r/valkey/valkey -docker.io/valkey/valkey:latest,8,8.1.,8.1.0: +docker.io/valkey/valkey:latest,8,8.1,8.1.0: - quay.io/qpod/valkey - registry.cn-hangzhou.aliyuncs.com/qpod/valkey - registry.cn-beijing.aliyuncs.com/qpod/valkey + + +# k3s +# - ref1: https://github.com/k3s-io/k3s/releases +# - ref2: https://docs.k3s.io/installation/airgap +rancher/k3s:latest,v1.32.3-k3s1: + - quay.io/qpod/k3s + - registry.cn-hangzhou.aliyuncs.com/qpod/k3s + - registry.cn-beijing.aliyuncs.com/qpod/k3s + + +# k3d: https://github.com/k3d-io/k3d/pkgs/container/k3d +ghcr.io/k3d-io/k3d:latest,5-dind,5: + - quay.io/qpod/k3s + - registry.cn-hangzhou.aliyuncs.com/qpod/k3d + - registry.cn-beijing.aliyuncs.com/qpod/k3d + + +# k3d-proxy:https://github.com/k3d-io/k3d/pkgs/container/k3d-proxy +ghcr.io/k3d-io/k3d-proxy:latest,5: + - quay.io/qpod/k3s + - registry.cn-hangzhou.aliyuncs.com/qpod/k3d-proxy + - registry.cn-beijing.aliyuncs.com/qpod/k3d-proxy + + +# k3d-tools: https://github.com/k3d-io/k3d/pkgs/container/k3d-tools +ghcr.io/k3d-io/k3d-tools:latest,5: + - quay.io/qpod/k3s + - registry.cn-hangzhou.aliyuncs.com/qpod/k3d-tools + - registry.cn-beijing.aliyuncs.com/qpod/k3d-tools