Init ctk - sync jobs

.gitattributes

@@ -0,0 +1,25 @@
+# Ignore all differences in line endings
+*               text=auto eol=lf
+*.md            text eol=lf
+*.py            text eol=lf
+*.sh            text eol=lf
+*.yml           text eol=lf
+*.yaml          text eol=lf
+*.json          text eol=lf
+*.properties    text eol=lf
+*.conf          text eol=lf
+*.ipynb         text eol=lf
+Dockerfile*     text eol=lf
+.gitattributes  text eol=lf
+.gitignore      text eol=lf
+.dockerignore   text eol=lf
+
+# Files using LFS to track
+*.tgz   filter=lfs diff=lfs merge=lfs -text
+*.h5    filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text
+*.xlsx  filter=lfs diff=lfs merge=lfs -text
+*.bin   filter=lfs diff=lfs merge=lfs -text
+*.png   filter=lfs diff=lfs merge=lfs -text
+*.jpg   filter=lfs diff=lfs merge=lfs -text
+*.jpeg  filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-docker.yml

@@ -0,0 +1,43 @@
+name: build-docker-images
+
+on:
+  push:
+    branches: [ "main" ]
+    paths-ignore: [ "*.md" ]
+
+  pull_request:
+    branches: [ "main" ]
+    paths-ignore: [ "*.md" ]
+
+  workflow_dispatch:  # Allows you to run this workflow manually from the Actions tab
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  BUILDKIT_PROGRESS: "plain"  # Full logs for CI build.
+  REGISTRY_SRC: ${{ vars.REGISTRY_SRC || 'docker.io' }} # For BASE_NAMESPACE of images: where to pull base images from, docker.io or other source registry URL.
+  REGISTRY_DST: ${{ vars.REGISTRY_DST || 'docker.io' }} # For tags of built images: where to push images to, docker.io or other destination registry URL.
+  # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets.
+  DOCKER_REGISTRY_USERNAME: ${{ vars.DOCKER_REGISTRY_USERNAME }}
+  DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
+  # used to sync image to mirror registry
+  DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }}
+  DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }}
+
+jobs:
+  ## Sync all images in this build (listed by "names") to mirror registry.
+  sync_images:
+    needs: []
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - env:
+          AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }}
+        run: |
+          source ./tool.sh && ls -alh ./
+          printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows
+          printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env
+          docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp qpod/docker-kit \
+            image-syncer --proc=8 --retries=2 --images /tmp/task_sync_imgs/images.yaml --auth /tmp/.github/workflows/auth.json

.gitignore

@@ -0,0 +1,67 @@
+# Byte-compiled / optimized / DLL files
+
+*~
+
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# IDE
+.vscode/
+.idea/
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Mac OS X
+.DS_Store
+
+dockerspawner
+dockerspawner.tar.gz
+*.orig
+.ipynb_checkpoints/
+.vscode/
+.pytest_cache/

README.md

@@ -0,0 +1 @@
+# QPod Lab Container Kit

task_sync_imgs/README.md

@@ -0,0 +1,32 @@
+# Task Sync Images
+
+This task sync docker images from source registries to target registries based on the configuration files.
+
+```shell
+docker run -it --rm -v $(pwd):/root/app -w /root/app docker.io/qpod/docker-kit
+
+image-syncer --proc=8 --retries=2 --images ./images.yaml --auth ./auth.json
+```
+
+To sync images in batch, two config files (or combine them in one as `--config`) are needed.
+
+The `auth.yaml` file should look like:
+
+```yaml
+docker.io:
+  username: ""
+  password: ""
+  insecure: true
+registry.cn-hangzhou.aliyuncs.com:
+  username: ""
+  password: ""
+  insecure: true
+```
+
+The `images.yaml` file should look like:
+
+```yaml
+quay.io/qpod/docker-kit:
+  - docker.io/qpod/docker-kit
+  - registry.cn-hangzhou.aliyuncs.com/qpod/docker-kit
+```

task_sync_imgs/images.yaml

@@ -0,0 +1,57 @@
+# Ref: https://github.com/AliyunContainerService/image-syncer/blob/master/README-zh_CN.md
+
+# Elasticserach
+#  - ref1: https://github.com/elastic/elasticsearch/releases
+#  - ref2: https://www.elastic.co/docs/deploy-manage/deploy/self-managed/install-elasticsearch-with-docker
+docker.elastic.co/elasticsearch/elasticsearch:7.17.28,8.17.5,8.18.0,9.0.0:
+ - quay.io/qpod/elasticsearch
+ - registry.cn-hangzhou.aliyuncs.com/qpod/elasticsearch
+ - registry.cn-beijing.aliyuncs.com/qpod/elasticsearch
+
+
+# Minio
+# - ref1: https://github.com/minio/minio/releases
+# - ref2: https://min.io/docs/minio/container/index.html
+docker.io/minio/minio:latest,RELEASE.2023-12-20T01-00-02Z,RELEASE.2025-04-08T15-41-24Z:
+ - quay.io/qpod/minio
+ - registry.cn-hangzhou.aliyuncs.com/qpod/minio
+ - registry.cn-beijing.aliyuncs.com/qpod/minio
+
+
+# valkey
+# - ref1: https://github.com/valkey-io/valkey/releases
+# - ref2: https://hub.docker.com/r/valkey/valkey
+docker.io/valkey/valkey:latest,8,8.1,8.1.0:
+ - quay.io/qpod/valkey
+ - registry.cn-hangzhou.aliyuncs.com/qpod/valkey
+ - registry.cn-beijing.aliyuncs.com/qpod/valkey
+
+
+# k3s
+# - ref1: https://github.com/k3s-io/k3s/releases
+# - ref2: https://docs.k3s.io/installation/airgap
+rancher/k3s:latest,v1.32.3-k3s1:
+ - quay.io/qpod/k3s
+ - registry.cn-hangzhou.aliyuncs.com/qpod/k3s
+ - registry.cn-beijing.aliyuncs.com/qpod/k3s
+
+
+# k3d: https://github.com/k3d-io/k3d/pkgs/container/k3d
+ghcr.io/k3d-io/k3d:latest,5-dind,5:
+ - quay.io/qpod/k3s
+ - registry.cn-hangzhou.aliyuncs.com/qpod/k3d
+ - registry.cn-beijing.aliyuncs.com/qpod/k3d
+
+
+# k3d-proxy:https://github.com/k3d-io/k3d/pkgs/container/k3d-proxy
+ghcr.io/k3d-io/k3d-proxy:latest,5:
+ - quay.io/qpod/k3s
+ - registry.cn-hangzhou.aliyuncs.com/qpod/k3d-proxy
+ - registry.cn-beijing.aliyuncs.com/qpod/k3d-proxy
+
+
+# k3d-tools: https://github.com/k3d-io/k3d/pkgs/container/k3d-tools
+ghcr.io/k3d-io/k3d-tools:latest,5:
+ - quay.io/qpod/k3s
+ - registry.cn-hangzhou.aliyuncs.com/qpod/k3d-tools
+ - registry.cn-beijing.aliyuncs.com/qpod/k3d-tools

tool.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+set -xu
+
+CI_PROJECT_NAME=${GITHUB_REPOSITORY:-"QPod/lab-foundation"}
+CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"}
+CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/')
+
+if [ "${CI_PROJECT_BRANCH}" = "main" ] ; then
+    # If on the main branch, docker images namespace will be same as CI_PROJECT_NAME's name space
+    export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})" ;
+else
+    # not main branch, docker namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE}
+    export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})0${CI_PROJECT_SPACE}" ;
+fi
+
+export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}')
+export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}')
+export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}')
+export TAG_SUFFIX="-$(git rev-parse --short HEAD)"
+
+echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}"
+echo "--------> DOCKER_IMG_NAMESPACE=${IMG_NAMESPACE}"
+echo "--------> DOCKER_IMG_PREFIX_SRC=${IMG_PREFIX_SRC}"
+echo "--------> DOCKER_IMG_PREFIX_DST=${IMG_PREFIX_DST}"
+echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}"
+
+[ ! -f /etc/docker/daemon.json ] && sudo tee /etc/docker/daemon.json > /dev/null <<< '{}'
+jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/
+( sudo service docker restart || true ) && cat /etc/docker/daemon.json && docker info
+
+build_image() {
+    echo "$@" ;
+    IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)";
+    docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ;
+    docker tag "${IMG_PREFIX_DST}/${IMG}:${TAG}" "${IMG_PREFIX_DST}/${IMG}:${VER}" ;
+}
+
+build_image_no_tag() {
+    echo "$@" ;
+    IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)";
+    docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ;
+}
+
+build_image_common() {
+    echo "$@" ;
+    IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)";
+    docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ;
+    docker tag "${IMG_PREFIX_DST}/${IMG}:${TAG}" "${IMG_PREFIX_DST}/${IMG}:${VER}" ;
+}
+
+alias_image() {
+    IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX};
+    docker tag "${IMG_PREFIX_DST}/${IMG_1}:${TAG_1}" "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" ;
+    docker tag "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" "${IMG_PREFIX_DST}/${IMG_2}:${VER}" ;
+}
+
+push_image() {
+    KEYWORD="${1:-second}";
+    docker image prune --force && docker images | sort;
+    IMAGES=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ;
+    echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_DST}" -u "$DOCKER_REGISTRY_USERNAME" --password-stdin ;
+    for IMG in $(echo "${IMAGES}" | tr " " "\n") ;
+    do
+      docker push "${IMG}";
+      status=$?;
+      echo "[${status}] Image pushed > ${IMG}";
+    done
+}
+
+clear_images() {
+    KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images build days ago
+    IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ;
+    IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ;
+
+    for IMG in $(echo "$IMGS_1 $IMGS_2" | tr " " "\n") ; do
+      docker rmi "${IMG}" || true; status=$?; echo "[${status}] image removed > ${IMG}";
+    done
+    docker image prune --force && docker images ;
+}
+
+
+remove_folder() {
+    sudo du -h -d1 "$1" || true ;
+    sudo rm -rf "$1" || true ;
+}
+
+free_diskspace() {
+    remove_folder /usr/share/dotnet
+    remove_folder /usr/local/lib/android
+    # remove_folder /var/lib/docker
+    df -h
+}