diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2c9f817 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,25 @@ +# Ignore all differences in line endings +* text=auto eol=lf +*.md text eol=lf +*.py text eol=lf +*.sh text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.json text eol=lf +*.properties text eol=lf +*.conf text eol=lf +*.ipynb text eol=lf +Dockerfile* text eol=lf +.gitattributes text eol=lf +.gitignore text eol=lf +.dockerignore text eol=lf + +# Files using LFS to track +*.tgz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.jsonl filter=lfs diff=lfs merge=lfs -text +*.xlsx filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 0000000..494a23c --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,43 @@ +name: build-docker-images + +on: + push: + branches: [ "main" ] + paths-ignore: [ "*.md" ] + + pull_request: + branches: [ "main" ] + paths-ignore: [ "*.md" ] + + workflow_dispatch: # Allows you to run this workflow manually from the Actions tab + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + BUILDKIT_PROGRESS: "plain" # Full logs for CI build. + REGISTRY_SRC: ${{ vars.REGISTRY_SRC || 'docker.io' }} # For BASE_NAMESPACE of images: where to pull base images from, docker.io or other source registry URL. + REGISTRY_DST: ${{ vars.REGISTRY_DST || 'docker.io' }} # For tags of built images: where to push images to, docker.io or other destination registry URL. + # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. + DOCKER_REGISTRY_USERNAME: ${{ vars.DOCKER_REGISTRY_USERNAME }} + DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + # used to sync image to mirror registry + DOCKER_MIRROR_REGISTRY_USERNAME: ${{ vars.DOCKER_MIRROR_REGISTRY_USERNAME }} + DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} + +jobs: + ## Sync all images in this build (listed by "names") to mirror registry. + sync_images: + needs: [] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - env: + AUTH_FILE_CONTENT: ${{ secrets.AUTH_FILE_CONTENT }} + run: | + source ./tool.sh && ls -alh ./ + printf '%s' "$AUTH_FILE_CONTENT" > .github/workflows/auth.json && ls -alh ./.github/workflows + printenv | grep -v 'PATH' > /tmp/docker.env && echo "REGISTRY_URL=${REGISTRY_DST}" >> /tmp/docker.env + docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp qpod/docker-kit \ + image-syncer --proc=8 --retries=2 --images /tmp/task_sync_imgs/images.yaml --auth /tmp/.github/workflows/auth.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9adc602 --- /dev/null +++ b/.gitignore @@ -0,0 +1,67 @@ +# Byte-compiled / optimized / DLL files + +*~ + +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# IDE +.vscode/ +.idea/ + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Mac OS X +.DS_Store + +dockerspawner +dockerspawner.tar.gz +*.orig +.ipynb_checkpoints/ +.vscode/ +.pytest_cache/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..bcb2a6c --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# QPod Lab Container Kit diff --git a/task_sync_imgs/README.md b/task_sync_imgs/README.md new file mode 100644 index 0000000..419dec7 --- /dev/null +++ b/task_sync_imgs/README.md @@ -0,0 +1,32 @@ +# Task Sync Images + +This task sync docker images from source registries to target registries based on the configuration files. + +```shell +docker run -it --rm -v $(pwd):/root/app -w /root/app docker.io/qpod/docker-kit + +image-syncer --proc=8 --retries=2 --images ./images.yaml --auth ./auth.json +``` + +To sync images in batch, two config files (or combine them in one as `--config`) are needed. + +The `auth.yaml` file should look like: + +```yaml +docker.io: + username: "" + password: "" + insecure: true +registry.cn-hangzhou.aliyuncs.com: + username: "" + password: "" + insecure: true +``` + +The `images.yaml` file should look like: + +```yaml +quay.io/qpod/docker-kit: + - docker.io/qpod/docker-kit + - registry.cn-hangzhou.aliyuncs.com/qpod/docker-kit +``` diff --git a/task_sync_imgs/images.yaml b/task_sync_imgs/images.yaml new file mode 100644 index 0000000..b8b37e3 --- /dev/null +++ b/task_sync_imgs/images.yaml @@ -0,0 +1,57 @@ +# Ref: https://github.com/AliyunContainerService/image-syncer/blob/master/README-zh_CN.md + +# Elasticserach +# - ref1: https://github.com/elastic/elasticsearch/releases +# - ref2: https://www.elastic.co/docs/deploy-manage/deploy/self-managed/install-elasticsearch-with-docker +docker.elastic.co/elasticsearch/elasticsearch:7.17.28,8.17.5,8.18.0,9.0.0: + - quay.io/qpod/elasticsearch + - registry.cn-hangzhou.aliyuncs.com/qpod/elasticsearch + - registry.cn-beijing.aliyuncs.com/qpod/elasticsearch + + +# Minio +# - ref1: https://github.com/minio/minio/releases +# - ref2: https://min.io/docs/minio/container/index.html +docker.io/minio/minio:latest,RELEASE.2023-12-20T01-00-02Z,RELEASE.2025-04-08T15-41-24Z: + - quay.io/qpod/minio + - registry.cn-hangzhou.aliyuncs.com/qpod/minio + - registry.cn-beijing.aliyuncs.com/qpod/minio + + +# valkey +# - ref1: https://github.com/valkey-io/valkey/releases +# - ref2: https://hub.docker.com/r/valkey/valkey +docker.io/valkey/valkey:latest,8,8.1,8.1.0: + - quay.io/qpod/valkey + - registry.cn-hangzhou.aliyuncs.com/qpod/valkey + - registry.cn-beijing.aliyuncs.com/qpod/valkey + + +# k3s +# - ref1: https://github.com/k3s-io/k3s/releases +# - ref2: https://docs.k3s.io/installation/airgap +rancher/k3s:latest,v1.32.3-k3s1: + - quay.io/qpod/k3s + - registry.cn-hangzhou.aliyuncs.com/qpod/k3s + - registry.cn-beijing.aliyuncs.com/qpod/k3s + + +# k3d: https://github.com/k3d-io/k3d/pkgs/container/k3d +ghcr.io/k3d-io/k3d:latest,5-dind,5: + - quay.io/qpod/k3s + - registry.cn-hangzhou.aliyuncs.com/qpod/k3d + - registry.cn-beijing.aliyuncs.com/qpod/k3d + + +# k3d-proxy:https://github.com/k3d-io/k3d/pkgs/container/k3d-proxy +ghcr.io/k3d-io/k3d-proxy:latest,5: + - quay.io/qpod/k3s + - registry.cn-hangzhou.aliyuncs.com/qpod/k3d-proxy + - registry.cn-beijing.aliyuncs.com/qpod/k3d-proxy + + +# k3d-tools: https://github.com/k3d-io/k3d/pkgs/container/k3d-tools +ghcr.io/k3d-io/k3d-tools:latest,5: + - quay.io/qpod/k3s + - registry.cn-hangzhou.aliyuncs.com/qpod/k3d-tools + - registry.cn-beijing.aliyuncs.com/qpod/k3d-tools diff --git a/tool.sh b/tool.sh new file mode 100644 index 0000000..30030ac --- /dev/null +++ b/tool.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -xu + +CI_PROJECT_NAME=${GITHUB_REPOSITORY:-"QPod/lab-foundation"} +CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"} +CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/') + +if [ "${CI_PROJECT_BRANCH}" = "main" ] ; then + # If on the main branch, docker images namespace will be same as CI_PROJECT_NAME's name space + export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})" ; +else + # not main branch, docker namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE} + export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})0${CI_PROJECT_SPACE}" ; +fi + +export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') +export TAG_SUFFIX="-$(git rev-parse --short HEAD)" + +echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" +echo "--------> DOCKER_IMG_NAMESPACE=${IMG_NAMESPACE}" +echo "--------> DOCKER_IMG_PREFIX_SRC=${IMG_PREFIX_SRC}" +echo "--------> DOCKER_IMG_PREFIX_DST=${IMG_PREFIX_DST}" +echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}" + +[ ! -f /etc/docker/daemon.json ] && sudo tee /etc/docker/daemon.json > /dev/null <<< '{}' +jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ +( sudo service docker restart || true ) && cat /etc/docker/daemon.json && docker info + +build_image() { + echo "$@" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; + docker tag "${IMG_PREFIX_DST}/${IMG}:${TAG}" "${IMG_PREFIX_DST}/${IMG}:${VER}" ; +} + +build_image_no_tag() { + echo "$@" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; +} + +build_image_common() { + echo "$@" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; + docker tag "${IMG_PREFIX_DST}/${IMG}:${TAG}" "${IMG_PREFIX_DST}/${IMG}:${VER}" ; +} + +alias_image() { + IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; + docker tag "${IMG_PREFIX_DST}/${IMG_1}:${TAG_1}" "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" ; + docker tag "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" "${IMG_PREFIX_DST}/${IMG_2}:${VER}" ; +} + +push_image() { + KEYWORD="${1:-second}"; + docker image prune --force && docker images | sort; + IMAGES=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; + echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_DST}" -u "$DOCKER_REGISTRY_USERNAME" --password-stdin ; + for IMG in $(echo "${IMAGES}" | tr " " "\n") ; + do + docker push "${IMG}"; + status=$?; + echo "[${status}] Image pushed > ${IMG}"; + done +} + +clear_images() { + KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images build days ago + IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; + IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ; + + for IMG in $(echo "$IMGS_1 $IMGS_2" | tr " " "\n") ; do + docker rmi "${IMG}" || true; status=$?; echo "[${status}] image removed > ${IMG}"; + done + docker image prune --force && docker images ; +} + + +remove_folder() { + sudo du -h -d1 "$1" || true ; + sudo rm -rf "$1" || true ; +} + +free_diskspace() { + remove_folder /usr/share/dotnet + remove_folder /usr/local/lib/android + # remove_folder /var/lib/docker + df -h +}