update ci and sync

.github/workflows/build-docker.yml

@@ -15,13 +15,12 @@ on:
   workflow_dispatch:
 
 env:
-  REGISTRY_URL: "docker.io"   # docker.io or other registry URL, DOCKER_REGISTRY_USERNAME/DOCKER_REGISTRY_PASSWORD to be set in CI env.
   BUILDKIT_PROGRESS: "plain"  # Full logs for CI build.
-
+  REGISTRY_URL: "docker.io"  # docker.io or other target registry URL: where to push images to.
+  REGISTRY_SRC: "docker.io"  # For BASE_NAMESPACE of images: where to pull base images from.
   # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets.
   DOCKER_REGISTRY_USERNAME: ${{ secrets.DOCKER_REGISTRY_USERNAME }}
   DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
-
   # used to sync image to mirror registry
   DOCKER_MIRROR_REGISTRY_USERNAME: ${{ secrets.DOCKER_MIRROR_REGISTRY_USERNAME }}
   DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }}
@@ -379,7 +378,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: |
-          source ./tool.sh && build_image docker-kit latest docker_docker_kit/Dockerfile && push_image
+          source ./tool.sh && build_image docker-kit latest docker_docker_kit/Dockerfile
+          printenv > /tmp/docker.env
+          docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp \
+            ${IMG_PREFIX}/docker-kit /opt/conda/bin/python /opt/utils/image-syncer/run_sync.py ${IMG_NAMESPACE}/base
+          push_image docker-kit
 
   ## Sync all images in this build (listed by "names") to mirror registry.
   sync_images:
@@ -390,8 +393,5 @@ jobs:
       - run: |
           source ./tool.sh
           printenv > /tmp/docker.env
-          docker run --rm \
-            --env-file /tmp/docker.env \
-            -v $(pwd):/tmp \
-            -w /tmp \
-            ${IMG_NAMESPACE:-qpod}/docker-kit python /opt/utils/image-syncer/run_jobs.py
+          docker run --rm --env-file /tmp/docker.env -v $(pwd):/tmp -w /tmp \
+            ${IMG_NAMESPACE:-qpod}/docker-kit /opt/conda/bin/python /opt/utils/image-syncer/run_jobs.py

docker_docker_kit/Dockerfile

@@ -15,3 +15,11 @@ RUN source /opt/utils/script-setup-docker.sh \
  && ln -sf /usr/bin/image-syncer /opt/utils/image-syncer/ \
  # Clean up and display components version information...
  && source /opt/utils/script-utils.sh && install__clean && list_installed_packages
+
+# '-c' option make bash commands are read from string.
+#   If there are arguments after the string, they are assigned to the positional parameters, starting with $0.
+# '-o pipefail'  prevents errors in a pipeline from being masked.
+#   If any command in a pipeline fails, that return code will be used as the return code of the whole pipeline.
+# '--login': make bash first reads and executes commands from  the file /etc/profile, if that file exists.
+#   After that, it looks for ~/.bash_profile, ~/.bash_login, and ~/.profile, in that order, and reads and executes commands from the first one that exists and is readable.
+SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"]

docker_docker_kit/work/image-syncer/run_sync.py

@@ -65,7 +65,7 @@ def main():
     parser.add_argument('--target-registry', type=str, action='extend', nargs='*', help='Target registry URL')
     args = parser.parse_args()
 
-    configs = generate(image=args.img, tags=args.tags, source_registry=args.target_registry, target_registries=args.dest_registry)
+    configs = generate(image=args.img, tags=args.tags, source_registry=args.source_registry, target_registries=args.target_registry)
     ret = 0
     for _, c in enumerate(configs):
         ret += sync_image(cfg=c)

tool.sh

@@ -15,11 +15,12 @@ fi
 
 export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}')
 export IMG_PREFIX=$(echo "${REGISTRY_URL:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}')
+export TAG_SUFFIX="-$(git rev-parse --short HEAD)"
 
 echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}"
 echo "--------> DOCKER_IMG_NAMESPACE=${IMG_NAMESPACE}"
 echo "--------> DOCKER_IMG_PREFIX=${IMG_PREFIX}"
-
+echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}"
 
 if [ -f /etc/docker/daemon.json ]; then
        jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ \
@@ -30,7 +31,7 @@ docker info
 
 build_image() {
     echo "$@" ;
-    IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)";
+    IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)";
     docker build --compress --force-rm=true -t "${IMG_PREFIX}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX}" "$@" "${WORKDIR}" ;
     docker tag "${IMG_PREFIX}/${IMG}:${TAG}" "${IMG_PREFIX}/${IMG}:${VER}" ;
 }
@@ -43,13 +44,13 @@ build_image_no_tag() {
 
 build_image_common() {
     echo "$@" ;
-    IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)";
+    IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)";
     docker build --compress --force-rm=true -t "${IMG_PREFIX}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX}" "$@" "${WORKDIR}" ;
     docker tag "${IMG_PREFIX}/${IMG}:${TAG}" "${IMG_PREFIX}/${IMG}:${VER}" ;
 }
 
 alias_image() {
-    IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M);
+    IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX};
     docker tag "${IMG_PREFIX}/${IMG_1}:${TAG_1}" "${IMG_PREFIX}/${IMG_2}:${TAG_2}" ;
     docker tag "${IMG_PREFIX}/${IMG_2}:${TAG_2}" "${IMG_PREFIX}/${IMG_2}:${VER}" ;
 }