Sync Upstream/master with ODH/master

.github/actions/minikube-setup/action.yml

@@ -1,6 +1,20 @@
 name: 'Minikube setup action'
 description: 'Sets up minikube on the github runner'
 
+inputs:
+  nodes:
+    description: 'Number of nodes to start minikube with'
+    required: false
+    default: '1'
+  driver:
+    description: 'Driver to use for minikube'
+    required: false
+    default: 'none'
+  start-args:
+    description: 'Additional arguments to pass to minikube start'
+    required: false
+    default: ''
+
 runs:
   using: "composite"
   steps:
@@ -14,9 +28,11 @@ runs:
       with:
         minikube-version: '1.33.1'
         kubernetes-version: 'v1.29.7'
-        driver: 'none'
+        driver: ${{ inputs.driver }}
         wait: 'all'
-        start-args: --wait-timeout=6m0s
+        cpus: 'max'
+        memory: 'max'
+        start-args: --wait-timeout=6m0s --nodes=${{ inputs.nodes }} ${{ inputs.start-args }}
 
     - name: Check Kubernetes pods
       shell: bash

.github/workflows/e2e-test.yml

@@ -24,6 +24,7 @@ env:
   # Controller images
   CONTROLLER_IMG: "kserve-controller"
   LOCALMODEL_CONTROLLER_IMG: "kserve-localmodel-controller"
+  LOCALMODEL_AGENT_IMG: "kserve-localmodelnode-agent"
   STORAGE_INIT_IMG: "storage-initializer"
   AGENT_IMG: "agent"
   ROUTER_IMG: "router"
@@ -87,6 +88,14 @@ jobs:
           compression-level: 0
           if-no-files-found: error
 
+      - name: Upload localmodel agent image
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.BASE_ARTIFACT_PREFIX }}-${{ env.LOCALMODEL_AGENT_IMG }}-${{ github.sha }}
+          path: ${{ env.DOCKER_IMAGES_PATH }}/${{ env.LOCALMODEL_AGENT_IMG }}-${{ github.sha }}
+          compression-level: 0
+          if-no-files-found: error
+
       - name: Upload agent image
         uses: actions/upload-artifact@v4
         with:
@@ -900,7 +909,7 @@ jobs:
   test-llm:
     runs-on: ubuntu-22.04
     needs:
-      [ kserve-image-build, predictor-runtime-build]
+      [kserve-image-build, predictor-runtime-build]
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
@@ -960,7 +969,7 @@ jobs:
   test-huggingface-server-vllm:
     runs-on: ubuntu-22.04
     needs:
-      [ kserve-image-build, predictor-runtime-build]
+      [kserve-image-build, predictor-runtime-build]
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
@@ -1015,4 +1024,129 @@ jobs:
       - name: Check system status
         if: always()
         run: |
+          ./test/scripts/gh-actions/status-check.sh
+
+  test-modelcache:
+    runs-on: ubuntu-22.04
+    needs:
+      [kserve-image-build, predictor-runtime-build]
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Free-up disk space
+        uses: ./.github/actions/free-up-disk-space
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+
+      - name: Create tmp-images directory
+        run: |
+          sudo mkdir -p /tmp-images
+          sudo chown -R $USER /tmp-images
+
+      - name: Setup Minikube
+        uses: ./.github/actions/minikube-setup
+        with:
+          nodes: 3
+          driver: "docker"
+          start-args: "--mount --mount-string=/tmp-images:/tmp-images"
+
+      - name: Create minikube tunnel
+        run: |
+          nohup minikube tunnel > minikube-tunnel.log 2>&1 &
+
+      - name: KServe dependency setup
+        uses: ./.github/actions/kserve-dep-setup
+
+      - name: Download base images
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp-images
+          pattern: ${{ env.BASE_ARTIFACT_PREFIX }}-*
+          merge-multiple: true
+
+      - name: Load base docker images
+        run: |
+          ls -l /tmp-images
+          minikube ssh -n minikube -- ls -l /tmp-images
+          files=$(find /tmp-images -maxdepth 1 -type f)
+          for file in ${files[@]};do
+            echo "Loading image $(basename ${file})"
+            minikube ssh -n minikube -- docker image load -i ${file}
+            minikube ssh -n minikube-m02 -- docker image load -i ${file}
+            minikube ssh -n minikube-m03 -- docker image load -i ${file}
+          done
+          sudo rm -rf /tmp-images/*
+          minikube ssh -n minikube -- docker image ls
+          minikube ssh -n minikube-m02 -- docker image ls
+          minikube ssh -n minikube-m03 -- docker image ls
+
+      - name: Download huggingface server image
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.HUGGINGFACE_IMG }}-${{ github.sha }}
+          path: /tmp-images
+
+      - name: Load runtime docker images
+        run: |
+          files=$(find /tmp-images -maxdepth 1 -type f)
+          for file in ${files[@]};do
+            echo "Loading image $(basename ${file})"
+            minikube ssh -n minikube-m02 -- docker image load -i ${file}
+          done
+          sudo rm -rf tmp-images/*
+          minikube ssh -n minikube-m02 -- docker image ls
+
+      - name: Create model root directory
+        run: |
+          minikube ssh -n minikube-m02 -- sudo mkdir -p -m=777 /models
+          minikube ssh -n minikube-m03 -- sudo mkdir -p -m=777 /models
+
+      - name: Install Poetry and version plugin
+        run: ./test/scripts/gh-actions/setup-poetry.sh
+
+      - name: Install KServe
+        run: |
+          ./test/scripts/gh-actions/setup-kserve.sh
+
+          kubectl get pods -n kserve
+          kubectl describe pods -n kserve
+
+      - name: Enable modelcache
+        run: |
+          sed -i -e "s/latest/${GITHUB_SHA}/g" config/overlays/test/configmap/inferenceservice-enable-modelcache.yaml
+          kubectl patch configmaps -n kserve inferenceservice-config --patch-file config/overlays/test/configmap/inferenceservice-enable-modelcache.yaml
+          kubectl describe configmaps -n kserve inferenceservice-config
+
+      - name: Create localmodel job namespace
+        run: |
+          kubectl create ns kserve-localmodel-jobs
+
+      - name: Label worker nodes for modelcache
+        run: |
+          kubectl label nodes -l '!node-role.kubernetes.io/control-plane' kserve/localmodel=worker
+
+      - name: Enable nodeselector in knative
+        run: |
+          kubectl patch configmaps -n knative-serving config-features --patch '{"data": {"kubernetes.podspec-nodeselector": "enabled"}}'
+
+      - name: Run E2E tests
+        timeout-minutes: 15
+        run: |
+          ./test/scripts/gh-actions/run-e2e-tests.sh "modelcache" "1"
+
+      - name: Check system status
+        if: always()
+        run: |
+          echo "::group::Minikube tunnel logs"
+          cat minikube-tunnel.log
+          echo "::endgroup::"
           ./test/scripts/gh-actions/status-check.sh

.github/workflows/golangci-lint.yml

@@ -18,11 +18,47 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: "1.22"
           cache: false
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v4
         with:
           version: v1.56
           args: --out-format=line-number
 
+  verify-go-mod:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.22
+          cache: false
+
+      - name: Run go mod tidy
+        run: |
+          go mod tidy
+
+      - name: Verify
+        shell: bash
+        run: |
+          # From: https://backreference.org/2009/12/23/how-to-match-newlines-in-sed/
+          # This is to leverage this workaround:
+          # https://github.com/actions/toolkit/issues/193#issuecomment-605394935
+          urlencode() {
+            sed ':begin;$!N;s/\n/%0A/;tbegin'
+          }
+
+          if [ -z "$(git status --porcelain)" ]; then
+              echo "${{ github.repository }} up to date."
+          else
+              echo "Found diffs in: $(git diff-index --name-only HEAD)"
+              for x in $(git diff-index --name-only HEAD); do
+                  echo "::error file=$x::Please run 'go mod tidy'.%0A$(git diff $x | urlencode)"
+              done
+              echo "${{ github.repository }} is out of date. Please run 'go mod tidy'"
+              exit 1
+          fi

.github/workflows/kserve-localmodel-agent-docker-publish.yml

@@ -0,0 +1,95 @@
+name: Kserve localmodel agent Docker Publisher
+
+on:
+  push:
+    # Publish `master` as Docker `latest` image.
+    branches:
+      - master
+
+    # Publish `v1.2.3` tags as releases.
+    tags:
+      - v*
+
+  # Run tests for any PRs.
+  pull_request:
+
+env:
+  IMAGE_NAME: kserve-localmodelnode-agent
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Run tests.
+  # See also https://docs.docker.com/docker-hub/builds/automated-testing/
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Run tests
+        run: |
+          if [ -f docker-compose.test.yml ]; then
+            docker-compose --file docker-compose.test.yml build
+            docker-compose --file docker-compose.test.yml run sut
+          else
+            docker buildx build . --file localmodel-agent.Dockerfile
+          fi
+
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    # Ensure test job passes before pushing image.
+    needs: test
+
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: export version variable
+        run: |
+          IMAGE_ID=kserve/$IMAGE_NAME
+
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # Strip "v" prefix from tag name
+          # [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "master" ] && VERSION=latest
+
+          echo VERSION=$VERSION >> $GITHUB_ENV
+          echo IMAGE_ID=$IMAGE_ID >> $GITHUB_ENV
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/amd64,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x
+          context: .
+          file: localmodel-agent.Dockerfile
+          push: true
+          tags: ${{ env.IMAGE_ID }}:${{ env.VERSION }}
+          # https://github.com/docker/buildx/issues/1533
+          provenance: false

.github/workflows/scheduled-image-scan.yml

@@ -19,12 +19,10 @@ jobs:
           [
             { name: kserve-controller, file: Dockerfile },
             { name: agent, file: agent.Dockerfile },
-            {
-              name: storage-initializer,
-              file: python/storage-initializer.Dockerfile,
-            },
+            { name: storage-initializer, file: python/storage-initializer.Dockerfile },
             { name: router, file: router.Dockerfile },
             { name: kserve-localmodel-controller, file: localmodel.Dockerfile },
+            { name: kserve-localmodelnode-agent, file: localmodel-agent.Dockerfile },
           ]
 
     steps:
@@ -112,10 +110,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        image:
-          [
-            { name: art-explainer, file: python/artexplainer.Dockerfile },
-          ]
+        image: [{ name: art-explainer, file: python/artexplainer.Dockerfile }]
 
     steps:
       - name: Checkout