Merge pull request #144 from Jooho/odh_0.11.1_pr3250

[Cherry Pick - 3250] feat: CA bundle mount options for storage initializer

config/configmap/inferenceservice.yaml

@@ -10,7 +10,7 @@ data:
     #    EXAMPLE CONFIGURATION     #
     #                              #
     ################################
-    
+
     # This block is not actually functional configuration,
     # but serves to illustrate the available configuration
     # options and document them in a way that is accessible
@@ -19,7 +19,7 @@ data:
     # These sample configuration options may be copied out of
     # this example block and unindented to be in the data block
     # to actually change the configuration.
-    
+
     # ====================================== EXPLAINERS CONFIGURATION ======================================
     # Example
     explainers: |-
@@ -63,7 +63,9 @@ data:
            "memoryLimit": "1Gi",
            "cpuRequest": "100m",
            "cpuLimit": "1",
-           "enableDirectPvcVolumeMount": false
+           "caBundleConfigMapName": "",
+           "caBundleVolumeMountPath": "/etc/ssl/custom-certs",
+           "enableDirectPvcVolumeMount": false,
        }
      storageInitializer: |-
        {
@@ -82,6 +84,12 @@ data:
            # cpuLimit is the limits.cpu to set for the storage initializer init container.
            "cpuLimit": "1",
        
+           # caBundleConfigMapName is the ConfigMap will be copied to a user namespace for the storage initializer init container.
+           "caBundleConfigMapName": "",
+
+           # caBundleVolumeMountPath is the mount point for the configmap set by caBundleConfigMapName for the storage initializer init container.
+           "caBundleVolumeMountPath": "/etc/ssl/custom-certs",
+
            # enableDirectPvcVolumeMount controls whether users can mount pvc volumes directly.
            # if pvc volume is provided in storageuri then the pvc volume is directly mounted to /mnt/models in the user container.
            # rather than symlink it to a shared volume. For more info see https://github.com/kserve/kserve/issues/2737
@@ -417,6 +425,8 @@ data:
         "memoryLimit": "1Gi",
         "cpuRequest": "100m",
         "cpuLimit": "1",
+        "caBundleConfigMapName": "",
+        "caBundleVolumeMountPath": "/etc/ssl/custom-certs",
         "enableDirectPvcVolumeMount": false
     }
 

config/overlays/odh/params.env

@@ -1,4 +1,4 @@
-kserve-controller=quay.io/opendatahub/kserve-controller:v0.11.1.0
-kserve-agent=quay.io/opendatahub/kserve-agent:v0.11.1.0
-kserve-router=quay.io/opendatahub/kserve-router:v0.11.1.0
-kserve-storage-initializer=quay.io/opendatahub/kserve-storage-initializer:v0.11.1.0
+kserve-controller=quay.io/opendatahub/kserve-controller:v0.11.1-latest
+kserve-agent=quay.io/opendatahub/kserve-agent:v0.11.1-latest
+kserve-router=quay.io/opendatahub/kserve-router:v0.11.1-latest
+kserve-storage-initializer=quay.io/opendatahub/kserve-storage-initializer:v0.11.1-latest

docs/samples/cabundle/README.md

@@ -0,0 +1,139 @@
+# KServe with Self Signed Certificate Model Registry
+
+If you are using a model registry with a self-signed certificate, you must either skip ssl verify or apply the appropriate CA bundle to the storage-initializer to create a connection with the registry.
+This document explains three methods that can be used in KServe, described below:
+
+- Configure CA bundle for storage-initializer
+  - Global configuration
+  - Using `storage-config` Secret
+
+- Skip SSL Verification
+
+## Configure CaBundle for storage-initializer  
+### Global Configuration
+
+KServe use `inferenceservice-config` ConfigMap for default configuration. If you want to add `cabundle` cert for every inference service, you can set `caBundleConfigMapName` in the ConfigMap. Before updating the ConfigMap, you have to create a ConfigMap for CA bundle certificate in the namespace that KServe controller is running and the data key in the ConfigMap must be `cabundle.crt`. 
+
+- Create a ConfigMap with the CA bundle cert
+  ~~~
+  kubectl create configmap cabundle --from-file=/path/to/cabundle.crt
+
+  kubectl get configmap cabundle -o yaml
+  apiVersion: v1
+  data:
+    cabundle.crt: XXXXX
+  kind: ConfigMap
+  metadata:
+    name: cabundle
+    namespace: kserve
+  ~~~
+- Update `inferenceservice-config` ConfigMap 
+  ~~~
+    storageInitializer: |-
+    {
+        ...
+        "caBundleConfigMapName": "cabundle",
+        ...
+    }
+  ~~~
+
+If you update this configuration after, please restart KServe controller pod.  
+
+### Using storage-config Secret
+
+If you want to apply the cabundle only to a specific inferenceservice, you can use a specific annotation or variable(`cabundle_configmap`) on the `storage-config` Secret used by the inferenceservice.
+In this case, you have to create the cabundle ConfigMap in the user namespace before you create the inferenceservice.
+
+
+- Create a ConfigMap with the cabundle cert
+  ~~~
+  kubectl create configmap local-cabundle --from-file=/path/to/cabundle.crt
+
+  kubectl get configmap cabundle -o yaml
+  apiVersion: v1
+  data:
+    cabundle.crt: XXXXX
+  kind: ConfigMap
+  metadata:
+    name: local-cabundle
+    namespace: kserve-demo
+  ~~~
+
+- Add an annotation `serving.kserve.io/s3-cabundle-configmap` to `storage-config` Secret
+  ~~~
+  apiVersion: v1
+  data:
+    AWS_ACCESS_KEY_ID: VEhFQUNDRVNTS0VZ
+    AWS_SECRET_ACCESS_KEY: VEhFUEFTU1dPUkQ=
+  kind: Secret
+  metadata:
+    annotations:
+      serving.kserve.io/s3-cabundle-configmap: local-cabundle
+      ...
+    name: storage-config
+    namespace: kserve-demo
+  type: Opaque
+  ~~~
+
+- Or, set a variable `cabundle_configmap` to `storage-config` Secret
+  ~~~
+  apiVersion: v1
+  stringData:
+    localMinIO: |
+    {
+      "type": "s3",
+      "access_key_id": "THEACCESSKEY",
+      "secret_access_key": "THEPASSWORD",
+      "endpoint_url": "https://minio.minio.svc:9000",
+      "bucket": "modelmesh-example-models",
+      "region": "us-south"
+      "cabundle_configmap": "local-cabundle"
+    }
+  kind: Secret
+  metadata:
+    name: storage-config
+    namespace: kserve-demo
+  type: Opaque
+  ~~~
+
+## Skip SSL Verification
+
+For testing purposes or when there is no cabundle, you can easily create an SSL connection by disabling SSL verification.
+This can also be used by adding an annotation or setting a variable in `secret-config` Secret.
+
+- Add an annotation(`serving.kserve.io/s3-verifyssl`) to `storage-config` Secret
+  ~~~
+  apiVersion: v1
+  data:
+    AWS_ACCESS_KEY_ID: VEhFQUNDRVNTS0VZ
+    AWS_SECRET_ACCESS_KEY: VEhFUEFTU1dPUkQ=
+  kind: Secret
+  metadata:
+    annotations:
+         serving.kserve.io/s3-verifyssl: "0" # 1 is true, 0 is false
+      ...
+    name: storage-config
+    namespace: kserve-demo
+  type: Opaque
+  ~~~
+
+- Or, set a variable (`verify_ssl`) to `storage-config` Secret
+  ~~~
+  apiVersion: v1
+  stringData:
+    localMinIO: |
+      {
+        "type": "s3",
+        "access_key_id": "THEACCESSKEY",
+        "secret_access_key": "THEPASSWORD",
+        "endpoint_url": "https://minio.minio.svc:9000",
+        "bucket": "modelmesh-example-models",
+        "region": "us-south",
+        "verify_ssl": "0"  # 1 is true, 0 is false  (You can set True/true/False/false too)
+      }
+  kind: Secret
+  metadata:
+    name: storage-config
+    namespace: kserve-demo
+  type: Opaque
+  ~~~

pkg/constants/constants.go

@@ -301,6 +301,21 @@ const (
 // DefaultModelLocalMountPath is where models will be mounted by the storage-initializer
 const DefaultModelLocalMountPath = "/mnt/models"
 
+// Default path to mount CA bundle configmap volume
+const DefaultCaBundleVolumeMountPath = "/etc/ssl/custom-certs"
+
+// Default name for CA bundle file
+const DefaultCaBundleFileName = "cabundle.crt"
+
+// Default CA bundle configmap name that will be created in the user namespace.
+const DefaultGlobalCaBundleConfigMapName = "global-ca-bundle"
+
+// Custom CA bundle configmap Environment Variables
+const (
+	CaBundleConfigMapNameEnvVarKey   = "CA_BUNDLE_CONFIGMAP_NAME"
+	CaBundleVolumeMountPathEnvVarKey = "CA_BUNDLE_VOLUME_MOUNT_POINT"
+)
+
 // Multi-model InferenceService
 const (
 	ModelConfigVolumeName = "model-config"

pkg/controller/v1beta1/inferenceservice/controller.go

@@ -28,6 +28,7 @@ import (
 	v1beta1api "github.com/kserve/kserve/pkg/apis/serving/v1beta1"
 	"github.com/kserve/kserve/pkg/constants"
 	"github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/components"
+	"github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/cabundleconfigmap"
 	"github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress"
 	modelconfig "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/modelconfig"
 	isvcutils "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/utils"
@@ -168,6 +169,13 @@ func (r *InferenceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	if err != nil {
 		return reconcile.Result{}, errors.Wrapf(err, "fails to create InferenceServicesConfig")
 	}
+
+	// Reconcile cabundleConfigMap
+	caBundleConfigMapReconciler := cabundleconfigmap.NewCaBundleConfigMapReconciler(r.Client, r.Scheme)
+	if err := caBundleConfigMapReconciler.Reconcile(isvc); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	reconcilers := []components.Component{}
 	if deploymentMode != constants.ModelMeshDeployment {
 		reconcilers = append(reconcilers, components.NewPredictor(r.Client, r.Scheme, isvcConfig, deploymentMode))