diff --git a/config/configmap/inferenceservice.yaml b/config/configmap/inferenceservice.yaml index d58408a350c..c3c6b631e17 100644 --- a/config/configmap/inferenceservice.yaml +++ b/config/configmap/inferenceservice.yaml @@ -10,7 +10,7 @@ data: # EXAMPLE CONFIGURATION # # # ################################ - + # This block is not actually functional configuration, # but serves to illustrate the available configuration # options and document them in a way that is accessible @@ -19,7 +19,7 @@ data: # These sample configuration options may be copied out of # this example block and unindented to be in the data block # to actually change the configuration. - + # ====================================== EXPLAINERS CONFIGURATION ====================================== # Example explainers: |- @@ -63,7 +63,9 @@ data: "memoryLimit": "1Gi", "cpuRequest": "100m", "cpuLimit": "1", - "enableDirectPvcVolumeMount": false + "caBundleConfigMapName": "", + "caBundleVolumeMountPath": "/etc/ssl/custom-certs", + "enableDirectPvcVolumeMount": false, } storageInitializer: |- { @@ -82,6 +84,12 @@ data: # cpuLimit is the limits.cpu to set for the storage initializer init container. "cpuLimit": "1", + # caBundleConfigMapName is the ConfigMap will be copied to a user namespace for the storage initializer init container. + "caBundleConfigMapName": "", + + # caBundleVolumeMountPath is the mount point for the configmap set by caBundleConfigMapName for the storage initializer init container. + "caBundleVolumeMountPath": "/etc/ssl/custom-certs", + # enableDirectPvcVolumeMount controls whether users can mount pvc volumes directly. # if pvc volume is provided in storageuri then the pvc volume is directly mounted to /mnt/models in the user container. # rather than symlink it to a shared volume. For more info see https://github.com/kserve/kserve/issues/2737 @@ -417,6 +425,8 @@ data: "memoryLimit": "1Gi", "cpuRequest": "100m", "cpuLimit": "1", + "caBundleConfigMapName": "", + "caBundleVolumeMountPath": "/etc/ssl/custom-certs", "enableDirectPvcVolumeMount": false } diff --git a/config/overlays/odh/params.env b/config/overlays/odh/params.env index a263a322924..010046ffe84 100644 --- a/config/overlays/odh/params.env +++ b/config/overlays/odh/params.env @@ -1,4 +1,4 @@ -kserve-controller=quay.io/opendatahub/kserve-controller:v0.11.1.0 -kserve-agent=quay.io/opendatahub/kserve-agent:v0.11.1.0 -kserve-router=quay.io/opendatahub/kserve-router:v0.11.1.0 -kserve-storage-initializer=quay.io/opendatahub/kserve-storage-initializer:v0.11.1.0 +kserve-controller=quay.io/opendatahub/kserve-controller:v0.11.1-latest +kserve-agent=quay.io/opendatahub/kserve-agent:v0.11.1-latest +kserve-router=quay.io/opendatahub/kserve-router:v0.11.1-latest +kserve-storage-initializer=quay.io/opendatahub/kserve-storage-initializer:v0.11.1-latest diff --git a/docs/samples/cabundle/README.md b/docs/samples/cabundle/README.md new file mode 100644 index 00000000000..739376f7b12 --- /dev/null +++ b/docs/samples/cabundle/README.md @@ -0,0 +1,139 @@ +# KServe with Self Signed Certificate Model Registry + +If you are using a model registry with a self-signed certificate, you must either skip ssl verify or apply the appropriate CA bundle to the storage-initializer to create a connection with the registry. +This document explains three methods that can be used in KServe, described below: + +- Configure CA bundle for storage-initializer + - Global configuration + - Using `storage-config` Secret + +- Skip SSL Verification + +## Configure CaBundle for storage-initializer +### Global Configuration + +KServe use `inferenceservice-config` ConfigMap for default configuration. If you want to add `cabundle` cert for every inference service, you can set `caBundleConfigMapName` in the ConfigMap. Before updating the ConfigMap, you have to create a ConfigMap for CA bundle certificate in the namespace that KServe controller is running and the data key in the ConfigMap must be `cabundle.crt`. + +- Create a ConfigMap with the CA bundle cert + ~~~ + kubectl create configmap cabundle --from-file=/path/to/cabundle.crt + + kubectl get configmap cabundle -o yaml + apiVersion: v1 + data: + cabundle.crt: XXXXX + kind: ConfigMap + metadata: + name: cabundle + namespace: kserve + ~~~ +- Update `inferenceservice-config` ConfigMap + ~~~ + storageInitializer: |- + { + ... + "caBundleConfigMapName": "cabundle", + ... + } + ~~~ + +If you update this configuration after, please restart KServe controller pod. + +### Using storage-config Secret + +If you want to apply the cabundle only to a specific inferenceservice, you can use a specific annotation or variable(`cabundle_configmap`) on the `storage-config` Secret used by the inferenceservice. +In this case, you have to create the cabundle ConfigMap in the user namespace before you create the inferenceservice. + + +- Create a ConfigMap with the cabundle cert + ~~~ + kubectl create configmap local-cabundle --from-file=/path/to/cabundle.crt + + kubectl get configmap cabundle -o yaml + apiVersion: v1 + data: + cabundle.crt: XXXXX + kind: ConfigMap + metadata: + name: local-cabundle + namespace: kserve-demo + ~~~ + +- Add an annotation `serving.kserve.io/s3-cabundle-configmap` to `storage-config` Secret + ~~~ + apiVersion: v1 + data: + AWS_ACCESS_KEY_ID: VEhFQUNDRVNTS0VZ + AWS_SECRET_ACCESS_KEY: VEhFUEFTU1dPUkQ= + kind: Secret + metadata: + annotations: + serving.kserve.io/s3-cabundle-configmap: local-cabundle + ... + name: storage-config + namespace: kserve-demo + type: Opaque + ~~~ + +- Or, set a variable `cabundle_configmap` to `storage-config` Secret + ~~~ + apiVersion: v1 + stringData: + localMinIO: | + { + "type": "s3", + "access_key_id": "THEACCESSKEY", + "secret_access_key": "THEPASSWORD", + "endpoint_url": "https://minio.minio.svc:9000", + "bucket": "modelmesh-example-models", + "region": "us-south" + "cabundle_configmap": "local-cabundle" + } + kind: Secret + metadata: + name: storage-config + namespace: kserve-demo + type: Opaque + ~~~ + +## Skip SSL Verification + +For testing purposes or when there is no cabundle, you can easily create an SSL connection by disabling SSL verification. +This can also be used by adding an annotation or setting a variable in `secret-config` Secret. + +- Add an annotation(`serving.kserve.io/s3-verifyssl`) to `storage-config` Secret + ~~~ + apiVersion: v1 + data: + AWS_ACCESS_KEY_ID: VEhFQUNDRVNTS0VZ + AWS_SECRET_ACCESS_KEY: VEhFUEFTU1dPUkQ= + kind: Secret + metadata: + annotations: + serving.kserve.io/s3-verifyssl: "0" # 1 is true, 0 is false + ... + name: storage-config + namespace: kserve-demo + type: Opaque + ~~~ + +- Or, set a variable (`verify_ssl`) to `storage-config` Secret + ~~~ + apiVersion: v1 + stringData: + localMinIO: | + { + "type": "s3", + "access_key_id": "THEACCESSKEY", + "secret_access_key": "THEPASSWORD", + "endpoint_url": "https://minio.minio.svc:9000", + "bucket": "modelmesh-example-models", + "region": "us-south", + "verify_ssl": "0" # 1 is true, 0 is false (You can set True/true/False/false too) + } + kind: Secret + metadata: + name: storage-config + namespace: kserve-demo + type: Opaque + ~~~ diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 73e97d4cf3e..827532dc984 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -301,6 +301,21 @@ const ( // DefaultModelLocalMountPath is where models will be mounted by the storage-initializer const DefaultModelLocalMountPath = "/mnt/models" +// Default path to mount CA bundle configmap volume +const DefaultCaBundleVolumeMountPath = "/etc/ssl/custom-certs" + +// Default name for CA bundle file +const DefaultCaBundleFileName = "cabundle.crt" + +// Default CA bundle configmap name that will be created in the user namespace. +const DefaultGlobalCaBundleConfigMapName = "global-ca-bundle" + +// Custom CA bundle configmap Environment Variables +const ( + CaBundleConfigMapNameEnvVarKey = "CA_BUNDLE_CONFIGMAP_NAME" + CaBundleVolumeMountPathEnvVarKey = "CA_BUNDLE_VOLUME_MOUNT_POINT" +) + // Multi-model InferenceService const ( ModelConfigVolumeName = "model-config" diff --git a/pkg/controller/v1beta1/inferenceservice/controller.go b/pkg/controller/v1beta1/inferenceservice/controller.go index df7fb286827..6ee7c79384a 100644 --- a/pkg/controller/v1beta1/inferenceservice/controller.go +++ b/pkg/controller/v1beta1/inferenceservice/controller.go @@ -28,6 +28,7 @@ import ( v1beta1api "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/components" + "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/cabundleconfigmap" "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress" modelconfig "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/reconcilers/modelconfig" isvcutils "github.com/kserve/kserve/pkg/controller/v1beta1/inferenceservice/utils" @@ -168,6 +169,13 @@ func (r *InferenceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Req if err != nil { return reconcile.Result{}, errors.Wrapf(err, "fails to create InferenceServicesConfig") } + + // Reconcile cabundleConfigMap + caBundleConfigMapReconciler := cabundleconfigmap.NewCaBundleConfigMapReconciler(r.Client, r.Scheme) + if err := caBundleConfigMapReconciler.Reconcile(isvc); err != nil { + return reconcile.Result{}, err + } + reconcilers := []components.Component{} if deploymentMode != constants.ModelMeshDeployment { reconcilers = append(reconcilers, components.NewPredictor(r.Client, r.Scheme, isvcConfig, deploymentMode)) diff --git a/pkg/controller/v1beta1/inferenceservice/controller_test.go b/pkg/controller/v1beta1/inferenceservice/controller_test.go index bbefe3ed2f7..20be321b868 100644 --- a/pkg/controller/v1beta1/inferenceservice/controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/controller_test.go @@ -35,6 +35,7 @@ import ( istiov1alpha3 "istio.io/api/networking/v1alpha3" "istio.io/client-go/pkg/apis/networking/v1alpha3" v1 "k8s.io/api/core/v1" + apierr "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -67,17 +68,27 @@ var _ = Describe("v1beta1 inference service controller", func() { } configs = map[string]string{ "explainers": `{ - "alibi": { - "image": "kserve/alibi-explainer", - "defaultImageVersion": "latest" - } - }`, + "alibi": { + "image": "kserve/alibi-explainer", + "defaultImageVersion": "latest" + } + }`, "ingress": `{ - "ingressGateway": "knative-serving/knative-ingress-gateway", - "ingressService": "test-destination", - "localGateway": "knative-serving/knative-local-gateway", - "localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local" - }`, + "ingressGateway": "knative-serving/knative-ingress-gateway", + "ingressService": "test-destination", + "localGateway": "knative-serving/knative-local-gateway", + "localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local" + }`, + "storageInitializer": `{ + "image" : "kserve/storage-initializer:latest", + "memoryRequest": "100Mi", + "memoryLimit": "1Gi", + "cpuRequest": "100m", + "cpuLimit": "1", + "CaBundleConfigMapName": "", + "caBundleVolumeMountPath": "/etc/ssl/custom-certs", + "enableDirectPvcVolumeMount": false + }`, } ) Context("When creating inference service with predictor", func() { @@ -2060,4 +2071,317 @@ var _ = Describe("v1beta1 inference service controller", func() { })) }) }) + Context("Set CaBundle ConfigMap in inferenceservice-config confimap", func() { + It("Should not create a global cabundle configMap in a user namespace when CaBundleConfigMapName set ''", func() { + // Create configmap + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: configs, + } + + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + + By("By creating a new InferenceService") + serviceName := "sample-isvc" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: serviceName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + var storageUri = "s3://test/mnist/export" + ctx := context.Background() + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + }, + Spec: v1beta1.InferenceServiceSpec{ + Predictor: v1beta1.PredictorSpec{ + ComponentExtensionSpec: v1beta1.ComponentExtensionSpec{ + MinReplicas: v1beta1.GetIntReference(1), + MaxReplicas: 3, + }, + Tensorflow: &v1beta1.TFServingSpec{ + PredictorExtensionSpec: v1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + RuntimeVersion: proto.String("1.14.0"), + Container: v1.Container{ + Name: constants.InferenceServiceContainerName, + Resources: defaultResource, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) + + caBundleConfigMap := &v1.ConfigMap{} + Consistently(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: constants.DefaultGlobalCaBundleConfigMapName, Namespace: "default"}, caBundleConfigMap) + if err != nil { + if apierr.IsNotFound(err) { + return true + } + } + return false + }, timeout, interval).Should(BeTrue()) + + }) + It("Should not create a global cabundle configmap in a user namespace when the target cabundle configmap in the 'inferenceservice-config' configmap does not exist", func() { + // Create configmap + copiedConfigs := make(map[string]string) + for key, value := range configs { + if key == "storageInitializer" { + copiedConfigs[key] = `{ + "image" : "kserve/storage-initializer:latest", + "memoryRequest": "100Mi", + "memoryLimit": "1Gi", + "cpuRequest": "100m", + "cpuLimit": "1", + "CaBundleConfigMapName": "not-exist-configmap", + "caBundleVolumeMountPath": "/etc/ssl/custom-certs", + "enableDirectPvcVolumeMount": false + }` + } else { + copiedConfigs[key] = value + } + } + + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: copiedConfigs, + } + + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + + By("By creating a new InferenceService") + serviceName := "sample-isvc-2" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: serviceName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + var storageUri = "s3://test/mnist/export" + ctx := context.Background() + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + }, + Spec: v1beta1.InferenceServiceSpec{ + Predictor: v1beta1.PredictorSpec{ + ComponentExtensionSpec: v1beta1.ComponentExtensionSpec{ + MinReplicas: v1beta1.GetIntReference(1), + MaxReplicas: 3, + }, + Tensorflow: &v1beta1.TFServingSpec{ + PredictorExtensionSpec: v1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + RuntimeVersion: proto.String("1.14.0"), + Container: v1.Container{ + Name: constants.InferenceServiceContainerName, + Resources: defaultResource, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) + defer k8sClient.Delete(ctx, isvc) + + caBundleConfigMap := &v1.ConfigMap{} + Consistently(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: constants.DefaultGlobalCaBundleConfigMapName, Namespace: "default"}, caBundleConfigMap) + if err != nil { + if apierr.IsNotFound(err) { + return true + } + } + return false + }, timeout, interval).Should(BeTrue()) + + }) + It("Should not create a global cabundle configmap in a user namespace when the cabundle.crt file data does not exist in the target cabundle configmap in the 'inferenceservice-config' configmap", func() { + // Create configmap + copiedConfigs := make(map[string]string) + for key, value := range configs { + if key == "storageInitializer" { + copiedConfigs[key] = `{ + "image" : "kserve/storage-initializer:latest", + "memoryRequest": "100Mi", + "memoryLimit": "1Gi", + "cpuRequest": "100m", + "cpuLimit": "1", + "CaBundleConfigMapName": "test-cabundle-with-wrong-file-name", + "caBundleVolumeMountPath": "/etc/ssl/custom-certs", + "enableDirectPvcVolumeMount": false + }` + } else { + copiedConfigs[key] = value + } + } + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: copiedConfigs, + } + + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + + // Create original cabundle configmap with wrong file name + cabundleConfigMapData := make(map[string]string) + cabundleConfigMapData["wrong-cabundle-name.crt"] = "SAMPLE_CA_BUNDLE" + var originalCabundleConfigMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cabundle-with-wrong-file-name", + Namespace: constants.KServeNamespace, + }, + Data: cabundleConfigMapData, + } + + Expect(k8sClient.Create(context.TODO(), originalCabundleConfigMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), originalCabundleConfigMap) + + By("By creating a new InferenceService") + serviceName := "sample-isvc-3" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: serviceName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + var storageUri = "s3://test/mnist/export" + ctx := context.Background() + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + }, + Spec: v1beta1.InferenceServiceSpec{ + Predictor: v1beta1.PredictorSpec{ + ComponentExtensionSpec: v1beta1.ComponentExtensionSpec{ + MinReplicas: v1beta1.GetIntReference(1), + MaxReplicas: 3, + }, + Tensorflow: &v1beta1.TFServingSpec{ + PredictorExtensionSpec: v1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + RuntimeVersion: proto.String("1.14.0"), + Container: v1.Container{ + Name: constants.InferenceServiceContainerName, + Resources: defaultResource, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) + defer k8sClient.Delete(ctx, isvc) + + caBundleConfigMap := &v1.ConfigMap{} + Consistently(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: constants.DefaultGlobalCaBundleConfigMapName, Namespace: "default"}, caBundleConfigMap) + if err != nil { + if apierr.IsNotFound(err) { + return true + } + } + return false + }, timeout, interval).Should(BeTrue()) + }) + + It("Should create a global cabundle configmap in a user namespace when it meets all conditions and an inferenceservice is created", func() { + // Create configmap + copiedConfigs := make(map[string]string) + for key, value := range configs { + if key == "storageInitializer" { + copiedConfigs[key] = `{ + "image" : "kserve/storage-initializer:latest", + "memoryRequest": "100Mi", + "memoryLimit": "1Gi", + "cpuRequest": "100m", + "cpuLimit": "1", + "CaBundleConfigMapName": "test-cabundle-with-right-file-name", + "caBundleVolumeMountPath": "/etc/ssl/custom-certs", + "enableDirectPvcVolumeMount": false + }` + } else { + copiedConfigs[key] = value + } + } + var configMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.InferenceServiceConfigMapName, + Namespace: constants.KServeNamespace, + }, + Data: copiedConfigs, + } + + Expect(k8sClient.Create(context.TODO(), configMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configMap) + + //Create original cabundle configmap with right file name + cabundleConfigMapData := make(map[string]string) + // cabundle data + cabundleConfigMapData["cabundle.crt"] = "SAMPLE_CA_BUNDLE" + var originalCabundleConfigMap = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cabundle-with-right-file-name", + Namespace: constants.KServeNamespace, + }, + Data: cabundleConfigMapData, + } + + Expect(k8sClient.Create(context.TODO(), originalCabundleConfigMap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), originalCabundleConfigMap) + + By("By creating a new InferenceService") + serviceName := "sample-isvc-4" + var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: serviceName, Namespace: "default"}} + var serviceKey = expectedRequest.NamespacedName + var storageUri = "s3://test/mnist/export" + ctx := context.Background() + isvc := &v1beta1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceKey.Name, + Namespace: serviceKey.Namespace, + }, + Spec: v1beta1.InferenceServiceSpec{ + Predictor: v1beta1.PredictorSpec{ + ComponentExtensionSpec: v1beta1.ComponentExtensionSpec{ + MinReplicas: v1beta1.GetIntReference(1), + MaxReplicas: 3, + }, + Tensorflow: &v1beta1.TFServingSpec{ + PredictorExtensionSpec: v1beta1.PredictorExtensionSpec{ + StorageURI: &storageUri, + RuntimeVersion: proto.String("1.14.0"), + Container: v1.Container{ + Name: constants.InferenceServiceContainerName, + Resources: defaultResource, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, isvc)).Should(Succeed()) + defer k8sClient.Delete(ctx, isvc) + + caBundleConfigMap := &v1.ConfigMap{} + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: constants.DefaultGlobalCaBundleConfigMapName, Namespace: "default"}, caBundleConfigMap) + if err != nil { + if apierr.IsNotFound(err) { + return false + } + } + return true + }, timeout, interval).Should(BeTrue()) + }) + }) }) diff --git a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go index e4926e4bb87..e97e1ea74f0 100644 --- a/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go +++ b/pkg/controller/v1beta1/inferenceservice/rawkube_controller_test.go @@ -67,18 +67,27 @@ var _ = Describe("v1beta1 inference service controller", func() { Context("When creating inference service with raw kube predictor", func() { configs := map[string]string{ "explainers": `{ - "alibi": { - "image": "kfserving/alibi-explainer", - "defaultImageVersion": "latest" - } - }`, + "alibi": { + "image": "kserve/alibi-explainer", + "defaultImageVersion": "latest" + } + }`, "ingress": `{ - "ingressGateway": "knative-serving/knative-ingress-gateway", - "ingressService": "test-destination", - "localGateway": "knative-serving/knative-local-gateway", - "localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local", - "ingressDomain": "example.com" - }`, + "ingressGateway": "knative-serving/knative-ingress-gateway", + "ingressService": "test-destination", + "localGateway": "knative-serving/knative-local-gateway", + "localGatewayService": "knative-local-gateway.istio-system.svc.cluster.local" + }`, + "storageInitializer": `{ + "image" : "kserve/storage-initializer:latest", + "memoryRequest": "100Mi", + "memoryLimit": "1Gi", + "cpuRequest": "100m", + "cpuLimit": "1", + "CaBundleConfigMapName": "", + "caBundleVolumeMountPath": "/etc/ssl/custom-certs", + "enableDirectPvcVolumeMount": false + }`, } It("Should have ingress/service/deployment/hpa created", func() { diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/cabundleconfigmap/cabundle_configmap_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/cabundleconfigmap/cabundle_configmap_reconciler.go new file mode 100644 index 00000000000..4f32de2c72b --- /dev/null +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/cabundleconfigmap/cabundle_configmap_reconciler.go @@ -0,0 +1,159 @@ +/* +Copyright 2023 The KServe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cabundleconfigmap + +import ( + "context" + "encoding/json" + "fmt" + + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + "github.com/kserve/kserve/pkg/constants" + "github.com/kserve/kserve/pkg/webhook/admission/pod" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "k8s.io/apimachinery/pkg/api/equality" + apierr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "knative.dev/pkg/kmp" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var log = logf.Log.WithName("CaBundleConfigMapReconciler") + +type CaBundleConfigMapReconciler struct { + client client.Client + scheme *runtime.Scheme +} + +func NewCaBundleConfigMapReconciler(client client.Client, scheme *runtime.Scheme) *CaBundleConfigMapReconciler { + return &CaBundleConfigMapReconciler{ + client: client, + scheme: scheme, + } +} + +func (c *CaBundleConfigMapReconciler) Reconcile(isvc *kservev1beta1.InferenceService) error { + log.Info("Reconciling CaBundleConfigMap", "namespace", isvc.Namespace) + + isvcConfigMap := &corev1.ConfigMap{} + err := c.client.Get(context.TODO(), types.NamespacedName{Name: constants.InferenceServiceConfigMapName, Namespace: constants.KServeNamespace}, isvcConfigMap) + if err != nil { + log.Error(err, "failed to find config map", "name", constants.InferenceServiceConfigMapName) + return err + } + + storageInitializerConfig := &pod.StorageInitializerConfig{} + if storageInitializerConfigValue, ok := isvcConfigMap.Data["storageInitializer"]; ok { + err := json.Unmarshal([]byte(storageInitializerConfigValue), &storageInitializerConfig) + if err != nil { + return fmt.Errorf("unable to unmarshal storage initializer json string due to %w ", err) + } + } + + var newCaBundleConfigMap *corev1.ConfigMap + if storageInitializerConfig.CaBundleConfigMapName == "" { + return nil + } else { + newCaBundleConfigMap, err = c.getCabundleConfigMapForUserNS(storageInitializerConfig.CaBundleConfigMapName, constants.KServeNamespace, isvc.Namespace) + if err != nil { + return fmt.Errorf("fails to get cabundle configmap for creating to user namespace: %w", err) + } + } + + if err := c.ReconcileCaBundleConfigMap(newCaBundleConfigMap); err != nil { + return fmt.Errorf("fails to reconcile cabundle configmap: %w", err) + } + + return nil +} + +func (c *CaBundleConfigMapReconciler) getCabundleConfigMapForUserNS(caBundleNameInConfig string, kserveNamespace string, isvcNamespace string) (*corev1.ConfigMap, error) { + var newCaBundleConfigMap *corev1.ConfigMap + + // Check if cabundle configmap exist & the cabundle.crt exist in the data in controller namespace + // If it does not exist, return error + caBundleConfigMap := &corev1.ConfigMap{} + if err := c.client.Get(context.TODO(), + types.NamespacedName{Name: caBundleNameInConfig, Namespace: kserveNamespace}, caBundleConfigMap); err == nil { + + if caBundleConfigMapData := caBundleConfigMap.Data[constants.DefaultCaBundleFileName]; caBundleConfigMapData == "" { + return nil, fmt.Errorf("specified cabundle file %s not found in cabundle configmap %s", + constants.DefaultCaBundleFileName, caBundleNameInConfig) + } else { + configData := map[string]string{ + constants.DefaultCaBundleFileName: caBundleConfigMapData, + } + newCaBundleConfigMap = getDesiredCaBundleConfigMapForUserNS(constants.DefaultGlobalCaBundleConfigMapName, isvcNamespace, configData) + } + } else { + return nil, fmt.Errorf("can't read cabundle configmap %s: %w", constants.DefaultCaBundleFileName, err) + } + + return newCaBundleConfigMap, nil +} + +func getDesiredCaBundleConfigMapForUserNS(configmapName string, namespace string, cabundleData map[string]string) *corev1.ConfigMap { + desiredConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configmapName, + Namespace: namespace, + }, + Data: cabundleData, + } + + return desiredConfigMap +} + +// ReconcileCaBundleConfigMap will manage the creation, update and deletion of the ca bundle ConfigMap +func (c *CaBundleConfigMapReconciler) ReconcileCaBundleConfigMap(desiredConfigMap *corev1.ConfigMap) error { + + // Create ConfigMap if does not exist + existingConfigMap := &corev1.ConfigMap{} + err := c.client.Get(context.TODO(), types.NamespacedName{Name: desiredConfigMap.Name, Namespace: desiredConfigMap.Namespace}, existingConfigMap) + if err != nil { + if apierr.IsNotFound(err) { + log.Info("Creating cabundle configmap", "namespace", desiredConfigMap.Namespace, "name", desiredConfigMap.Name) + err = c.client.Create(context.TODO(), desiredConfigMap) + } + return err + } + + // Return if no differences to reconcile. + if equality.Semantic.DeepEqual(desiredConfigMap, existingConfigMap) { + return nil + } + + // Reconcile differences and update + diff, err := kmp.SafeDiff(desiredConfigMap.Data, existingConfigMap.Data) + if err != nil { + return fmt.Errorf("failed to diff cabundle configmap: %w", err) + } + log.V(1).Info("Reconciling cabundle configmap diff (-desired, +observed):", "diff", diff) + log.Info("Updating cabundle configmap", "namespace", existingConfigMap.Namespace, "name", existingConfigMap.Name) + existingConfigMap.Data = desiredConfigMap.Data + err = c.client.Update(context.TODO(), existingConfigMap) + if err != nil { + return fmt.Errorf("fails to update cabundle configmap: %w", err) + } + + return nil +} diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/cabundleconfigmap/cabundle_configmap_reconciler_test.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/cabundleconfigmap/cabundle_configmap_reconciler_test.go new file mode 100644 index 00000000000..52b3252e912 --- /dev/null +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/cabundleconfigmap/cabundle_configmap_reconciler_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2023 The KServe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cabundleconfigmap + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/kserve/kserve/pkg/constants" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGetDesiredCaBundleConfigMapForUserNS(t *testing.T) { + cabundleConfigMapData := make(map[string]string) + + // cabundle data + cabundleConfigMapData["cabundle.crt"] = "SAMPLE_CA_BUNDLE" + targetNamespace := "test" + testCases := []struct { + name string + namespace string + configMapData map[string]string + expectedCopiedCaConfigMap *corev1.ConfigMap + }{ + { + name: "Do not create a ca bundle configmap,if CaBundleConfigMapName is '' in storageConfig of inference-config configmap", + namespace: targetNamespace, + configMapData: cabundleConfigMapData, + expectedCopiedCaConfigMap: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.DefaultGlobalCaBundleConfigMapName, + Namespace: targetNamespace, + }, + Data: cabundleConfigMapData, + }, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + result := getDesiredCaBundleConfigMapForUserNS(constants.DefaultGlobalCaBundleConfigMapName, tt.namespace, tt.configMapData) + if diff := cmp.Diff(tt.expectedCopiedCaConfigMap, result); diff != "" { + t.Errorf("Test %q unexpected result (-want +got): %v", t.Name(), diff) + } + }) + } +} diff --git a/pkg/credentials/s3/s3_secret.go b/pkg/credentials/s3/s3_secret.go index dc3c7c621a9..10d4c75da4c 100644 --- a/pkg/credentials/s3/s3_secret.go +++ b/pkg/credentials/s3/s3_secret.go @@ -39,6 +39,7 @@ const ( S3UseVirtualBucket = "S3_USER_VIRTUAL_BUCKET" AWSAnonymousCredential = "awsAnonymousCredential" AWSCABundle = "AWS_CA_BUNDLE" + AWSCABundleConfigMap = "AWS_CA_BUNDLE_CONFIGMAP" ) type S3Config struct { @@ -50,17 +51,19 @@ type S3Config struct { S3VerifySSL string `json:"s3VerifySSL,omitempty"` S3UseVirtualBucket string `json:"s3UseVirtualBucket,omitempty"` S3UseAnonymousCredential string `json:"s3UseAnonymousCredential,omitempty"` + S3CABundleConfigMap string `json:"s3CABundleConfigMap,omitempty"` S3CABundle string `json:"s3CABundle,omitempty"` } var ( - InferenceServiceS3SecretEndpointAnnotation = constants.KServeAPIGroupName + "/" + "s3-endpoint" - InferenceServiceS3SecretRegionAnnotation = constants.KServeAPIGroupName + "/" + "s3-region" - InferenceServiceS3SecretSSLAnnotation = constants.KServeAPIGroupName + "/" + "s3-verifyssl" - InferenceServiceS3SecretHttpsAnnotation = constants.KServeAPIGroupName + "/" + "s3-usehttps" - InferenceServiceS3UseVirtualBucketAnnotation = constants.KServeAPIGroupName + "/" + "s3-usevirtualbucket" - InferenceServiceS3UseAnonymousCredential = constants.KServeAPIGroupName + "/" + "s3-useanoncredential" - InferenceServiceS3CABundleAnnotation = constants.KServeAPIGroupName + "/" + "s3-cabundle" + InferenceServiceS3SecretEndpointAnnotation = constants.KServeAPIGroupName + "/" + "s3-endpoint" + InferenceServiceS3SecretRegionAnnotation = constants.KServeAPIGroupName + "/" + "s3-region" + InferenceServiceS3SecretSSLAnnotation = constants.KServeAPIGroupName + "/" + "s3-verifyssl" + InferenceServiceS3SecretHttpsAnnotation = constants.KServeAPIGroupName + "/" + "s3-usehttps" + InferenceServiceS3UseVirtualBucketAnnotation = constants.KServeAPIGroupName + "/" + "s3-usevirtualbucket" + InferenceServiceS3UseAnonymousCredential = constants.KServeAPIGroupName + "/" + "s3-useanoncredential" + InferenceServiceS3CABundleConfigMapAnnotation = constants.KServeAPIGroupName + "/" + "s3-cabundle-configmap" + InferenceServiceS3CABundleAnnotation = constants.KServeAPIGroupName + "/" + "s3-cabundle" ) func BuildSecretEnvs(secret *v1.Secret, s3Config *S3Config) []v1.EnvVar { diff --git a/pkg/credentials/s3/utils.go b/pkg/credentials/s3/utils.go index 37d31bc9d04..b23e4be83e4 100644 --- a/pkg/credentials/s3/utils.go +++ b/pkg/credentials/s3/utils.go @@ -117,5 +117,16 @@ func BuildS3EnvVars(annotations map[string]string, s3Config *S3Config) []v1.EnvV }) } + customCABundleConfigMap, ok := annotations[InferenceServiceS3CABundleConfigMapAnnotation] + if !ok { + customCABundleConfigMap = s3Config.S3CABundleConfigMap + } + if customCABundleConfigMap != "" { + envs = append(envs, v1.EnvVar{ + Name: AWSCABundleConfigMap, + Value: customCABundleConfigMap, + }) + } + return envs } diff --git a/pkg/credentials/s3/utils_test.go b/pkg/credentials/s3/utils_test.go index d97fbe5fad7..928510c83d0 100644 --- a/pkg/credentials/s3/utils_test.go +++ b/pkg/credentials/s3/utils_test.go @@ -46,13 +46,14 @@ func TestBuildS3EnvVars(t *testing.T) { }, "AllAnnotations": { annotations: map[string]string{ - InferenceServiceS3SecretEndpointAnnotation: "s3.aws.com", - InferenceServiceS3SecretRegionAnnotation: "us-east-2", - InferenceServiceS3SecretSSLAnnotation: "0", - InferenceServiceS3SecretHttpsAnnotation: "0", - InferenceServiceS3UseVirtualBucketAnnotation: "true", - InferenceServiceS3UseAnonymousCredential: "true", - InferenceServiceS3CABundleAnnotation: "value", + InferenceServiceS3SecretEndpointAnnotation: "s3.aws.com", + InferenceServiceS3SecretRegionAnnotation: "us-east-2", + InferenceServiceS3SecretSSLAnnotation: "0", + InferenceServiceS3SecretHttpsAnnotation: "0", + InferenceServiceS3UseVirtualBucketAnnotation: "true", + InferenceServiceS3UseAnonymousCredential: "true", + InferenceServiceS3CABundleAnnotation: "value", + InferenceServiceS3CABundleConfigMapAnnotation: "value", }, expected: []v1.EnvVar{ { @@ -87,6 +88,10 @@ func TestBuildS3EnvVars(t *testing.T) { Name: AWSCABundle, Value: "value", }, + { + Name: AWSCABundleConfigMap, + Value: "value", + }, }, }, } diff --git a/pkg/credentials/service_account_credentials.go b/pkg/credentials/service_account_credentials.go index e63adb919e8..0de908cb0d4 100644 --- a/pkg/credentials/service_account_credentials.go +++ b/pkg/credentials/service_account_credentials.go @@ -137,6 +137,12 @@ func (c *CredentialBuilder) CreateStorageSpecSecretEnvs(namespace string, annota if _, ok = storageDataJson["bucket"]; ok && bucket == "" { bucket = storageDataJson["bucket"] } + if cabundle_configmap, ok := storageDataJson["cabundle_configmap"]; ok { + container.Env = append(container.Env, v1.EnvVar{ + Name: s3.AWSCABundleConfigMap, + Value: cabundle_configmap, + }) + } } // Pass storage config json as SecretKeyRef env var diff --git a/pkg/webhook/admission/pod/storage_initializer_injector.go b/pkg/webhook/admission/pod/storage_initializer_injector.go index f9983efce84..cba5e5eede2 100644 --- a/pkg/webhook/admission/pod/storage_initializer_injector.go +++ b/pkg/webhook/admission/pod/storage_initializer_injector.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "path/filepath" "strconv" "strings" @@ -29,6 +30,7 @@ import ( "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" "github.com/kserve/kserve/pkg/constants" "github.com/kserve/kserve/pkg/credentials" + "github.com/kserve/kserve/pkg/credentials/s3" v1 "k8s.io/api/core/v1" "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -44,6 +46,7 @@ const ( PvcSourceMountName = "kserve-pvc-source" PvcSourceMountPath = "/mnt/pvc" OpenShiftUidRangeAnnotationKey = "openshift.io/sa.scc.uid-range" + CaBundleVolumeName = "cabundle-cert" ) type StorageInitializerConfig struct { @@ -52,6 +55,8 @@ type StorageInitializerConfig struct { CpuLimit string `json:"cpuLimit"` MemoryRequest string `json:"memoryRequest"` MemoryLimit string `json:"memoryLimit"` + CaBundleConfigMapName string `json:"caBundleConfigMapName"` + CaBundleVolumeMountPath string `json:"caBundleVolumeMountPath"` EnableDirectPvcVolumeMount bool `json:"enableDirectPvcVolumeMount"` } @@ -355,6 +360,58 @@ func (mi *StorageInitializerInjector) InjectStorageInitializer(pod *v1.Pod, targ } } + // Inject CA bundle configMap if caBundleConfigMapName or constants.DefaultGlobalCaBundleConfigMapName annotation is set + caBundleConfigMapName := mi.config.CaBundleConfigMapName + if ok := needCaBundleMount(caBundleConfigMapName, initContainer); ok { + if pod.Namespace != constants.KServeNamespace { + caBundleConfigMapName = constants.DefaultGlobalCaBundleConfigMapName + } + + caBundleVolumeMountPath := mi.config.CaBundleVolumeMountPath + if caBundleVolumeMountPath == "" { + caBundleVolumeMountPath = constants.DefaultCaBundleVolumeMountPath + } + + for _, envVar := range initContainer.Env { + if envVar.Name == s3.AWSCABundleConfigMap { + caBundleConfigMapName = envVar.Value + } + if envVar.Name == s3.AWSCABundle { + caBundleVolumeMountPath = filepath.Dir(envVar.Value) + } + } + + initContainer.Env = append(initContainer.Env, v1.EnvVar{ + Name: constants.CaBundleConfigMapNameEnvVarKey, + Value: caBundleConfigMapName, + }) + + initContainer.Env = append(initContainer.Env, v1.EnvVar{ + Name: constants.CaBundleVolumeMountPathEnvVarKey, + Value: caBundleVolumeMountPath, + }) + + caBundleVolume := v1.Volume{ + Name: CaBundleVolumeName, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: caBundleConfigMapName, + }, + }, + }, + } + + caBundleVolumeMount := v1.VolumeMount{ + Name: CaBundleVolumeName, + MountPath: caBundleVolumeMountPath, + ReadOnly: true, + } + + pod.Spec.Volumes = append(pod.Spec.Volumes, caBundleVolume) + initContainer.VolumeMounts = append(initContainer.VolumeMounts, caBundleVolumeMount) + } + // Update initContainer (container spec) from a storage container CR if there is a match, // otherwise initContainer is not updated. // Priority: CR > configMap @@ -458,3 +515,17 @@ func parsePvcURI(srcURI string) (pvcName string, pvcPath string, err error) { return pvcName, pvcPath, nil } + +func needCaBundleMount(caBundleConfigMapName string, initContainer *v1.Container) bool { + result := false + if caBundleConfigMapName != "" { + result = true + } + for _, envVar := range initContainer.Env { + if envVar.Name == s3.AWSCABundleConfigMap { + result = true + break + } + } + return result +} diff --git a/pkg/webhook/admission/pod/storage_initializer_injector_test.go b/pkg/webhook/admission/pod/storage_initializer_injector_test.go index 873c9ed93bc..175f153f2dc 100644 --- a/pkg/webhook/admission/pod/storage_initializer_injector_test.go +++ b/pkg/webhook/admission/pod/storage_initializer_injector_test.go @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package pod import ( @@ -42,6 +43,8 @@ const ( StorageInitializerDefaultCPULimit = "1" StorageInitializerDefaultMemoryRequest = "200Mi" StorageInitializerDefaultMemoryLimit = "1Gi" + StorageInitializerDefaultCaBundleConfigMapName = "" + StorageInitializerDefaultCaBundleVolumeMountPath = "/etc/ssl/custom-certs" StorageInitializerDefaultEnableDirectPvcVolumeMount = false ) @@ -51,6 +54,8 @@ var ( CpuLimit: StorageInitializerDefaultCPULimit, MemoryRequest: StorageInitializerDefaultMemoryRequest, MemoryLimit: StorageInitializerDefaultMemoryLimit, + CaBundleConfigMapName: StorageInitializerDefaultCaBundleConfigMapName, + CaBundleVolumeMountPath: StorageInitializerDefaultCaBundleVolumeMountPath, EnableDirectPvcVolumeMount: StorageInitializerDefaultEnableDirectPvcVolumeMount, } @@ -1210,11 +1215,13 @@ func TestStorageInitializerConfigmap(t *testing.T) { Data: map[string]string{}, }), config: &StorageInitializerConfig{ - Image: "kserve/storage-initializer@sha256:xxx", - CpuRequest: StorageInitializerDefaultCPURequest, - CpuLimit: StorageInitializerDefaultCPULimit, - MemoryRequest: StorageInitializerDefaultMemoryRequest, - MemoryLimit: StorageInitializerDefaultMemoryLimit, + Image: "kserve/storage-initializer@sha256:xxx", + CpuRequest: StorageInitializerDefaultCPURequest, + CpuLimit: StorageInitializerDefaultCPULimit, + MemoryRequest: StorageInitializerDefaultMemoryRequest, + MemoryLimit: StorageInitializerDefaultMemoryLimit, + CaBundleConfigMapName: StorageInitializerDefaultCaBundleConfigMapName, + CaBundleVolumeMountPath: StorageInitializerDefaultCaBundleVolumeMountPath, }, client: c, } @@ -1245,18 +1252,22 @@ func TestGetStorageInitializerConfigs(t *testing.T) { "CpuRequest": "100m", "CpuLimit": "1", "MemoryRequest": "200Mi", - "MemoryLimit": "1Gi" + "MemoryLimit": "1Gi", + "CaBundleConfigMapName": "", + "CaBundleVolumeMountPath": "/etc/ssl/custom-certs" }`, }, BinaryData: map[string][]byte{}, }, matchers: []types.GomegaMatcher{ gomega.Equal(&StorageInitializerConfig{ - Image: "gcr.io/kserve/storage-initializer:latest", - CpuRequest: "100m", - CpuLimit: "1", - MemoryRequest: "200Mi", - MemoryLimit: "1Gi", + Image: "gcr.io/kserve/storage-initializer:latest", + CpuRequest: "100m", + CpuLimit: "1", + MemoryRequest: "200Mi", + MemoryLimit: "1Gi", + CaBundleConfigMapName: "", + CaBundleVolumeMountPath: "/etc/ssl/custom-certs", }), gomega.BeNil(), }, @@ -1272,18 +1283,22 @@ func TestGetStorageInitializerConfigs(t *testing.T) { "CpuRequest": "100m", "CpuLimit": "1", "MemoryRequest": "200MC", - "MemoryLimit": "1Gi" + "MemoryLimit": "1Gi", + "CaBundleConfigMapName": "", + "CaBundleVolumeMountPath": "/etc/ssl/custom-certs" }`, }, BinaryData: map[string][]byte{}, }, matchers: []types.GomegaMatcher{ gomega.Equal(&StorageInitializerConfig{ - Image: "gcr.io/kserve/storage-initializer:latest", - CpuRequest: "100m", - CpuLimit: "1", - MemoryRequest: "200MC", - MemoryLimit: "1Gi", + Image: "gcr.io/kserve/storage-initializer:latest", + CpuRequest: "100m", + CpuLimit: "1", + MemoryRequest: "200MC", + MemoryLimit: "1Gi", + CaBundleConfigMapName: "", + CaBundleVolumeMountPath: "/etc/ssl/custom-certs", }), gomega.HaveOccurred(), }, @@ -1335,6 +1350,734 @@ func TestParsePvcURI(t *testing.T) { } } +func TestCaBundleConfigMapVolumeMountInStorageInitializer(t *testing.T) { + g := gomega.NewGomegaWithT(t) + var configMap = &v1.ConfigMap{ + Data: map[string]string{ + "credentials": `{ + "gcs" : {"gcsCredentialFileName": "gcloud-application-credentials.json"}, + "s3" : { + "s3AccessKeyIDName": "awsAccessKeyID", + "s3SecretAccessKeyName": "awsSecretAccessKey" + } + }`, + }, + } + scenarios := map[string]struct { + storageConfig *StorageInitializerConfig + secret *v1.Secret + sa *v1.ServiceAccount + original *v1.Pod + expected *v1.Pod + }{ + "DoNotMountWithCaBundleConfigMapVolumeWhenCaBundleConfigMapNameNotSet": { + storageConfig: storageInitializerConfig, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "s3-secret", + Namespace: "default", + }, + Data: map[string][]byte{ + "awsAccessKeyID": {}, + "awsSecretAccessKey": {}, + }, + }, + sa: &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "default", + }, + Secrets: []v1.ObjectReference{ + { + Name: "s3-secret", + Namespace: "default", + }, + }, + }, + original: makePod(), + expected: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.StorageInitializerSourceUriInternalAnnotationKey: "gs://foo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: constants.InferenceServiceContainerName, + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + ReadOnly: true, + }, + }, + }, + }, + InitContainers: []v1.Container{ + { + Name: "storage-initializer", + Image: StorageInitializerContainerImage + ":" + StorageInitializerContainerImageVersion, + Args: []string{"gs://foo", constants.DefaultModelLocalMountPath}, + Env: []v1.EnvVar{ + { + Name: s3.AWSAccessKeyId, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsAccessKeyID", + }, + }, + }, + { + Name: s3.AWSSecretAccessKey, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsSecretAccessKey", + }, + }, + }, + }, + Resources: resourceRequirement, + TerminationMessagePolicy: "FallbackToLogsOnError", + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "kserve-provision-location", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + }, + "MountsCaBundleConfigMapVolumeWhenCaBundleConfigMapNameSet": { + storageConfig: &StorageInitializerConfig{ + Image: "kserve/storage-initializer:latest", + CpuRequest: "100m", + CpuLimit: "1", + MemoryRequest: "200Mi", + MemoryLimit: "1Gi", + CaBundleConfigMapName: "custom-certs", // enable CA bundle config volume mount + }, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "s3-secret", + Namespace: "default", + }, + Data: map[string][]byte{ + "awsAccessKeyID": {}, + "awsSecretAccessKey": {}, + }, + }, + sa: &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "default", + }, + Secrets: []v1.ObjectReference{ + { + Name: "s3-secret", + Namespace: "default", + }, + }, + }, + original: makePod(), + expected: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.StorageInitializerSourceUriInternalAnnotationKey: "gs://foo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: constants.InferenceServiceContainerName, + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + ReadOnly: true, + }, + }, + }, + }, + InitContainers: []v1.Container{ + { + Name: "storage-initializer", + Image: StorageInitializerContainerImage + ":" + StorageInitializerContainerImageVersion, + Args: []string{"gs://foo", constants.DefaultModelLocalMountPath}, + Env: []v1.EnvVar{ + { + Name: s3.AWSAccessKeyId, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsAccessKeyID", + }, + }, + }, + { + Name: s3.AWSSecretAccessKey, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsSecretAccessKey", + }, + }, + }, + {Name: "CA_BUNDLE_CONFIGMAP_NAME", Value: constants.DefaultGlobalCaBundleConfigMapName}, + {Name: "CA_BUNDLE_VOLUME_MOUNT_POINT", Value: "/etc/ssl/custom-certs"}, + }, + Resources: resourceRequirement, + TerminationMessagePolicy: "FallbackToLogsOnError", + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + }, + { + Name: CaBundleVolumeName, + MountPath: constants.DefaultCaBundleVolumeMountPath, + ReadOnly: true, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "kserve-provision-location", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + { + Name: CaBundleVolumeName, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: constants.DefaultGlobalCaBundleConfigMapName, + }, + }, + }, + }, + }, + }, + }, + }, + "MountsCaBundleConfigMapVolumeByAnnotation": { + storageConfig: &StorageInitializerConfig{ + Image: "kserve/storage-initializer:latest", + CpuRequest: "100m", + CpuLimit: "1", + MemoryRequest: "200Mi", + MemoryLimit: "1Gi", + }, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "s3-secret", + Namespace: "default", + Annotations: map[string]string{ + s3.InferenceServiceS3CABundleConfigMapAnnotation: "cabundle-annotation", + }, + }, + Data: map[string][]byte{ + "awsAccessKeyID": {}, + "awsSecretAccessKey": {}, + }, + }, + sa: &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "default", + }, + Secrets: []v1.ObjectReference{ + { + Name: "s3-secret", + Namespace: "default", + }, + }, + }, + original: makePod(), + expected: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.StorageInitializerSourceUriInternalAnnotationKey: "gs://foo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: constants.InferenceServiceContainerName, + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + ReadOnly: true, + }, + }, + }, + }, + InitContainers: []v1.Container{ + { + Name: "storage-initializer", + Image: StorageInitializerContainerImage + ":" + StorageInitializerContainerImageVersion, + Args: []string{"gs://foo", constants.DefaultModelLocalMountPath}, + Env: []v1.EnvVar{ + { + Name: s3.AWSAccessKeyId, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsAccessKeyID", + }, + }, + }, + { + Name: s3.AWSSecretAccessKey, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsSecretAccessKey", + }, + }, + }, + {Name: "AWS_CA_BUNDLE_CONFIGMAP", Value: "cabundle-annotation"}, + {Name: "CA_BUNDLE_CONFIGMAP_NAME", Value: "cabundle-annotation"}, + {Name: "CA_BUNDLE_VOLUME_MOUNT_POINT", Value: "/etc/ssl/custom-certs"}, + }, + Resources: resourceRequirement, + TerminationMessagePolicy: "FallbackToLogsOnError", + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + }, + { + Name: CaBundleVolumeName, + MountPath: constants.DefaultCaBundleVolumeMountPath, + ReadOnly: true, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "kserve-provision-location", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + { + Name: CaBundleVolumeName, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "cabundle-annotation", + }, + }, + }, + }, + }, + }, + }, + }, + "MountsCaBundleConfigMapVolumeByAnnotationInstreadOfConfigMap": { + storageConfig: &StorageInitializerConfig{ + Image: "kserve/storage-initializer:latest", + CpuRequest: "100m", + CpuLimit: "1", + MemoryRequest: "200Mi", + MemoryLimit: "1Gi", + CaBundleConfigMapName: "custom-certs", // enable CA bundle configmap volume mount + }, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "s3-secret", + Namespace: "default", + Annotations: map[string]string{ + s3.InferenceServiceS3CABundleConfigMapAnnotation: "cabundle-annotation", + }, + }, + Data: map[string][]byte{ + "awsAccessKeyID": {}, + "awsSecretAccessKey": {}, + }, + }, + sa: &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "default", + }, + Secrets: []v1.ObjectReference{ + { + Name: "s3-secret", + Namespace: "default", + }, + }, + }, + original: makePod(), + expected: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.StorageInitializerSourceUriInternalAnnotationKey: "gs://foo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: constants.InferenceServiceContainerName, + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + ReadOnly: true, + }, + }, + }, + }, + InitContainers: []v1.Container{ + { + Name: "storage-initializer", + Image: StorageInitializerContainerImage + ":" + StorageInitializerContainerImageVersion, + Args: []string{"gs://foo", constants.DefaultModelLocalMountPath}, + Env: []v1.EnvVar{ + { + Name: s3.AWSAccessKeyId, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsAccessKeyID", + }, + }, + }, + { + Name: s3.AWSSecretAccessKey, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsSecretAccessKey", + }, + }, + }, + {Name: "AWS_CA_BUNDLE_CONFIGMAP", Value: "cabundle-annotation"}, + {Name: "CA_BUNDLE_CONFIGMAP_NAME", Value: "cabundle-annotation"}, + {Name: "CA_BUNDLE_VOLUME_MOUNT_POINT", Value: "/etc/ssl/custom-certs"}, + }, + Resources: resourceRequirement, + TerminationMessagePolicy: "FallbackToLogsOnError", + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + }, + { + Name: CaBundleVolumeName, + MountPath: constants.DefaultCaBundleVolumeMountPath, + ReadOnly: true, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "kserve-provision-location", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + { + Name: CaBundleVolumeName, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "cabundle-annotation", + }, + }, + }, + }, + }, + }, + }, + }, + "DoNotSetMountsCaBundleConfigMapVolumePathByAnnotationIfCaBundleConfigMapNameDidNotSet": { + storageConfig: storageInitializerConfig, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "s3-secret", + Namespace: "default", + Annotations: map[string]string{ + s3.InferenceServiceS3CABundleAnnotation: "/path/to/ca.crt", + }, + }, + Data: map[string][]byte{ + "awsAccessKeyID": {}, + "awsSecretAccessKey": {}, + }, + }, + sa: &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "default", + }, + Secrets: []v1.ObjectReference{ + { + Name: "s3-secret", + Namespace: "default", + }, + }, + }, + original: makePod(), + expected: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.StorageInitializerSourceUriInternalAnnotationKey: "gs://foo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: constants.InferenceServiceContainerName, + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + ReadOnly: true, + }, + }, + }, + }, + InitContainers: []v1.Container{ + { + Name: "storage-initializer", + Image: StorageInitializerContainerImage + ":" + StorageInitializerContainerImageVersion, + Args: []string{"gs://foo", constants.DefaultModelLocalMountPath}, + Env: []v1.EnvVar{ + { + Name: s3.AWSAccessKeyId, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsAccessKeyID", + }, + }, + }, + { + Name: s3.AWSSecretAccessKey, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsSecretAccessKey", + }, + }, + }, + {Name: "AWS_CA_BUNDLE", Value: "/path/to/ca.crt"}, + }, + Resources: resourceRequirement, + TerminationMessagePolicy: "FallbackToLogsOnError", + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "kserve-provision-location", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + }, + "SetMountsCaBundleConfigMapVolumePathByAnnotationInstreadOfConfigMap": { + storageConfig: &StorageInitializerConfig{ + Image: "kserve/storage-initializer:latest", + CpuRequest: "100m", + CpuLimit: "1", + MemoryRequest: "200Mi", + MemoryLimit: "1Gi", + CaBundleConfigMapName: "custom-certs", // enable CA bundle configmap volume mount + CaBundleVolumeMountPath: "/path/to", // set CA bundle configmap volume mount path + }, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "s3-secret", + Namespace: "default", + Annotations: map[string]string{ + s3.InferenceServiceS3CABundleAnnotation: "/annotation/path/to/annotation-ca.crt", + }, + }, + Data: map[string][]byte{ + "awsAccessKeyID": {}, + "awsSecretAccessKey": {}, + }, + }, + sa: &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "default", + }, + Secrets: []v1.ObjectReference{ + { + Name: "s3-secret", + Namespace: "default", + }, + }, + }, + original: makePod(), + expected: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.StorageInitializerSourceUriInternalAnnotationKey: "gs://foo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: constants.InferenceServiceContainerName, + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + ReadOnly: true, + }, + }, + }, + }, + InitContainers: []v1.Container{ + { + Name: "storage-initializer", + Image: StorageInitializerContainerImage + ":" + StorageInitializerContainerImageVersion, + Args: []string{"gs://foo", constants.DefaultModelLocalMountPath}, + Env: []v1.EnvVar{ + { + Name: s3.AWSAccessKeyId, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsAccessKeyID", + }, + }, + }, + { + Name: s3.AWSSecretAccessKey, + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "s3-secret", + }, + Key: "awsSecretAccessKey", + }, + }, + }, + {Name: "AWS_CA_BUNDLE", Value: "/annotation/path/to/annotation-ca.crt"}, + {Name: "CA_BUNDLE_CONFIGMAP_NAME", Value: constants.DefaultGlobalCaBundleConfigMapName}, + {Name: "CA_BUNDLE_VOLUME_MOUNT_POINT", Value: "/annotation/path/to"}, + }, + Resources: resourceRequirement, + TerminationMessagePolicy: "FallbackToLogsOnError", + VolumeMounts: []v1.VolumeMount{ + { + Name: "kserve-provision-location", + MountPath: constants.DefaultModelLocalMountPath, + }, + { + Name: CaBundleVolumeName, + MountPath: "/annotation/path/to", + ReadOnly: true, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "kserve-provision-location", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + { + Name: CaBundleVolumeName, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: constants.DefaultGlobalCaBundleConfigMapName, + }, + }, + }, + }, + }, + }, + }, + }, + } + + builder := credentials.NewCredentialBuilder(c, configMap) + + ns := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + } + + for name, scenario := range scenarios { + g.Expect(c.Create(context.TODO(), scenario.sa)).NotTo(gomega.HaveOccurred()) + g.Expect(c.Create(context.TODO(), scenario.secret)).NotTo(gomega.HaveOccurred()) + + injector := &StorageInitializerInjector{ + credentialBuilder: builder, + config: scenario.storageConfig, + client: c, + } + if err := injector.InjectStorageInitializer(scenario.original,ns); err != nil { + t.Errorf("Test %q unexpected failure [%s]", name, err.Error()) + } + if diff, _ := kmp.SafeDiff(scenario.expected.Spec, scenario.original.Spec); diff != "" { + t.Errorf("Test %q unexpected result (-want +got): %v", name, diff) + } + + g.Expect(c.Delete(context.TODO(), scenario.secret)).NotTo(gomega.HaveOccurred()) + g.Expect(c.Delete(context.TODO(), scenario.sa)).NotTo(gomega.HaveOccurred()) + } + +} + func TestDirectVolumeMountForPvc(t *testing.T) { scenarios := map[string]struct { original *v1.Pod diff --git a/python/kserve/kserve/storage/storage.py b/python/kserve/kserve/storage/storage.py index 7dcd299c2ae..cac673c4aff 100644 --- a/python/kserve/kserve/storage/storage.py +++ b/python/kserve/kserve/storage/storage.py @@ -171,6 +171,25 @@ def _download_s3(uri, temp_dir: str): if verify_ssl: verify_ssl = not verify_ssl.lower() in ["0", "false"] kwargs.update({"verify": verify_ssl}) + else: + verify_ssl = True + + # If verify_ssl is true, then check there is custom ca bundle cert + if verify_ssl: + global_ca_bundle_configmap = os.getenv("CA_BUNDLE_CONFIGMAP_NAME") + if global_ca_bundle_configmap: + isvc_aws_ca_bundle_path = os.getenv("AWS_CA_BUNDLE") + if isvc_aws_ca_bundle_path and isvc_aws_ca_bundle_path != "": + ca_bundle_full_path = isvc_aws_ca_bundle_path + else: + global_ca_bundle_volume_mount_path = os.getenv("CA_BUNDLE_VOLUME_MOUNT_POINT") + ca_bundle_full_path = global_ca_bundle_volume_mount_path + "/cabundle.crt" + if os.path.exists(ca_bundle_full_path): + logging.info('ca bundle file(%s) exists.' % (ca_bundle_full_path)) + kwargs.update({"verify": ca_bundle_full_path}) + else: + raise RuntimeError( + "Failed to find ca bundle file(%s)." % ca_bundle_full_path) s3 = boto3.resource("s3", **kwargs) parsed = urlparse(uri, scheme='s3') bucket_name = parsed.netloc