Merge pull request #83 from srl-labs/feat/cri-sock-global-config

Feat/cri sock global config

Makefile

@@ -14,7 +14,7 @@ fmt: ## Run formatters
 	goimports -w .
 	golines -w .
 
-lint: fmt ## Run linters; runs with GOOS env var for linting on darwin
+lint: fmt ## Run linters
 	golangci-lint run
 	helm lint --quiet charts/clabernetes
 	helm lint --quiet charts/clicker

apis/v1alpha1/configspec.go

@@ -75,4 +75,20 @@ type ConfigImagePull struct {
 	// +kubebuilder:validation:Enum=auto;always;never
 	// +optional
 	PullThroughOverride string `json:"pullThroughOverride,omitempty"`
+	// CRISockOverride allows for overriding the path of the CRI sock that is mounted in the
+	// launcher pods (if/when image pull through mode is auto or always). This can be useful if,
+	// for example, the CRI sock is in a "non-standard" location like K3s which puts the containerd
+	// sock at `/run/k3s/containerd/containerd.sock` rather than the "normal" (whatever that means)
+	// location of `/run/containerd/containerd.sock`. The value must end with "containerd.sock" for
+	// now, in the future maybe crio support will be added.
+	// +kubebuilder:validation:Pattern=(.*containerd\.sock)
+	// +optional
+	CRISockOverride string `json:"criSockOverride,omitempty"`
+	// CRIKindOverride allows for overriding the auto discovered cri flavor of the cluster -- this
+	// may be useful if we fail to parse the cri kind for some reason, or in mixed cri flavor
+	// clusters -- however in the latter case, make sure that if you are using image pull through
+	// that clabernetes workloads are only run on the nodes of the cri kind specified here!
+	// +kubebuilder:validation:Enum=containerd
+	// +optional
+	CRIKindOverride string `json:"criKindOverride,omitempty"`
 }

assets/crd/clabernetes.containerlab.dev_configs.yaml

@@ -205,6 +205,27 @@ spec:
                 description: ImagePull holds configurations relevant to how clabernetes
                   launcher pods handle pulling images.
                 properties:
+                  criKindOverride:
+                    description: CRIKindOverride allows for overriding the auto discovered
+                      cri flavor of the cluster -- this may be useful if we fail to
+                      parse the cri kind for some reason, or in mixed cri flavor clusters
+                      -- however in the latter case, make sure that if you are using
+                      image pull through that clabernetes workloads are only run on
+                      the nodes of the cri kind specified here!
+                    enum:
+                    - containerd
+                    type: string
+                  criSockOverride:
+                    description: CRISockOverride allows for overriding the path of
+                      the CRI sock that is mounted in the launcher pods (if/when image
+                      pull through mode is auto or always). This can be useful if,
+                      for example, the CRI sock is in a "non-standard" location like
+                      K3s which puts the containerd sock at `/run/k3s/containerd/containerd.sock`
+                      rather than the "normal" (whatever that means) location of `/run/containerd/containerd.sock`.
+                      The value must end with "containerd.sock" for now, in the future
+                      maybe crio support will be added.
+                    pattern: (.*containerd\.sock)
+                    type: string
                   pullThroughOverride:
                     description: PullThroughOverride allows for overriding the image
                       pull through mode for this particular topology.

charts/clabernetes/crds/clabernetes.containerlab.dev_configs.yaml

@@ -205,6 +205,27 @@ spec:
                 description: ImagePull holds configurations relevant to how clabernetes
                   launcher pods handle pulling images.
                 properties:
+                  criKindOverride:
+                    description: CRIKindOverride allows for overriding the auto discovered
+                      cri flavor of the cluster -- this may be useful if we fail to
+                      parse the cri kind for some reason, or in mixed cri flavor clusters
+                      -- however in the latter case, make sure that if you are using
+                      image pull through that clabernetes workloads are only run on
+                      the nodes of the cri kind specified here!
+                    enum:
+                    - containerd
+                    type: string
+                  criSockOverride:
+                    description: CRISockOverride allows for overriding the path of
+                      the CRI sock that is mounted in the launcher pods (if/when image
+                      pull through mode is auto or always). This can be useful if,
+                      for example, the CRI sock is in a "non-standard" location like
+                      K3s which puts the containerd sock at `/run/k3s/containerd/containerd.sock`
+                      rather than the "normal" (whatever that means) location of `/run/containerd/containerd.sock`.
+                      The value must end with "containerd.sock" for now, in the future
+                      maybe crio support will be added.
+                    pattern: (.*containerd\.sock)
+                    type: string
                   pullThroughOverride:
                     description: PullThroughOverride allows for overriding the image
                       pull through mode for this particular topology.

charts/clabernetes/templates/configmap.yaml

@@ -41,4 +41,10 @@ data:
   launcherImage: {{ .Values.globalConfig.deployment.launcherImage }}
   launcherImagePullPolicy: {{ .Values.globalConfig.deployment.launcherImagePullPolicy }}
   launcherLogLevel: {{ .Values.globalConfig.deployment.launcherLogLevel }}
+  {{- if .Values.globalConfig.imagePull.criSockOverride }}
+  criSockOverride: {{ .Values.globalConfig.imagePull.criSockOverride }}
+  {{- end }}
+  {{- if .Values.globalConfig.imagePull.criKindOverride }}
+  criKindOverride: {{ .Values.globalConfig.imagePull.criKindOverride }}
+  {{- end }}
 {{- end }}

charts/clabernetes/values.yaml

@@ -88,6 +88,16 @@ globalConfig:
     # ever pull via the docker daemon in the launcher pod itself (bypassing the cluster). Note that
     # "pull through mode" currently only supports containerd as a CRI.
     imagePullThroughMode: auto
+    # criSockOverride allows for overriding the path of the CRI sock that is mounted in the
+    # launcher pods (if/when image pull through mode is auto or always). This can be useful if,
+    # for example, the CRI sock is in a "non-standard" location like K3s which puts the containerd
+    # sock at `/run/k3s/containerd/containerd.sock` rather than the "normal" (whatever that means)
+    # location of `/run/containerd/containerd.sock`.
+    # criSockOverride: ""
+    # criKindOverride allows for overriding teh auto discovered cri kind. Probably/hopefully this
+    # won't be needed often, but could come in handy in multi-cri clusters or if nodes for some
+    # reason do not properly report their cri flavor (or we incorrectly parse it?!)
+    # criKindOverride: ""
 
   deployment:
     # resourcesDefault hold the default resources to apply to clabernetes launcher pods.

config/bootstrap.go

@@ -27,6 +27,8 @@ type bootstrapConfig struct {
 	launcherImage               string
 	launcherImagePullPolicy     string
 	launcherLogLevel            string
+	criSockOverride             string
+	criKindOverride             string
 }
 
 func bootstrapFromConfigMap( //nolint:gocyclo,funlen
@@ -125,6 +127,16 @@ func bootstrapFromConfigMap( //nolint:gocyclo,funlen
 		bc.launcherLogLevel = launcherLogLevel
 	}
 
+	criSockOverride, criSockOverrideOk := inMap["criSockOverride"]
+	if criSockOverrideOk {
+		bc.criSockOverride = criSockOverride
+	}
+
+	criKindOverride, criKindOverrideOk := inMap["criKindOverride"]
+	if criKindOverrideOk {
+		bc.criKindOverride = criKindOverride
+	}
+
 	var err error
 
 	if len(outErrors) > 0 {
@@ -217,6 +229,14 @@ func mergeFromBootstrapConfigMerge(
 	if config.Spec.Deployment.LauncherLogLevel == "" {
 		config.Spec.Deployment.LauncherLogLevel = bootstrap.launcherLogLevel
 	}
+
+	if config.Spec.ImagePull.CRISockOverride == "" {
+		config.Spec.ImagePull.CRISockOverride = bootstrap.criSockOverride
+	}
+
+	if config.Spec.ImagePull.CRIKindOverride == "" {
+		config.Spec.ImagePull.CRIKindOverride = bootstrap.criKindOverride
+	}
 }
 
 func mergeFromBootstrapConfigReplace(
@@ -231,6 +251,8 @@ func mergeFromBootstrapConfigReplace(
 		InClusterDNSSuffix: bootstrap.inClusterDNSSuffix,
 		ImagePull: clabernetesapisv1alpha1.ConfigImagePull{
 			PullThroughOverride: bootstrap.imagePullThroughMode,
+			CRISockOverride:     bootstrap.criSockOverride,
+			CRIKindOverride:     bootstrap.criKindOverride,
 		},
 		Deployment: clabernetesapisv1alpha1.ConfigDeployment{
 			ResourcesDefault:            bootstrap.resourcesDefault,

config/fake.go

@@ -58,6 +58,14 @@ func (f fakeManager) GetLauncherImage() string {
 	return "ghcr.io/srl-labs/clabernetes/clabernetes-launcher:latest"
 }
 
+func (f fakeManager) GetImagePullCriSockOverride() string {
+	return ""
+}
+
+func (f fakeManager) GetImagePullCriKindOverride() string {
+	return ""
+}
+
 func (f fakeManager) GetLauncherImagePullPolicy() string {
 	return clabernetesconstants.KubernetesImagePullIfNotPresent
 }

config/get.go

@@ -92,6 +92,20 @@ func (m *manager) GetImagePullThroughMode() string {
 	return m.config.ImagePull.PullThroughOverride
 }
 
+func (m *manager) GetImagePullCriSockOverride() string {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
+	return m.config.ImagePull.CRISockOverride
+}
+
+func (m *manager) GetImagePullCriKindOverride() string {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
+	return m.config.ImagePull.CRIKindOverride
+}
+
 func (m *manager) GetLauncherImage() string {
 	m.lock.RLock()
 	defer m.lock.RUnlock()

config/manager.go

@@ -130,6 +130,10 @@ type Manager interface {
 	GetInClusterDNSSuffix() string
 	// GetImagePullThroughMode returns the image pull through mode in the global config.
 	GetImagePullThroughMode() string
+	// GetImagePullCriSockOverride returns the cri sock path override.
+	GetImagePullCriSockOverride() string
+	// GetImagePullCriKindOverride returns the cri kind override.
+	GetImagePullCriKindOverride() string
 	// GetLauncherImage returns the global default launcher image.
 	GetLauncherImage() string
 	// GetLauncherImagePullPolicy returns the global default launcher image pull policy.

controllers/topology/deployment.go

@@ -2,6 +2,7 @@ package topology
 
 import (
 	"fmt"
+	"path/filepath"
 	"reflect"
 	"strings"
 
@@ -188,57 +189,35 @@ func (r *DeploymentReconciler) renderDeploymentVolumes(
 
 	volumeMountsFromCommonSpec := make([]k8scorev1.VolumeMount, 0)
 
-	// if we have containerd cri *and* pull through mode is auto or always, we need to mount the
-	// containerd sock
-	if r.configManagerGetter().
-		GetImagePullThroughMode() !=
-		clabernetesconstants.ImagePullThroughModeNever &&
-		owningTopology.Spec.ImagePull.PullThroughOverride != clabernetesconstants.ImagePullThroughModeNever { //nolint:lll
-		var path string
+	criPath, criSubPath := r.renderDeploymentVolumesGetCRISockPath(owningTopology)
 
-		var subPath string
-
-		switch r.criKind {
-		case clabernetesconstants.KubernetesCRIContainerd:
-			path = clabernetesconstants.KubernetesCRISockContainerdPath
-
-			subPath = clabernetesconstants.KubernetesCRISockContainerd
-		default:
-			r.log.Warnf(
-				"image pull through mode is auto or always but cri kind is not containerd!"+
-					" got cri kind %q",
-				r.criKind,
-			)
-		}
-
-		if path != "" && subPath != "" {
-			volumes = append(
-				volumes,
-				k8scorev1.Volume{
-					Name: "cri-sock",
-					VolumeSource: k8scorev1.VolumeSource{
-						HostPath: &k8scorev1.HostPathVolumeSource{
-							Path: path,
-							Type: clabernetesutil.ToPointer(k8scorev1.HostPathType("")),
-						},
+	if criPath != "" && criSubPath != "" {
+		volumes = append(
+			volumes,
+			k8scorev1.Volume{
+				Name: "cri-sock",
+				VolumeSource: k8scorev1.VolumeSource{
+					HostPath: &k8scorev1.HostPathVolumeSource{
+						Path: criPath,
+						Type: clabernetesutil.ToPointer(k8scorev1.HostPathType("")),
 					},
 				},
-			)
+			},
+		)
 
-			volumeMountsFromCommonSpec = append(
-				volumeMountsFromCommonSpec,
-				k8scorev1.VolumeMount{
-					Name:     "cri-sock",
-					ReadOnly: true,
-					MountPath: fmt.Sprintf(
-						"%s/%s",
-						clabernetesconstants.LauncherCRISockPath,
-						subPath,
-					),
-					SubPath: subPath,
-				},
-			)
-		}
+		volumeMountsFromCommonSpec = append(
+			volumeMountsFromCommonSpec,
+			k8scorev1.VolumeMount{
+				Name:     "cri-sock",
+				ReadOnly: true,
+				MountPath: fmt.Sprintf(
+					"%s/%s",
+					clabernetesconstants.LauncherCRISockPath,
+					criSubPath,
+				),
+				SubPath: criSubPath,
+			},
+		)
 	}
 
 	volumesFromConfigMaps := make([]clabernetesapisv1alpha1.FileFromConfigMap, 0)
@@ -286,6 +265,51 @@ func (r *DeploymentReconciler) renderDeploymentVolumes(
 	return volumeMountsFromCommonSpec
 }
 
+func (r *DeploymentReconciler) renderDeploymentVolumesGetCRISockPath(
+	owningTopology *clabernetesapisv1alpha1.Topology,
+) (path, subPath string) {
+	if owningTopology.Spec.ImagePull.PullThroughOverride == clabernetesconstants.ImagePullThroughModeNever { //nolint:lll
+		// obviously the topology is set to *never*, so nothing to do...
+		return path, subPath
+	}
+
+	if owningTopology.Spec.ImagePull.PullThroughOverride == "" && r.configManagerGetter().
+		GetImagePullThroughMode() == clabernetesconstants.ImagePullThroughModeNever {
+		// our specific topology is setting is unset, so we default to the global value, if that
+		// is never then we are obviously done here
+		return path, subPath
+	}
+
+	criSockOverrideFullPath := r.configManagerGetter().GetImagePullCriSockOverride()
+	if criSockOverrideFullPath != "" {
+		path, subPath = filepath.Split(criSockOverrideFullPath)
+
+		if path == "" {
+			r.log.Warn(
+				"image pull cri path override is set, but failed to parse path/subpath," +
+					" will skip mounting cri sock",
+			)
+
+			return path, subPath
+		}
+	} else {
+		switch r.criKind {
+		case clabernetesconstants.KubernetesCRIContainerd:
+			path = clabernetesconstants.KubernetesCRISockContainerdPath
+
+			subPath = clabernetesconstants.KubernetesCRISockContainerd
+		default:
+			r.log.Warnf(
+				"image pull through mode is auto or always but cri kind is not containerd!"+
+					" got cri kind %q",
+				r.criKind,
+			)
+		}
+	}
+
+	return path, subPath
+}
+
 func (r *DeploymentReconciler) renderDeploymentContainer(
 	deployment *k8sappsv1.Deployment,
 	nodeName,
@@ -368,6 +392,11 @@ func (r *DeploymentReconciler) renderDeploymentContainerEnv(
 		imagePullThroughMode = r.configManagerGetter().GetImagePullThroughMode()
 	}
 
+	criKind := r.configManagerGetter().GetImagePullCriKindOverride()
+	if criKind == "" {
+		criKind = r.criKind
+	}
+
 	envs := []k8scorev1.EnvVar{
 		{
 			Name: clabernetesconstants.NodeNameEnv,
@@ -406,7 +435,7 @@ func (r *DeploymentReconciler) renderDeploymentContainerEnv(
 		},
 		{
 			Name:  clabernetesconstants.LauncherCRIKindEnv,
-			Value: r.criKind,
+			Value: criKind,
 		},
 		{
 			Name:  clabernetesconstants.LauncherImagePullThroughModeEnv,