From c48caf8590bbb6a03367895c285ce440fda3f23a Mon Sep 17 00:00:00 2001 From: Carl Montanari Date: Sun, 17 Dec 2023 14:48:39 -0800 Subject: [PATCH 1/2] feat: allow mounting docker daemon config from secret --- apis/v1alpha1/configspec.go | 10 + apis/v1alpha1/topologyspec.go | 6 + .../clabernetes.containerlab.dev_configs.yaml | 14 + ...abernetes.containerlab.dev_topologies.yaml | 8 + .../clabernetes.containerlab.dev_configs.yaml | 14 + ...abernetes.containerlab.dev_topologies.yaml | 8 + config/fake.go | 4 + config/get.go | 7 + config/manager.go | 2 + controllers/topology/deployment.go | 31 +++ controllers/topology/deployment_test.go | 4 +- .../render-deployment/docker-daemon.json | 249 ++++++++++++++++++ generated/openapi/openapi_generated.go | 14 + launcher/docker.go | 19 +- 14 files changed, 387 insertions(+), 3 deletions(-) create mode 100755 controllers/topology/test-fixtures/golden/deployment/render-deployment/docker-daemon.json diff --git a/apis/v1alpha1/configspec.go b/apis/v1alpha1/configspec.go index 50894f19..24702e94 100644 --- a/apis/v1alpha1/configspec.go +++ b/apis/v1alpha1/configspec.go @@ -91,4 +91,14 @@ type ConfigImagePull struct { // +kubebuilder:validation:Enum=containerd // +optional CRIKindOverride string `json:"criKindOverride,omitempty"` + // DockerDaemonConfig allows for setting a default docker daemon config for launcher pods + // with the specified secret. The secret *must be present in the namespace of any given + // topology* -- so if you are configuring this at the "global config" level, ensure that you are + // deploying topologies into a specific namespace, or have ensured there is a secret of the + // given name in every namespace you wish to deploy a topology to. When set, insecure registries + // config option is ignored as it is assumed you are handling that in the given docker config. + // Note that the secret *must* contain a key "daemon.json" -- as this secret will be mounted to + // /etc/docker and docker will be expecting the config at /etc/docker/daemon.json. + // +optional + DockerDaemonConfig string `json:"dockerDaemonConfig,omitempty"` } diff --git a/apis/v1alpha1/topologyspec.go b/apis/v1alpha1/topologyspec.go index d0b4b2ca..60cb78fa 100644 --- a/apis/v1alpha1/topologyspec.go +++ b/apis/v1alpha1/topologyspec.go @@ -187,4 +187,10 @@ type ImagePull struct { // +listType=set // +optional PullSecrets []string `json:"pullSecrets"` + // DockerDaemonConfig allows for setting the docker daemon config for all launchers in this + // topology. The secret *must be present in the namespace of this topology*. The secret *must* + // contain a key "daemon.json" -- as this secret will be mounted to /etc/docker and docker will + // be expecting the config at /etc/docker/daemon.json. + // +optional + DockerDaemonConfig string `json:"dockerDaemonConfig,omitempty"` } diff --git a/assets/crd/clabernetes.containerlab.dev_configs.yaml b/assets/crd/clabernetes.containerlab.dev_configs.yaml index ccdc8180..440ca324 100644 --- a/assets/crd/clabernetes.containerlab.dev_configs.yaml +++ b/assets/crd/clabernetes.containerlab.dev_configs.yaml @@ -226,6 +226,20 @@ spec: maybe crio support will be added. pattern: (.*containerd\.sock) type: string + dockerDaemonConfig: + description: DockerDaemonConfig allows for setting a default docker + daemon config for launcher pods with the specified secret. The + secret *must be present in the namespace of any given topology* + -- so if you are configuring this at the "global config" level, + ensure that you are deploying topologies into a specific namespace, + or have ensured there is a secret of the given name in every + namespace you wish to deploy a topology to. When set, insecure + registries config option is ignored as it is assumed you are + handling that in the given docker config. Note that the secret + *must* contain a key "daemon.json" -- as this secret will be + mounted to /etc/docker and docker will be expecting the config + at /etc/docker/daemon.json. + type: string pullThroughOverride: description: PullThroughOverride allows for overriding the image pull through mode for this particular topology. diff --git a/assets/crd/clabernetes.containerlab.dev_topologies.yaml b/assets/crd/clabernetes.containerlab.dev_topologies.yaml index d982bd51..8cf79db2 100644 --- a/assets/crd/clabernetes.containerlab.dev_topologies.yaml +++ b/assets/crd/clabernetes.containerlab.dev_topologies.yaml @@ -294,6 +294,14 @@ spec: description: ImagePull holds configurations relevant to how clabernetes launcher pods handle pulling images. properties: + dockerDaemonConfig: + description: DockerDaemonConfig allows for setting the docker + daemon config for all launchers in this topology. The secret + *must be present in the namespace of this topology*. The secret + *must* contain a key "daemon.json" -- as this secret will be + mounted to /etc/docker and docker will be expecting the config + at /etc/docker/daemon.json. + type: string insecureRegistries: description: InsecureRegistries is a slice of strings of insecure registries to configure in the launcher pods. diff --git a/charts/clabernetes/crds/clabernetes.containerlab.dev_configs.yaml b/charts/clabernetes/crds/clabernetes.containerlab.dev_configs.yaml index ccdc8180..440ca324 100644 --- a/charts/clabernetes/crds/clabernetes.containerlab.dev_configs.yaml +++ b/charts/clabernetes/crds/clabernetes.containerlab.dev_configs.yaml @@ -226,6 +226,20 @@ spec: maybe crio support will be added. pattern: (.*containerd\.sock) type: string + dockerDaemonConfig: + description: DockerDaemonConfig allows for setting a default docker + daemon config for launcher pods with the specified secret. The + secret *must be present in the namespace of any given topology* + -- so if you are configuring this at the "global config" level, + ensure that you are deploying topologies into a specific namespace, + or have ensured there is a secret of the given name in every + namespace you wish to deploy a topology to. When set, insecure + registries config option is ignored as it is assumed you are + handling that in the given docker config. Note that the secret + *must* contain a key "daemon.json" -- as this secret will be + mounted to /etc/docker and docker will be expecting the config + at /etc/docker/daemon.json. + type: string pullThroughOverride: description: PullThroughOverride allows for overriding the image pull through mode for this particular topology. diff --git a/charts/clabernetes/crds/clabernetes.containerlab.dev_topologies.yaml b/charts/clabernetes/crds/clabernetes.containerlab.dev_topologies.yaml index d982bd51..8cf79db2 100644 --- a/charts/clabernetes/crds/clabernetes.containerlab.dev_topologies.yaml +++ b/charts/clabernetes/crds/clabernetes.containerlab.dev_topologies.yaml @@ -294,6 +294,14 @@ spec: description: ImagePull holds configurations relevant to how clabernetes launcher pods handle pulling images. properties: + dockerDaemonConfig: + description: DockerDaemonConfig allows for setting the docker + daemon config for all launchers in this topology. The secret + *must be present in the namespace of this topology*. The secret + *must* contain a key "daemon.json" -- as this secret will be + mounted to /etc/docker and docker will be expecting the config + at /etc/docker/daemon.json. + type: string insecureRegistries: description: InsecureRegistries is a slice of strings of insecure registries to configure in the launcher pods. diff --git a/config/fake.go b/config/fake.go index 9a773918..b8490c14 100644 --- a/config/fake.go +++ b/config/fake.go @@ -66,6 +66,10 @@ func (f fakeManager) GetImagePullCriKindOverride() string { return "" } +func (f fakeManager) GetDockerDaemonConfig() string { + return "" +} + func (f fakeManager) GetLauncherImagePullPolicy() string { return clabernetesconstants.KubernetesImagePullIfNotPresent } diff --git a/config/get.go b/config/get.go index 784316f3..e66bd0fe 100644 --- a/config/get.go +++ b/config/get.go @@ -106,6 +106,13 @@ func (m *manager) GetImagePullCriKindOverride() string { return m.config.ImagePull.CRIKindOverride } +func (m *manager) GetDockerDaemonConfig() string { + m.lock.RLock() + defer m.lock.RUnlock() + + return m.config.ImagePull.DockerDaemonConfig +} + func (m *manager) GetLauncherImage() string { m.lock.RLock() defer m.lock.RUnlock() diff --git a/config/manager.go b/config/manager.go index c98f28bc..639765e3 100644 --- a/config/manager.go +++ b/config/manager.go @@ -130,6 +130,8 @@ type Manager interface { GetImagePullCriSockOverride() string // GetImagePullCriKindOverride returns the cri kind override. GetImagePullCriKindOverride() string + // GetDockerDaemonConfig returns the secret name to mount in /etc/docker. + GetDockerDaemonConfig() string // GetLauncherImage returns the global default launcher image. GetLauncherImage() string // GetLauncherImagePullPolicy returns the global default launcher image pull policy. diff --git a/controllers/topology/deployment.go b/controllers/topology/deployment.go index 6bec60b4..e8680f85 100644 --- a/controllers/topology/deployment.go +++ b/controllers/topology/deployment.go @@ -216,6 +216,37 @@ func (r *DeploymentReconciler) renderDeploymentVolumes( ) } + dockerDaemonConfigSecret := owningTopology.Spec.ImagePull.DockerDaemonConfig + if dockerDaemonConfigSecret == "" { + dockerDaemonConfigSecret = r.configManagerGetter().GetDockerDaemonConfig() + } + + if dockerDaemonConfigSecret != "" { + volumes = append( + volumes, + k8scorev1.Volume{ + Name: "docker-daemon-config", + VolumeSource: k8scorev1.VolumeSource{ + Secret: &k8scorev1.SecretVolumeSource{ + SecretName: dockerDaemonConfigSecret, + DefaultMode: clabernetesutil.ToPointer( + int32(clabernetesconstants.PermissionsEveryoneRead), + ), + }, + }, + }, + ) + + volumeMountsFromCommonSpec = append( + volumeMountsFromCommonSpec, + k8scorev1.VolumeMount{ + Name: "docker-daemon-config", + ReadOnly: true, + MountPath: "/etc/docker", + }, + ) + } + volumesFromConfigMaps := make([]clabernetesapisv1alpha1.FileFromConfigMap, 0) volumesFromConfigMaps = append( diff --git a/controllers/topology/deployment_test.go b/controllers/topology/deployment_test.go index f412929d..47f8f1de 100644 --- a/controllers/topology/deployment_test.go +++ b/controllers/topology/deployment_test.go @@ -374,7 +374,7 @@ func TestRenderDeployment(t *testing.T) { nodeName: "srl1", }, { - name: "insecure-registries", + name: "docker-daemon", owningTopology: &clabernetesapisv1alpha1.Topology{ ObjectMeta: metav1.ObjectMeta{ Name: "render-deployment-test", @@ -382,7 +382,7 @@ func TestRenderDeployment(t *testing.T) { }, Spec: clabernetesapisv1alpha1.TopologySpec{ ImagePull: clabernetesapisv1alpha1.ImagePull{ - InsecureRegistries: []string{"1.2.3.4", "potato.com"}, + DockerDaemonConfig: "sneakydockerdaemonconfig", }, Definition: clabernetesapisv1alpha1.Definition{ Containerlab: `--- diff --git a/controllers/topology/test-fixtures/golden/deployment/render-deployment/docker-daemon.json b/controllers/topology/test-fixtures/golden/deployment/render-deployment/docker-daemon.json new file mode 100755 index 00000000..4d9a68b1 --- /dev/null +++ b/controllers/topology/test-fixtures/golden/deployment/render-deployment/docker-daemon.json @@ -0,0 +1,249 @@ +{ + "metadata": { + "name": "render-deployment-test-srl1", + "namespace": "clabernetes", + "creationTimestamp": null, + "labels": { + "clabernetes/app": "clabernetes", + "clabernetes/name": "render-deployment-test-srl1", + "clabernetes/topologyNode": "srl1", + "clabernetes/topologyOwner": "render-deployment-test" + }, + "annotations": { + "container.apparmor.security.beta.kubernetes.io/srl1": "unconfined" + } + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "clabernetes/app": "clabernetes", + "clabernetes/name": "render-deployment-test-srl1", + "clabernetes/topologyNode": "srl1", + "clabernetes/topologyOwner": "render-deployment-test" + } + }, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "clabernetes/app": "clabernetes", + "clabernetes/name": "render-deployment-test-srl1", + "clabernetes/topologyNode": "srl1", + "clabernetes/topologyOwner": "render-deployment-test" + }, + "annotations": { + "container.apparmor.security.beta.kubernetes.io/srl1": "unconfined" + } + }, + "spec": { + "volumes": [ + { + "name": "render-deployment-test-config", + "configMap": { + "name": "render-deployment-test", + "defaultMode": 493 + } + }, + { + "name": "docker-daemon-config", + "secret": { + "secretName": "sneakydockerdaemonconfig", + "defaultMode": 493 + } + }, + { + "name": "dev-kvm", + "hostPath": { + "path": "/dev/kvm", + "type": "" + } + }, + { + "name": "dev-fuse", + "hostPath": { + "path": "/dev/fuse", + "type": "" + } + }, + { + "name": "dev-net-tun", + "hostPath": { + "path": "/dev/net/tun", + "type": "" + } + } + ], + "containers": [ + { + "name": "srl1", + "image": "ghcr.io/srl-labs/clabernetes/clabernetes-launcher:latest", + "command": [ + "/clabernetes/manager", + "launch" + ], + "workingDir": "/clabernetes", + "ports": [ + { + "name": "vxlan", + "containerPort": 14789, + "protocol": "UDP" + } + ], + "env": [ + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "POD_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.name" + } + } + }, + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "APP_NAME", + "value": "clabernetes" + }, + { + "name": "MANAGER_NAMESPACE", + "value": "clabernetes" + }, + { + "name": "LAUNCHER_CRI_KIND" + }, + { + "name": "LAUNCHER_IMAGE_PULL_THROUGH_MODE", + "value": "auto" + }, + { + "name": "LAUNCHER_LOGGER_LEVEL", + "value": "info" + }, + { + "name": "LAUNCHER_TOPOLOGY_NAME", + "value": "render-deployment-test" + }, + { + "name": "LAUNCHER_NODE_NAME", + "value": "srl1" + }, + { + "name": "LAUNCHER_NODE_IMAGE", + "value": "ghcr.io/nokia/srlinux" + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "render-deployment-test-config", + "readOnly": true, + "mountPath": "/clabernetes/topo.clab.yaml", + "subPath": "srl1" + }, + { + "name": "render-deployment-test-config", + "readOnly": true, + "mountPath": "/clabernetes/tunnels.yaml", + "subPath": "srl1-tunnels" + }, + { + "name": "render-deployment-test-config", + "readOnly": true, + "mountPath": "/clabernetes/files-from-url.yaml", + "subPath": "srl1-files-from-url" + }, + { + "name": "render-deployment-test-config", + "readOnly": true, + "mountPath": "/clabernetes/configured-pull-secrets.yaml", + "subPath": "configured-pull-secrets" + }, + { + "name": "docker-daemon-config", + "readOnly": true, + "mountPath": "/etc/docker" + }, + { + "name": "dev-kvm", + "readOnly": true, + "mountPath": "/dev/kvm" + }, + { + "name": "dev-fuse", + "readOnly": true, + "mountPath": "/dev/fuse" + }, + { + "name": "dev-net-tun", + "readOnly": true, + "mountPath": "/dev/net/tun" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": { + "add": [ + "CHOWN", + "DAC_OVERRIDE", + "FSETID", + "FOWNER", + "MKNOD", + "NET_RAW", + "SETGID", + "SETUID", + "SETFCAP", + "SETPCAP", + "NET_BIND_SERVICE", + "SYS_CHROOT", + "KILL", + "AUDIT_WRITE", + "NET_ADMIN", + "SYS_ADMIN", + "SYS_RESOURCE", + "LINUX_IMMUTABLE", + "SYS_BOOT", + "SYS_TIME", + "SYS_MODULE", + "SYS_RAWIO", + "SYS_PTRACE", + "SYS_NICE", + "IPC_LOCK" + ] + }, + "privileged": false, + "runAsUser": 0 + } + } + ], + "restartPolicy": "Always", + "serviceAccountName": "clabernetes-launcher-service-account", + "hostname": "srl1" + } + }, + "strategy": { + "type": "Recreate" + }, + "revisionHistoryLimit": 0 + }, + "status": {} +} \ No newline at end of file diff --git a/generated/openapi/openapi_generated.go b/generated/openapi/openapi_generated.go index de6df023..b9a7581f 100644 --- a/generated/openapi/openapi_generated.go +++ b/generated/openapi/openapi_generated.go @@ -281,6 +281,13 @@ func schema_srl_labs_clabernetes_apis_v1alpha1_ConfigImagePull( Format: "", }, }, + "dockerDaemonConfig": { + SchemaProps: spec.SchemaProps{ + Description: "DockerDaemonConfig allows for setting a default docker daemon config for launcher pods with the specified secret. The secret *must be present in the namespace of any given topology* -- so if you are configuring this at the \"global config\" level, ensure that you are deploying topologies into a specific namespace, or have ensured there is a secret of the given name in every namespace you wish to deploy a topology to. When set, insecure registries config option is ignored as it is assumed you are handling that in the given docker config. Note that the secret *must* contain a key \"daemon.json\" -- as this secret will be mounted to /etc/docker and docker will be expecting the config at /etc/docker/daemon.json.", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, @@ -830,6 +837,13 @@ func schema_srl_labs_clabernetes_apis_v1alpha1_ImagePull( }, }, }, + "dockerDaemonConfig": { + SchemaProps: spec.SchemaProps{ + Description: "DockerDaemonConfig allows for setting the docker daemon config for all launchers in this topology. The secret *must be present in the namespace of this topology*. The secret *must* contain a key \"daemon.json\" -- as this secret will be mounted to /etc/docker and docker will be expecting the config at /etc/docker/daemon.json.", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, diff --git a/launcher/docker.go b/launcher/docker.go index 51a26de0..18a4a703 100644 --- a/launcher/docker.go +++ b/launcher/docker.go @@ -14,7 +14,24 @@ import ( claberneteserrors "github.com/srl-labs/clabernetes/errors" ) +const ( + dockerDaemonConfig = "/etc/docker/daemon.json" +) + +func daemonConfigExists() bool { + _, err := os.Stat(dockerDaemonConfig) + + return err == nil +} + func (c *clabernetes) handleInsecureRegistries() error { + if daemonConfigExists() { + // user has provided a docker daemon config, we ignore insecure + c.logger.Infof("%q exists, skipping insecure registries", dockerDaemonConfig) + + return nil + } + insecureRegistries := os.Getenv(clabernetesconstants.LauncherInsecureRegistries) if insecureRegistries == "" { @@ -48,7 +65,7 @@ func (c *clabernetes) handleInsecureRegistries() error { } err = os.WriteFile( - "/etc/docker/daemon.json", + dockerDaemonConfig, rendered.Bytes(), clabernetesconstants.PermissionsEveryoneRead, ) From bb59c75a8e83bf72b1797e35720249a64ab84b19 Mon Sep 17 00:00:00 2001 From: Carl Montanari Date: Sun, 17 Dec 2023 14:53:02 -0800 Subject: [PATCH 2/2] fix: dont delete insecure registry test --- controllers/topology/deployment_test.go | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/controllers/topology/deployment_test.go b/controllers/topology/deployment_test.go index 47f8f1de..f8a4766e 100644 --- a/controllers/topology/deployment_test.go +++ b/controllers/topology/deployment_test.go @@ -373,6 +373,66 @@ func TestRenderDeployment(t *testing.T) { }, nodeName: "srl1", }, + { + name: "insecure-registries", + owningTopology: &clabernetesapisv1alpha1.Topology{ + ObjectMeta: metav1.ObjectMeta{ + Name: "render-deployment-test", + Namespace: "clabernetes", + }, + Spec: clabernetesapisv1alpha1.TopologySpec{ + ImagePull: clabernetesapisv1alpha1.ImagePull{ + InsecureRegistries: []string{"1.2.3.4", "potato.com"}, + }, + Definition: clabernetesapisv1alpha1.Definition{ + Containerlab: `--- + name: test + topology: + nodes: + srl1: + kind: srl + image: ghcr.io/nokia/srlinux + `, + }, + }, + }, + clabernetesConfigs: map[string]*clabernetesutilcontainerlab.Config{ + "srl1": { + Name: "srl1", + Prefix: clabernetesutil.ToPointer(""), + Topology: &clabernetesutilcontainerlab.Topology{ + Defaults: &clabernetesutilcontainerlab.NodeDefinition{ + Ports: []string{ + "21022:22/tcp", + "21023:23/tcp", + "21161:161/udp", + "33333:57400/tcp", + "60000:21/tcp", + "60001:80/tcp", + "60002:443/tcp", + "60003:830/tcp", + "60004:5000/tcp", + "60005:5900/tcp", + "60006:6030/tcp", + "60007:9339/tcp", + "60008:9340/tcp", + "60009:9559/tcp", + }, + }, + Kinds: nil, + Nodes: map[string]*clabernetesutilcontainerlab.NodeDefinition{ + "srl1": { + Kind: "srl", + Image: "ghcr.io/nokia/srlinux", + }, + }, + Links: nil, + }, + Debug: false, + }, + }, + nodeName: "srl1", + }, { name: "docker-daemon", owningTopology: &clabernetesapisv1alpha1.Topology{