diff --git a/pkg/karmadactl/addons/descheduler/manifests.go b/pkg/karmadactl/addons/descheduler/manifests.go index 59e5755741ae..4560da0f3c69 100644 --- a/pkg/karmadactl/addons/descheduler/manifests.go +++ b/pkg/karmadactl/addons/descheduler/manifests.go @@ -44,7 +44,7 @@ spec: imagePullPolicy: IfNotPresent command: - /bin/karmada-descheduler - - --kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config - --metrics-bind-address=0.0.0.0:8080 - --health-probe-bind-address=0.0.0.0:10358 - --leader-elect-resource-namespace={{ .Namespace }} @@ -66,19 +66,18 @@ spec: name: metrics protocol: TCP volumeMounts: + - name: karmada-config + mountPath: /etc/karmada/config - name: k8s-certs mountPath: /etc/karmada/pki readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig volumes: + - name: karmada-config + secret: + secretName: karmada-descheduler-config - name: k8s-certs secret: secretName: karmada-cert - - name: kubeconfig - secret: - secretName: kubeconfig ` // DeploymentReplace is a struct to help to concrete diff --git a/pkg/karmadactl/addons/init/enable_option.go b/pkg/karmadactl/addons/init/enable_option.go index 206a44608a69..ffe6fd35ae98 100644 --- a/pkg/karmadactl/addons/init/enable_option.go +++ b/pkg/karmadactl/addons/init/enable_option.go @@ -26,7 +26,7 @@ import ( "k8s.io/klog/v2" "k8s.io/utils/strings/slices" - cmdinit "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes" + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" "github.com/karmada-io/karmada/pkg/util/names" "github.com/karmada-io/karmada/pkg/version" @@ -149,10 +149,17 @@ func (o *CommandAddonsEnableOption) Validate(args []string) error { } secretClient := o.KubeClientSet.CoreV1().Secrets(o.Namespace) - _, err = secretClient.Get(context.TODO(), cmdinit.KubeConfigSecretAndMountName, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return fmt.Errorf("secrets `kubeconfig` is not found in namespace %s, please execute karmadactl init to deploy karmada first", o.Namespace) + for _, addon := range getEnableAddons(args) { + if addon.Name == names.KarmadaSchedulerEstimatorComponentName { + // estimator not rely on karmada config secret + continue + } + karmadaConfigSecretName := options.KarmadaConfigName(addon.Name) + _, err = secretClient.Get(context.TODO(), karmadaConfigSecretName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return fmt.Errorf("secrets `%s` is not found in namespace %s, please execute karmadactl init to deploy karmada first", karmadaConfigSecretName, o.Namespace) + } } } @@ -188,21 +195,8 @@ func (o *CommandAddonsEnableOption) Validate(args []string) error { // Run start enable Karmada addons func (o *CommandAddonsEnableOption) Run(args []string) error { - var enableAddons = map[string]*Addon{} - - // collect enabled addons - for _, item := range args { - if item == "all" { - enableAddons = Addons - break - } - if addon := Addons[item]; addon != nil { - enableAddons[item] = addon - } - } - // enable addons - for name, addon := range enableAddons { + for name, addon := range getEnableAddons(args) { klog.Infof("Start to enable addon %s", name) if err := addon.Enable(o); err != nil { klog.Errorf("Install addon %s failed", name) @@ -230,3 +224,20 @@ func validAddonNames(addonNames []string) error { } return nil } + +func getEnableAddons(addonNames []string) map[string]*Addon { + var enableAddons = map[string]*Addon{} + + // collect enabled addons + for _, item := range addonNames { + if item == "all" { + enableAddons = Addons + break + } + if addon := Addons[item]; addon != nil { + enableAddons[item] = addon + } + } + + return enableAddons +} diff --git a/pkg/karmadactl/addons/metricsadapter/manifests.go b/pkg/karmadactl/addons/metricsadapter/manifests.go index 930e69a3db94..05d424a9619c 100644 --- a/pkg/karmadactl/addons/metricsadapter/manifests.go +++ b/pkg/karmadactl/addons/metricsadapter/manifests.go @@ -43,19 +43,12 @@ spec: - name: karmada-metrics-adapter image: {{ .Image }} imagePullPolicy: IfNotPresent - volumeMounts: - - name: k8s-certs - mountPath: /etc/karmada/pki - readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig command: - /bin/karmada-metrics-adapter - - --kubeconfig=/etc/kubeconfig - --metrics-bind-address=:8080 - - --authentication-kubeconfig=/etc/kubeconfig - - --authorization-kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config + - --authentication-kubeconfig=/etc/karmada/config/karmada.config + - --authorization-kubeconfig=/etc/karmada/config/karmada.config - --client-ca-file=/etc/karmada/pki/ca.crt - --tls-cert-file=/etc/karmada/pki/karmada.crt - --tls-private-key-file=/etc/karmada/pki/karmada.key @@ -84,13 +77,19 @@ spec: resources: requests: cpu: 100m + volumeMounts: + - name: karmada-config + mountPath: /etc/karmada/config + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true volumes: + - name: karmada-config + secret: + secretName: karmada-metrics-adapter-config - name: k8s-certs secret: secretName: karmada-cert - - name: kubeconfig - secret: - secretName: kubeconfig ` karmadaMetricsAdapterService = ` diff --git a/pkg/karmadactl/addons/search/manifests.go b/pkg/karmadactl/addons/search/manifests.go index 23e84ed65cde..fa3ee5915fc6 100644 --- a/pkg/karmadactl/addons/search/manifests.go +++ b/pkg/karmadactl/addons/search/manifests.go @@ -43,18 +43,11 @@ spec: - name: karmada-search image: {{ .Image }} imagePullPolicy: IfNotPresent - volumeMounts: - - name: k8s-certs - mountPath: /etc/karmada/pki - readOnly: true - - name: kubeconfig - subPath: kubeconfig - mountPath: /etc/kubeconfig command: - /bin/karmada-search - - --kubeconfig=/etc/kubeconfig - - --authentication-kubeconfig=/etc/kubeconfig - - --authorization-kubeconfig=/etc/kubeconfig + - --kubeconfig=/etc/karmada/config/karmada.config + - --authentication-kubeconfig=/etc/karmada/config/karmada.config + - --authorization-kubeconfig=/etc/karmada/config/karmada.config - --etcd-servers={{ .ETCDSevers }} - --etcd-cafile=/etc/karmada/pki/etcd-ca.crt - --etcd-certfile=/etc/karmada/pki/etcd-client.crt @@ -78,13 +71,19 @@ spec: resources: requests: cpu: 100m + volumeMounts: + - name: karmada-config + mountPath: /etc/karmada/config + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true volumes: + - name: karmada-config + secret: + secretName: karmada-search-config - name: k8s-certs secret: secretName: karmada-cert - - name: kubeconfig - secret: - secretName: kubeconfig ` karmadaSearchService = ` diff --git a/pkg/karmadactl/cmdinit/kubernetes/deploy.go b/pkg/karmadactl/cmdinit/kubernetes/deploy.go index 7e83b6dba831..f976c6c261d2 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deploy.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deploy.go @@ -44,6 +44,7 @@ import ( globaloptions "github.com/karmada-io/karmada/pkg/karmadactl/options" "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" + "github.com/karmada-io/karmada/pkg/util/names" "github.com/karmada-io/karmada/pkg/util/validation" "github.com/karmada-io/karmada/pkg/version" ) @@ -65,6 +66,17 @@ var ( options.FrontProxyClientCertAndKeyName, } + karmadaConfigList = []string{ + options.KarmadaConfigName(names.KarmadaAggregatedAPIServerComponentName), + options.KarmadaConfigName(names.KarmadaControllerManagerComponentName), + options.KarmadaConfigName(names.KubeControllerManagerComponentName), + options.KarmadaConfigName(names.KarmadaSchedulerComponentName), + options.KarmadaConfigName(names.KarmadaDeschedulerComponentName), + options.KarmadaConfigName(names.KarmadaMetricsAdapterComponentName), + options.KarmadaConfigName(names.KarmadaSearchComponentName), + options.KarmadaConfigName(names.KarmadaWebhookComponentName), + } + emptyByteSlice = make([]byte, 0) externalEtcdCertSpecialization = map[string]func(*CommandInitOption) ([]byte, []byte, error){ options.EtcdCaCertAndKeyName: func(option *CommandInitOption) (cert, key []byte, err error) { @@ -414,7 +426,7 @@ func (i *CommandInitOption) prepareCRD() error { } func (i *CommandInitOption) createCertsSecrets() error { - // Create kubeconfig Secret + // Create karmada-config Secret karmadaServerURL := fmt.Sprintf("https://%s.%s.svc.%s:%v", karmadaAPIServerDeploymentAndServiceName, i.Namespace, i.HostClusterDomain, karmadaAPIServerContainerPort) config := utils.CreateWithCerts(karmadaServerURL, options.UserName, options.UserName, i.CertAndKeyFileData[fmt.Sprintf("%s.crt", globaloptions.CaCertAndKeyName)], i.CertAndKeyFileData[fmt.Sprintf("%s.key", options.KarmadaCertAndKeyName)], i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.KarmadaCertAndKeyName)]) @@ -423,10 +435,13 @@ func (i *CommandInitOption) createCertsSecrets() error { return fmt.Errorf("failure while serializing admin kubeConfig. %v", err) } - kubeConfigSecret := i.SecretFromSpec(KubeConfigSecretAndMountName, corev1.SecretTypeOpaque, map[string]string{KubeConfigSecretAndMountName: string(configBytes)}) - if err = util.CreateOrUpdateSecret(i.KubeClientSet, kubeConfigSecret); err != nil { - return err + for _, karmadaConfigSecretName := range karmadaConfigList { + karmadaConfigSecret := i.SecretFromSpec(karmadaConfigSecretName, corev1.SecretTypeOpaque, map[string]string{options.KarmadaConfigFieldName: string(configBytes)}) + if err = util.CreateOrUpdateSecret(i.KubeClientSet, karmadaConfigSecret); err != nil { + return err + } } + // Create certs Secret etcdCert := map[string]string{ fmt.Sprintf("%s.crt", options.EtcdCaCertAndKeyName): string(i.CertAndKeyFileData[fmt.Sprintf("%s.crt", options.EtcdCaCertAndKeyName)]), diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index 1578dca793f0..67ab6df49027 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -18,6 +18,7 @@ package kubernetes import ( "fmt" + "path/filepath" "strings" appsv1 "k8s.io/api/apps/v1" @@ -39,10 +40,9 @@ const ( metricsPortName = "metrics" defaultMetricsPort = 8080 - // KubeConfigSecretAndMountName is the secret and volume mount name of karmada kubeconfig - KubeConfigSecretAndMountName = "kubeconfig" karmadaCertsVolumeMountPath = "/etc/karmada/pki" - kubeConfigContainerMountPath = "/etc/kubeconfig" + karmadaConfigVolumeName = "karmada-config" + karmadaConfigVolumeMountPath = "/etc/karmada/config" karmadaAPIServerDeploymentAndServiceName = "karmada-apiserver" karmadaAPIServerContainerPort = 5443 serviceClusterIP = "10.96.0.0/12" @@ -304,8 +304,9 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 Command: []string{ "kube-controller-manager", "--allocate-node-cidrs=true", - "--authentication-kubeconfig=/etc/kubeconfig", - "--authorization-kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), + fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), + fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), "--bind-address=0.0.0.0", fmt.Sprintf("--client-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), "--cluster-cidr=10.244.0.0/16", @@ -313,7 +314,6 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 fmt.Sprintf("--cluster-signing-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), fmt.Sprintf("--cluster-signing-key-file=%s/%s.key", karmadaCertsVolumeMountPath, globaloptions.CaCertAndKeyName), "--controllers=namespace,garbagecollector,serviceaccount-token,ttl-after-finished,bootstrapsigner,tokencleaner,csrcleaner,csrsigning,clusterrole-aggregation", - "--kubeconfig=/etc/kubeconfig", "--leader-elect=true", fmt.Sprintf("--leader-elect-resource-namespace=%s", i.Namespace), "--node-cidr-mask-size=24", @@ -333,10 +333,9 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 }, VolumeMounts: []corev1.VolumeMount{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, ReadOnly: true, - MountPath: kubeConfigContainerMountPath, - SubPath: KubeConfigSecretAndMountName, + MountPath: karmadaConfigVolumeMountPath, }, { Name: globaloptions.KarmadaCertsName, @@ -348,10 +347,10 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 }, Volumes: []corev1.Volume{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: KubeConfigSecretAndMountName, + SecretName: options.KarmadaConfigName(names.KubeControllerManagerComponentName), }, }, }, @@ -449,7 +448,7 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment ImagePullPolicy: corev1.PullPolicy(i.ImagePullPolicy), Command: []string{ "/bin/karmada-scheduler", - "--kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), "--metrics-bind-address=0.0.0.0:8080", "--health-probe-bind-address=0.0.0.0:10351", "--enable-scheduler-estimator=true", @@ -470,10 +469,9 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment }, VolumeMounts: []corev1.VolumeMount{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, ReadOnly: true, - MountPath: kubeConfigContainerMountPath, - SubPath: KubeConfigSecretAndMountName, + MountPath: karmadaConfigVolumeMountPath, }, { Name: globaloptions.KarmadaCertsName, @@ -485,10 +483,10 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment }, Volumes: []corev1.Volume{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: KubeConfigSecretAndMountName, + SecretName: options.KarmadaConfigName(names.KarmadaSchedulerComponentName), }, }, }, @@ -588,7 +586,7 @@ func (i *CommandInitOption) makeKarmadaControllerManagerDeployment() *appsv1.Dep ImagePullPolicy: corev1.PullPolicy(i.ImagePullPolicy), Command: []string{ "/bin/karmada-controller-manager", - "--kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), "--metrics-bind-address=:8080", "--health-probe-bind-address=0.0.0.0:10357", "--cluster-status-update-frequency=10s", @@ -610,20 +608,19 @@ func (i *CommandInitOption) makeKarmadaControllerManagerDeployment() *appsv1.Dep }, VolumeMounts: []corev1.VolumeMount{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, ReadOnly: true, - MountPath: kubeConfigContainerMountPath, - SubPath: KubeConfigSecretAndMountName, + MountPath: karmadaConfigVolumeMountPath, }, }, }, }, Volumes: []corev1.Volume{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: KubeConfigSecretAndMountName, + SecretName: options.KarmadaConfigName(names.KarmadaControllerManagerComponentName), }, }, }, @@ -712,7 +709,7 @@ func (i *CommandInitOption) makeKarmadaWebhookDeployment() *appsv1.Deployment { ImagePullPolicy: corev1.PullPolicy(i.ImagePullPolicy), Command: []string{ "/bin/karmada-webhook", - "--kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), "--bind-address=0.0.0.0", "--metrics-bind-address=:8080", fmt.Sprintf("--secure-port=%v", webhookTargetPort), @@ -733,10 +730,9 @@ func (i *CommandInitOption) makeKarmadaWebhookDeployment() *appsv1.Deployment { }, VolumeMounts: []corev1.VolumeMount{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, ReadOnly: true, - MountPath: kubeConfigContainerMountPath, - SubPath: KubeConfigSecretAndMountName, + MountPath: karmadaConfigVolumeMountPath, }, { Name: webhookCertsName, @@ -749,10 +745,10 @@ func (i *CommandInitOption) makeKarmadaWebhookDeployment() *appsv1.Deployment { }, Volumes: []corev1.Volume{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: KubeConfigSecretAndMountName, + SecretName: options.KarmadaConfigName(names.KarmadaWebhookComponentName), }, }, }, @@ -843,9 +839,9 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D } command := []string{ "/bin/karmada-aggregated-apiserver", - "--kubeconfig=/etc/kubeconfig", - "--authentication-kubeconfig=/etc/kubeconfig", - "--authorization-kubeconfig=/etc/kubeconfig", + fmt.Sprintf("--kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), + fmt.Sprintf("--authentication-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), + fmt.Sprintf("--authorization-kubeconfig=%s", filepath.Join(karmadaConfigVolumeMountPath, options.KarmadaConfigFieldName)), fmt.Sprintf("--etcd-servers=%s", etcdServers), fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName), fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), @@ -887,12 +883,18 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D Image: i.karmadaAggregatedAPIServerImage(), ImagePullPolicy: corev1.PullPolicy(i.ImagePullPolicy), Command: command, + ReadinessProbe: readinesProbe, + LivenessProbe: livenesProbe, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + }, VolumeMounts: []corev1.VolumeMount{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, ReadOnly: true, - MountPath: kubeConfigContainerMountPath, - SubPath: KubeConfigSecretAndMountName, + MountPath: karmadaConfigVolumeMountPath, }, { Name: globaloptions.KarmadaCertsName, @@ -900,21 +902,14 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D MountPath: karmadaCertsVolumeMountPath, }, }, - ReadinessProbe: readinesProbe, - LivenessProbe: livenesProbe, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - }, - }, }, }, Volumes: []corev1.Volume{ { - Name: KubeConfigSecretAndMountName, + Name: karmadaConfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: KubeConfigSecretAndMountName, + SecretName: options.KarmadaConfigName(names.KarmadaAggregatedAPIServerComponentName), }, }, }, diff --git a/pkg/karmadactl/cmdinit/options/global.go b/pkg/karmadactl/cmdinit/options/global.go index 9d132cea0583..fe52d12bf52c 100644 --- a/pkg/karmadactl/cmdinit/options/global.go +++ b/pkg/karmadactl/cmdinit/options/global.go @@ -39,4 +39,14 @@ const ( KarmadaKubeConfigName = "karmada-apiserver.config" // WaitComponentReadyTimeout wait component ready time WaitComponentReadyTimeout = 120 + + // KarmadaConfigSecretSuffix karmada config secret suffix + KarmadaConfigSecretSuffix = "-config" //nolint:gosec + // KarmadaConfigFieldName the field stores karmada config in karmada config secret + KarmadaConfigFieldName = "karmada.config" //nolint:gosec ) + +// KarmadaConfigName returns the name of karmada config secret +func KarmadaConfigName(component string) string { + return component + KarmadaConfigSecretSuffix +} diff --git a/pkg/util/names/names.go b/pkg/util/names/names.go index 82426b2e02a2..eb3b64907bd3 100644 --- a/pkg/util/names/names.go +++ b/pkg/util/names/names.go @@ -65,6 +65,9 @@ const ( // KarmadaControllerManagerComponentName is the name of the Karmada Controller Manager component. KarmadaControllerManagerComponentName = "karmada-controller-manager" + + // KubeControllerManagerComponentName is the name of the Kube Controller Manager component. + KubeControllerManagerComponentName = "kube-controller-manager" ) // ExecutionSpacePrefix is the prefix of execution space