From 04a909a257c19941528a37bbb45905afc26fac12 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Fri, 3 Feb 2017 17:10:53 -0500 Subject: [PATCH 1/4] Rename cgroups-per-qos flag to not be experimental --- cmd/kubelet/app/options/options.go | 2 +- cmd/kubelet/app/server.go | 2 +- pkg/apis/componentconfig/types.go | 4 ++-- pkg/apis/componentconfig/v1alpha1/defaults.go | 12 ++++++------ pkg/apis/componentconfig/v1alpha1/types.go | 2 +- pkg/kubelet/cm/container_manager_linux.go | 2 +- pkg/kubelet/kubelet.go | 2 +- pkg/kubemark/hollow_kubelet.go | 2 +- test/e2e_node/cgroup_manager_test.go | 8 ++++---- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index ee77b118cd3b5..109ebc0f50332 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -196,7 +196,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.MarkDeprecated("system-container", "Use --system-cgroups instead. Will be removed in a future version.") fs.StringVar(&s.SystemCgroups, "system-cgroups", s.SystemCgroups, "Optional absolute name of cgroups in which to place all non-kernel processes that are not already inside a cgroup under `/`. Empty for no container. Rolling back the flag requires a reboot. (Default: \"\").") - fs.BoolVar(&s.ExperimentalCgroupsPerQOS, "experimental-cgroups-per-qos", s.ExperimentalCgroupsPerQOS, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.") + fs.BoolVar(&s.CgroupsPerQOS, "cgroups-per-qos", s.CgroupsPerQOS, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.") fs.StringVar(&s.CgroupDriver, "cgroup-driver", s.CgroupDriver, "Driver that the kubelet uses to manipulate cgroups on the host. Possible values: 'cgroupfs', 'systemd'") fs.StringVar(&s.CgroupRoot, "cgroup-root", s.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '', which means use the container runtime default.") fs.StringVar(&s.ContainerRuntime, "container-runtime", s.ContainerRuntime, "The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 51e12915075df..d47152ace3ba3 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -460,7 +460,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) { SystemCgroupsName: s.SystemCgroups, KubeletCgroupsName: s.KubeletCgroups, ContainerRuntime: s.ContainerRuntime, - CgroupsPerQOS: s.ExperimentalCgroupsPerQOS, + CgroupsPerQOS: s.CgroupsPerQOS, CgroupRoot: s.CgroupRoot, CgroupDriver: s.CgroupDriver, ProtectKernelDefaults: s.ProtectKernelDefaults, diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index ed81643f8f935..bb52aff26e1aa 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -298,7 +298,7 @@ type KubeletConfiguration struct { // And all Burstable and BestEffort pods are brought up under their // specific top level QoS cgroup. // +optional - ExperimentalCgroupsPerQOS bool + CgroupsPerQOS bool // driver that the kubelet uses to manipulate cgroups on the host (cgroupfs or systemd) // +optional CgroupDriver string @@ -311,7 +311,7 @@ type KubeletConfiguration struct { // +optional SystemCgroups string // CgroupRoot is the root cgroup to use for pods. - // If ExperimentalCgroupsPerQOS is enabled, this is the root of the QoS cgroup hierarchy. + // If CgroupsPerQOS is enabled, this is the root of the QoS cgroup hierarchy. // +optional CgroupRoot string // containerRuntime is the container runtime to use. diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index 89e44f10eb776..a79202fca2609 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -204,8 +204,8 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.CertDirectory == "" { obj.CertDirectory = "/var/run/kubernetes" } - if obj.ExperimentalCgroupsPerQOS == nil { - obj.ExperimentalCgroupsPerQOS = boolVar(false) + if obj.CgroupsPerQOS == nil { + obj.CgroupsPerQOS = boolVar(false) } if obj.ContainerRuntime == "" { obj.ContainerRuntime = "docker" @@ -394,9 +394,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { temp := int32(defaultIPTablesDropBit) obj.IPTablesDropBit = &temp } - if obj.ExperimentalCgroupsPerQOS == nil { + if obj.CgroupsPerQOS == nil { temp := false - obj.ExperimentalCgroupsPerQOS = &temp + obj.CgroupsPerQOS = &temp } if obj.CgroupDriver == "" { obj.CgroupDriver = "cgroupfs" @@ -404,8 +404,8 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { // NOTE: this is for backwards compatibility with earlier releases where cgroup-root was optional. // if cgroups per qos is not enabled, and cgroup-root is not specified, we need to default to the // container runtime default and not default to the root cgroup. - if obj.ExperimentalCgroupsPerQOS != nil { - if *obj.ExperimentalCgroupsPerQOS { + if obj.CgroupsPerQOS != nil { + if *obj.CgroupsPerQOS { if obj.CgroupRoot == "" { obj.CgroupRoot = "/" } diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index 4fbc631ff1edc..31a6afedf5831 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -359,7 +359,7 @@ type KubeletConfiguration struct { // And all Burstable and BestEffort pods are brought up under their // specific top level QoS cgroup. // +optional - ExperimentalCgroupsPerQOS *bool `json:"experimentalCgroupsPerQOS,omitempty"` + CgroupsPerQOS *bool `json:"cgroupsPerQOS,omitempty"` // driver that the kubelet uses to manipulate cgroups on the host (cgroupfs or systemd) // +optional CgroupDriver string `json:"cgroupDriver,omitempty"` diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 5ec80446b0225..e0a1fa27c7689 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -209,7 +209,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I if nodeConfig.CgroupsPerQOS { // this does default to / when enabled, but this tests against regressions. if nodeConfig.CgroupRoot == "" { - return nil, fmt.Errorf("invalid configuration: experimental-cgroups-per-qos was specified and cgroup-root was not specified. To enable the QoS cgroup hierarchy you need to specify a valid cgroup-root") + return nil, fmt.Errorf("invalid configuration: cgroups-per-qos was specified and cgroup-root was not specified. To enable the QoS cgroup hierarchy you need to specify a valid cgroup-root") } // we need to check that the cgroup root actually exists for each subsystem diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 8c6bfd4a0fab2..62f66d80a42e3 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -453,7 +453,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub nodeStatusUpdateFrequency: kubeCfg.NodeStatusUpdateFrequency.Duration, os: kubeDeps.OSInterface, oomWatcher: oomWatcher, - cgroupsPerQOS: kubeCfg.ExperimentalCgroupsPerQOS, + cgroupsPerQOS: kubeCfg.CgroupsPerQOS, cgroupRoot: kubeCfg.CgroupRoot, mounter: kubeDeps.Mounter, writer: kubeDeps.Writer, diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index c80386c550668..926e76cef4b57 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -142,7 +142,7 @@ func GetHollowKubeletConfig( c.EnableCustomMetrics = false c.EnableDebuggingHandlers = true c.EnableServer = true - c.ExperimentalCgroupsPerQOS = false + c.CgroupsPerQOS = false // hairpin-veth is used to allow hairpin packets. Note that this deviates from // what the "real" kubelet currently does, because there's no way to // set promiscuous mode on docker0. diff --git a/test/e2e_node/cgroup_manager_test.go b/test/e2e_node/cgroup_manager_test.go index d9451371c452c..5f6095dc4fe97 100644 --- a/test/e2e_node/cgroup_manager_test.go +++ b/test/e2e_node/cgroup_manager_test.go @@ -143,7 +143,7 @@ var _ = framework.KubeDescribe("Kubelet Cgroup Manager", func() { Describe("QOS containers", func() { Context("On enabling QOS cgroup hierarchy", func() { It("Top level QoS containers should have been created", func() { - if !framework.TestContext.KubeletConfig.ExperimentalCgroupsPerQOS { + if !framework.TestContext.KubeletConfig.CgroupsPerQOS { return } cgroupsToVerify := []cm.CgroupName{cm.CgroupName(v1.PodQOSBurstable), cm.CgroupName(v1.PodQOSBestEffort)} @@ -158,7 +158,7 @@ var _ = framework.KubeDescribe("Kubelet Cgroup Manager", func() { Describe("Pod containers", func() { Context("On scheduling a Guaranteed Pod", func() { It("Pod containers should have been created under the cgroup-root", func() { - if !framework.TestContext.KubeletConfig.ExperimentalCgroupsPerQOS { + if !framework.TestContext.KubeletConfig.CgroupsPerQOS { return } var ( @@ -202,7 +202,7 @@ var _ = framework.KubeDescribe("Kubelet Cgroup Manager", func() { }) Context("On scheduling a BestEffort Pod", func() { It("Pod containers should have been created under the BestEffort cgroup", func() { - if !framework.TestContext.KubeletConfig.ExperimentalCgroupsPerQOS { + if !framework.TestContext.KubeletConfig.CgroupsPerQOS { return } var ( @@ -246,7 +246,7 @@ var _ = framework.KubeDescribe("Kubelet Cgroup Manager", func() { }) Context("On scheduling a Burstable Pod", func() { It("Pod containers should have been created under the Burstable cgroup", func() { - if !framework.TestContext.KubeletConfig.ExperimentalCgroupsPerQOS { + if !framework.TestContext.KubeletConfig.CgroupsPerQOS { return } var ( From d5aa35e3305c648536a01290d3f81364292bf81d Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Fri, 3 Feb 2017 17:19:53 -0500 Subject: [PATCH 2/4] Add generated artifacts --- Makefile | 2 +- hack/verify-flags/known-flags.txt | 2 +- pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go | 4 ++-- pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go | 4 ++-- pkg/generated/openapi/zz_generated.openapi.go | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 71b38274fecdd..44e9242182126 100644 --- a/Makefile +++ b/Makefile @@ -233,7 +233,7 @@ define TEST_E2E_NODE_HELP_INFO # Example: # make test-e2e-node FOCUS=Kubelet SKIP=container # make test-e2e-node REMOTE=true DELETE_INSTANCES=true -# make test-e2e-node TEST_ARGS="--experimental-cgroups-per-qos=true" +# make test-e2e-node TEST_ARGS="--cgroups-per-qos=true" # Build and run tests. endef .PHONY: test-e2e-node diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 74c527e14c3b0..54165de3c5b7b 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -63,6 +63,7 @@ cert-dir certificate-authority cgroup-driver cgroup-root +cgroups-per-qos chaos-chance clean-start cleanup @@ -197,7 +198,6 @@ executor-suicide-timeout exit-on-lock-contention experimental-allowed-unsafe-sysctls experimental-bootstrap-kubeconfig -experimental-cgroups-per-qos experimental-keystone-url experimental-keystone-ca-file experimental-mounter-path diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go index 9cfd13490091e..804fe639b89b1 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go @@ -400,7 +400,7 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu out.RuntimeCgroups = in.RuntimeCgroups out.SystemCgroups = in.SystemCgroups out.CgroupRoot = in.CgroupRoot - if err := api.Convert_Pointer_bool_To_bool(&in.ExperimentalCgroupsPerQOS, &out.ExperimentalCgroupsPerQOS, s); err != nil { + if err := api.Convert_Pointer_bool_To_bool(&in.CgroupsPerQOS, &out.CgroupsPerQOS, s); err != nil { return err } out.CgroupDriver = in.CgroupDriver @@ -569,7 +569,7 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu out.CloudProvider = in.CloudProvider out.CloudConfigFile = in.CloudConfigFile out.KubeletCgroups = in.KubeletCgroups - if err := api.Convert_bool_To_Pointer_bool(&in.ExperimentalCgroupsPerQOS, &out.ExperimentalCgroupsPerQOS, s); err != nil { + if err := api.Convert_bool_To_Pointer_bool(&in.CgroupsPerQOS, &out.CgroupsPerQOS, s); err != nil { return err } out.CgroupDriver = in.CgroupDriver diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go index e1bc1cd5c6de4..ab1e482f2a594 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go @@ -229,8 +229,8 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c * *out = new(int32) **out = **in } - if in.ExperimentalCgroupsPerQOS != nil { - in, out := &in.ExperimentalCgroupsPerQOS, &out.ExperimentalCgroupsPerQOS + if in.CgroupsPerQOS != nil { + in, out := &in.CgroupsPerQOS, &out.CgroupsPerQOS *out = new(bool) **out = **in } diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 7ef6d0c21e0f1..121439b3819ed 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -10168,7 +10168,7 @@ func GetOpenAPIDefinitions(ref openapi.ReferenceCallback) map[string]openapi.Ope Format: "", }, }, - "experimentalCgroupsPerQOS": { + "cgroupsPerQOS": { SchemaProps: spec.SchemaProps{ Description: "Enable QoS based Cgroup hierarchy: top level cgroups for QoS Classes And all Burstable and BestEffort pods are brought up under their specific top level QoS cgroup.", Type: []string{"boolean"}, From 2ab9f0384e031172cdf964a329ec0f1444aecaa2 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Fri, 3 Feb 2017 17:21:37 -0500 Subject: [PATCH 3/4] Update test e2e nodes to use new flag --- test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties | 2 +- .../e2e_node/jenkins/conformance/jenkins-conformance.properties | 2 +- test/e2e_node/jenkins/jenkins-ci.properties | 2 +- test/e2e_node/jenkins/jenkins-flaky.properties | 2 +- test/e2e_node/jenkins/jenkins-pull.properties | 2 +- test/e2e_node/jenkins/jenkins-serial.properties | 2 +- test/e2e_node/jenkins/template.properties | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties b/test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties index d07b60cf8337b..774110a7fbf0a 100644 --- a/test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties +++ b/test/e2e_node/jenkins/benchmark/jenkins-benchmark.properties @@ -5,5 +5,5 @@ GCE_PROJECT=k8s-jkns-ci-node-e2e CLEANUP=true GINKGO_FLAGS='--skip="\[Flaky\]"' TEST_ARGS='--feature-gates=DynamicKubeletConfig=true' -KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/' +KUBELET_ARGS='--cgroups-per-qos=false --cgroup-root=/' PARALLELISM=1 diff --git a/test/e2e_node/jenkins/conformance/jenkins-conformance.properties b/test/e2e_node/jenkins/conformance/jenkins-conformance.properties index f465d8081a949..f1ca88b2d6ddb 100644 --- a/test/e2e_node/jenkins/conformance/jenkins-conformance.properties +++ b/test/e2e_node/jenkins/conformance/jenkins-conformance.properties @@ -3,4 +3,4 @@ GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml GCE_ZONE=us-central1-f GCE_PROJECT=k8s-jkns-ci-node-e2e CLEANUP=true -KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/' +KUBELET_ARGS='--cgroups-per-qos=false --cgroup-root=/' diff --git a/test/e2e_node/jenkins/jenkins-ci.properties b/test/e2e_node/jenkins/jenkins-ci.properties index 313304877333e..9c563b6a05094 100644 --- a/test/e2e_node/jenkins/jenkins-ci.properties +++ b/test/e2e_node/jenkins/jenkins-ci.properties @@ -4,5 +4,5 @@ GCE_ZONE=us-central1-f GCE_PROJECT=k8s-jkns-ci-node-e2e CLEANUP=true GINKGO_FLAGS='--skip="\[Flaky\]|\[Serial\]"' -KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/' +KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' TIMEOUT=1h diff --git a/test/e2e_node/jenkins/jenkins-flaky.properties b/test/e2e_node/jenkins/jenkins-flaky.properties index 737533e0a9215..363b059b2469b 100644 --- a/test/e2e_node/jenkins/jenkins-flaky.properties +++ b/test/e2e_node/jenkins/jenkins-flaky.properties @@ -5,4 +5,4 @@ GCE_PROJECT=k8s-jkns-ci-node-e2e CLEANUP=true GINKGO_FLAGS='--focus="\[Flaky\]"' TEST_ARGS='--feature-gates=DynamicKubeletConfig=true' -KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/' +KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' \ No newline at end of file diff --git a/test/e2e_node/jenkins/jenkins-pull.properties b/test/e2e_node/jenkins/jenkins-pull.properties index d727b590dc3b4..884e45884f1a2 100644 --- a/test/e2e_node/jenkins/jenkins-pull.properties +++ b/test/e2e_node/jenkins/jenkins-pull.properties @@ -4,5 +4,5 @@ GCE_ZONE=us-central1-f GCE_PROJECT=k8s-jkns-pr-node-e2e CLEANUP=true GINKGO_FLAGS='--skip="\[Flaky\]|\[Slow\]|\[Serial\]" --flakeAttempts=2' -KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/' +KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' diff --git a/test/e2e_node/jenkins/jenkins-serial.properties b/test/e2e_node/jenkins/jenkins-serial.properties index acaafc45d269f..892fa27afca71 100644 --- a/test/e2e_node/jenkins/jenkins-serial.properties +++ b/test/e2e_node/jenkins/jenkins-serial.properties @@ -5,6 +5,6 @@ GCE_PROJECT=k8s-jkns-ci-node-e2e CLEANUP=true GINKGO_FLAGS='--focus="\[Serial\]" --skip="\[Flaky\]|\[Benchmark\]"' TEST_ARGS='--feature-gates=DynamicKubeletConfig=true' -KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/' +KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' PARALLELISM=1 TIMEOUT=3h diff --git a/test/e2e_node/jenkins/template.properties b/test/e2e_node/jenkins/template.properties index b23e5984055d6..bdc7cf6f55f5d 100644 --- a/test/e2e_node/jenkins/template.properties +++ b/test/e2e_node/jenkins/template.properties @@ -20,6 +20,6 @@ CLEANUP=true # KUBELET_ARGS are the arguments passed to kubelet. The args will override corresponding default kubelet # setting in the test framework and --kubelet-flags in TEST_ARGS. # If true QoS Cgroup Hierarchy is created and tests specifc to the cgroup hierarchy run -KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/' +KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' # TEST_ARGS are args passed to node e2e test. TEST_ARGS='' From 75fa285a56797c1d312df430eaf39025430d273f Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Fri, 3 Feb 2017 17:28:23 -0500 Subject: [PATCH 4/4] Make hack/local-up-cluster default to right cgroup driver --- hack/local-up-cluster.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index e9cbe0f0a5b62..7c7f2c609c4df 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -34,7 +34,7 @@ NET_PLUGIN=${NET_PLUGIN:-""} NET_PLUGIN_DIR=${NET_PLUGIN_DIR:-""} SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/24} # if enabled, must set CGROUP_ROOT -EXPERIMENTAL_CGROUPS_PER_QOS=${EXPERIMENTAL_CGROUPS_PER_QOS:-false} +CGROUPS_PER_QOS=${CGROUPS_PER_QOS:-false} # this is not defaulted to preserve backward compatibility. # if EXPERIMENTAL_CGROUPS_PER_QOS is enabled, recommend setting to / CGROUP_ROOT=${CGROUP_ROOT:-""} @@ -189,6 +189,17 @@ CERT_DIR=${CERT_DIR:-"/var/run/kubernetes"} ROOT_CA_FILE=$CERT_DIR/apiserver.crt EXPERIMENTAL_CRI=${EXPERIMENTAL_CRI:-"false"} +# name of the cgroup driver, i.e. cgroupfs or systemd +if [[ ${CONTAINER_RUNTIME} == "docker" ]]; then + # default cgroup driver to match what is reported by docker to simplify local development + if [[ -z ${CGROUP_DRIVER} ]]; then + # match driver with docker runtime reported value (they must match) + CGROUP_DRIVER=$(docker info | grep "Cgroup Driver:" | cut -f3- -d' ') + echo "Kubelet cgroup driver defaulted to use: ${CGROUP_DRIVER}" + fi +fi + + # Ensure CERT_DIR is created for auto-generated crt/key and kubeconfig mkdir -p "${CERT_DIR}" &>/dev/null || sudo mkdir -p "${CERT_DIR}" @@ -541,7 +552,7 @@ function start_kubelet { --feature-gates="${FEATURE_GATES}" \ --cpu-cfs-quota=${CPU_CFS_QUOTA} \ --enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}" \ - --experimental-cgroups-per-qos=${EXPERIMENTAL_CGROUPS_PER_QOS} \ + --cgroups-per-qos=${CGROUPS_PER_QOS} \ --cgroup-driver=${CGROUP_DRIVER} \ --cgroup-root=${CGROUP_ROOT} \ --keep-terminated-pod-volumes=true \