Upgrade k8s lib to 1.25 (#1)

* Upgrade k8s lib to 1.25
* Update client/client.go

Co-authored-by: Ivan Mikheykin <[email protected]>

client/client.go

@@ -19,7 +19,6 @@ import (
 	fakedynamic "k8s.io/client-go/dynamic/fake"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
-
 	// load the gcp plugin (only required to authenticate against GKE clusters)
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/client-go/rest"
@@ -55,20 +54,24 @@ type Client interface {
 	APIResourceList(apiVersion string) ([]*metav1.APIResourceList, error)
 	APIResource(apiVersion, kind string) (*metav1.APIResource, error)
 	GroupVersionResource(apiVersion, kind string) (schema.GroupVersionResource, error)
+
+	// ReloadDynamic reloads dynamic fake client.
+	// It is the only way to provide List operations on custom resources in client-go 1.20+.
+	// See https://github.com/kubernetes/client-go/issues/949#issuecomment-811154420
+	ReloadDynamic(gvrList map[schema.GroupVersionResource]string)
 }
 
 func New() Client {
 	return &client{}
 }
 
-func NewFake(_ map[schema.GroupVersionResource]string) Client {
-	scheme := runtime.NewScheme()
-	objs := []runtime.Object{}
-
+func NewFake(gvr map[schema.GroupVersionResource]string) Client {
+	sc := runtime.NewScheme()
 	return &client{
 		Interface:        fake.NewSimpleClientset(),
 		defaultNamespace: "default",
-		dynamicClient:    fakedynamic.NewSimpleDynamicClient(scheme, objs...),
+		dynamicClient:    fakedynamic.NewSimpleDynamicClientWithCustomListKinds(sc, gvr),
+		schema:           sc,
 	}
 }
 
@@ -88,6 +91,12 @@ type client struct {
 	server           string
 	metricStorage    MetricStorage
 	metricLabels     map[string]string
+	schema           *runtime.Scheme
+}
+
+// ReloadDynamic creates new dynamic client with the new set of CRDs.
+func (c *client) ReloadDynamic(gvrList map[schema.GroupVersionResource]string) {
+	c.dynamicClient = fakedynamic.NewSimpleDynamicClientWithCustomListKinds(c.schema, gvrList)
 }
 
 func (c *client) WithServer(server string) {

client/metrics.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"net/url"
 	"time"
 
@@ -68,7 +69,7 @@ type ClientRateLimiterLatencyMetric struct {
 }
 
 // Deprecated: to be removed since it is not a part of the client to support
-func (c ClientRateLimiterLatencyMetric) Observe(verb string, u url.URL, latency time.Duration) {
+func (c ClientRateLimiterLatencyMetric) Observe(ctx context.Context, verb string, u url.URL, latency time.Duration) {
 	c.metricStorage.HistogramObserve(
 		"{PREFIX}kubernetes_client_rate_limiter_latency_seconds",
 		latency.Seconds(),
@@ -91,7 +92,7 @@ type ClientRequestResultMetric struct {
 }
 
 // Deprecated: ClientRequestResultMetric
-func (c ClientRequestResultMetric) Increment(code, method, host string) {
+func (c ClientRequestResultMetric) Increment(ctx context.Context, code, method, host string) {
 	labels := map[string]string{}
 	for k, v := range c.labels {
 		labels[k] = v

fake/cluster.go

@@ -5,45 +5,45 @@ import (
 	"fmt"
 	"strings"
 
+	klient "github.com/flant/kube-client/client"
+	"github.com/flant/kube-client/manifest"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/version"
 	fakediscovery "k8s.io/client-go/discovery/fake"
-	"k8s.io/client-go/kubernetes/scheme"
-
-	klient "github.com/flant/kube-client/client"
-	"github.com/flant/kube-client/manifest"
 )
 
 type Cluster struct {
 	Client klient.Client
 
 	Discovery *fakediscovery.FakeDiscovery
+	gvrList   map[schema.GroupVersionResource]string
 }
 
 func NewFakeCluster(ver ClusterVersion) *Cluster {
 	if ver == "" {
-		ver = ClusterVersionV119
+		ver = ClusterVersionV123
 	}
 	cres := ClusterResources(ver)
 
-	// FIXME: below code will be used in go-client 0.20.x pass it to NewFakeKubernetesClient
-	// gvrToListKind := make(map[schema.GroupVersionResource]string)
-	// for _, gr := range cres {
-	// 	for _, res := range gr.APIResources {
-	// 		gvr := schema.GroupVersionResource{
-	// 			Group:    res.Group,
-	// 			Version:  res.Version,
-	// 			Resource: res.Name,
-	// 		}
-	// 		gvrToListKind[gvr] = res.Kind + "List"
-	// 	}
-	// }
-
-	fc := &Cluster{}
-	fc.Client = klient.NewFake(nil)
+	gvrToListKind := make(map[schema.GroupVersionResource]string)
+	for _, gr := range cres {
+		for _, res := range gr.APIResources {
+			gvr := schema.GroupVersionResource{
+				Group:    res.Group,
+				Version:  res.Version,
+				Resource: res.Name,
+			}
+			gvrToListKind[gvr] = res.Kind + "List"
+		}
+	}
+
+	fc := &Cluster{
+		gvrList: gvrToListKind,
+	}
+	fc.Client = klient.NewFake(gvrToListKind)
 
 	var ok bool
 	fc.Discovery, ok = fc.Client.Discovery().(*fakediscovery.FakeDiscovery)
@@ -56,6 +56,10 @@ func NewFakeCluster(ver ClusterVersion) *Cluster {
 	return fc
 }
 
+func (fc *Cluster) reloadDynamicClient() {
+	fc.Client.ReloadDynamic(fc.gvrList)
+}
+
 func (fc *Cluster) CreateNs(ns string) {
 	nsObj := &corev1.Namespace{}
 	nsObj.Name = ns
@@ -64,10 +68,19 @@ func (fc *Cluster) CreateNs(ns string) {
 
 // RegisterCRD registers custom resources for the cluster
 func (fc *Cluster) RegisterCRD(group, version, kind string, namespaced bool) {
-	scheme.Scheme.AddKnownTypeWithName(schema.GroupVersionKind{Group: group, Version: version, Kind: kind}, &unstructured.Unstructured{})
+	gvk := schema.GroupVersionKind{Group: group, Version: version, Kind: kind}
+	pluralGVR, _ := meta.UnsafeGuessKindToResource(gvk)
+
+	if _, ok := fc.gvrList[pluralGVR]; ok {
+		return
+	}
+
+	fc.gvrList[pluralGVR] = kind + "List"
+	fc.reloadDynamicClient()
+
 	newResource := metav1.APIResource{
 		Kind:       kind,
-		Name:       Pluralize(kind),
+		Name:       pluralGVR.Resource,
 		Verbs:      metav1.Verbs{"create", "delete", "deletecollection", "get", "list", "patch", "update", "watch"},
 		Group:      group,
 		Version:    version,
@@ -174,32 +187,3 @@ func findGvr(resources []*metav1.APIResourceList, apiVersion, kindOrName string)
 	}
 	return nil
 }
-
-// Pluralize is the simplest way to make a plural form (like resource) from k8s object Kind
-// ex: User -> users
-//     Prometheus -> prometheuses
-//     NetworkPolicy -> netwrokpolicies
-//     CustomPrometheusRules -> customprometheusrules
-//     Endpoints -> endpoints
-func Pluralize(kind string) string {
-	if kind == "" {
-		return kind
-	}
-
-	kind = strings.ToLower(kind)
-
-	// maybe we dont need more complex pluralizer here
-	// but if we do, can take smth like https://github.com/gertd/go-pluralize
-	switch {
-	case strings.HasSuffix(kind, "es"):
-		return kind
-	case strings.HasSuffix(kind, "ts"):
-		return kind
-	case strings.HasSuffix(kind, "s"):
-		return kind + "es"
-	case strings.HasSuffix(kind, "cy"):
-		return strings.TrimSuffix(kind, "y") + "ies"
-	}
-
-	return kind + "s"
-}

fake/cluster_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -13,27 +12,42 @@ import (
 func TestRegisterCRD(t *testing.T) {
 	f := NewFakeCluster("")
 
-	f.RegisterCRD("deckhouse.io", "v1alpha1", "KeepalivedInstance", false)
-	gvk := schema.GroupVersionResource{
-		Group:    "deckhouse.io",
-		Version:  "v1alpha1",
-		Resource: Pluralize("KeepalivedInstance"),
-	}
-	_, err := f.Client.Dynamic().Resource(gvk).Namespace("").List(context.TODO(), v1.ListOptions{})
-	require.NoError(t, err)
-}
+	t.Run("test CRD registration", func(t *testing.T) {
+		f.RegisterCRD("deckhouse.io", "v1alpha1", "KeepalivedInstance", false)
+		gvk := schema.GroupVersionResource{
+			Group:    "deckhouse.io",
+			Version:  "v1alpha1",
+			Resource: "keepalivedinstances",
+		}
+		_, err := f.Client.Dynamic().Resource(gvk).Namespace("").List(context.TODO(), v1.ListOptions{})
+		require.NoError(t, err)
+
+		_, err = f.Client.Dynamic().Resource(gvk).Namespace("").Get(context.TODO(), "foo", v1.GetOptions{})
+		require.ErrorContains(t, err, "not found")
+
+		// register next CRD
+		f.RegisterCRD("deckhouse.io", "v1", "NodeGroup", false)
+
+		// new crd exists
+		ngGVR := schema.GroupVersionResource{
+			Group:    "deckhouse.io",
+			Version:  "v1",
+			Resource: "nodegroups",
+		}
+		_, err = f.Client.Dynamic().Resource(ngGVR).Namespace("").List(context.TODO(), v1.ListOptions{})
+		require.NoError(t, err)
+
+		// previous crd exists
+		_, err = f.Client.Dynamic().Resource(gvk).Namespace("").List(context.TODO(), v1.ListOptions{})
+		require.NoError(t, err)
+	})
 
-func TestPluralize(t *testing.T) {
-	tests := map[string]string{
-		"Endpoints":             "endpoints",
-		"Prometheus":            "prometheuses",
-		"NetworkPolicy":         "networkpolicies",
-		"CustomPrometheusRules": "customprometheusrules",
-		"Alertmanager":          "alertmanagers",
-		"Node":                  "nodes",
-	}
-
-	for before, after := range tests {
-		assert.Equal(t, after, Pluralize(before))
-	}
+	t.Run("test default resources", func(t *testing.T) {
+		_, err := f.Client.Dynamic().Resource(schema.GroupVersionResource{
+			Group:    "",
+			Version:  "v1",
+			Resource: "pods",
+		}).Namespace("").List(context.TODO(), v1.ListOptions{})
+		require.NoError(t, err)
+	})
 }

fake/resources.go

@@ -3,6 +3,9 @@ package fake
 // set current kube-context to cluster with necessary version and run go generate
 // it will create file with desired version and resources
 // you can use existing cluster or kind/minikube/microk8s/etc
+// like: kind create cluster --image "kindest/node:v1.25.3"
+// you can images for kind here, in a release message: https://github.com/kubernetes-sigs/kind/releases
+
 //go:generate ./scripts/resources_generator
 
 import (
@@ -29,6 +32,18 @@ func ClusterResources(version ClusterVersion) []*metav1.APIResourceList {
 
 	case ClusterVersionV121:
 		return v121ClusterResources
+
+	case ClusterVersionV122:
+		return v122ClusterResources
+
+	case ClusterVersionV123:
+		return v123ClusterResources
+
+	case ClusterVersionV124:
+		return v124ClusterResources
+
+	case ClusterVersionV125:
+		return v125ClusterResources
 	}
 
 	return nil
@@ -44,6 +59,10 @@ const (
 	ClusterVersionV119 ClusterVersion = "v1.19.0"
 	ClusterVersionV120 ClusterVersion = "v1.20.0"
 	ClusterVersionV121 ClusterVersion = "v1.21.0"
+	ClusterVersionV122 ClusterVersion = "v1.22.0"
+	ClusterVersionV123 ClusterVersion = "v1.23.0"
+	ClusterVersionV124 ClusterVersion = "v1.24.0"
+	ClusterVersionV125 ClusterVersion = "v1.25.0"
 )
 
 func (cv ClusterVersion) String() string {