diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a50e1c9fb7..077a193c77 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: # separate job for parallelism lint: name: Lint - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Check out code uses: actions/checkout@v4 @@ -57,7 +57,7 @@ jobs: build-master: name: Build-master - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: # Create a cache for the built master image - name: Restore master image cache @@ -147,7 +147,7 @@ jobs: build-pr: name: Build-PR - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: # Create a cache for the build PR image - name: Restore PR image cache @@ -195,8 +195,9 @@ jobs: run: | set -x pushd go-controller - # exit early if there are gofmt issues + # exit early if there are gofmt or go mod / vendor issues make gofmt + make verify-go-mod-vendor make make windows COVERALLS=1 CONTAINER_RUNNABLE=1 make check @@ -256,7 +257,7 @@ jobs: ovn-upgrade-e2e: name: Upgrade OVN from Master to PR branch based image if: github.event_name != 'schedule' - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 120 needs: - build-master @@ -327,7 +328,7 @@ jobs: if: always() run: | mkdir -p /tmp/kind/logs - kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug /tmp/kind/logs + kind export logs --name ${KIND_CLUSTER_NAME} --verbosity 4 /tmp/kind/logs set -x docker ps -a docker exec ovn-control-plane crictl images @@ -375,7 +376,7 @@ jobs: if: always() run: | mkdir -p /tmp/kind/logs-kind-pr-branch - kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug /tmp/kind/logs-kind-pr-branch + kind export logs --name ${KIND_CLUSTER_NAME} --verbosity 4 /tmp/kind/logs-kind-pr-branch - name: Upload kind logs if: always() @@ -386,7 +387,7 @@ jobs: e2e: name: e2e - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 # 30 mins for kind, 180 mins for control-plane tests, 10 minutes for all other steps timeout-minutes: 220 strategy: @@ -406,13 +407,14 @@ jobs: # num-nodes-per-zone : "" # forwarding : ["", "disable-forwarding"] # dns-name-resolver : ["", "enable-dns-name-resolver"] + # traffic-flow-tests : "" include: - {"target": "shard-conformance", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-disabled"} - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "control-plane", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-disabled", "dns-name-resolver": "enable-dns-name-resolver"} - - {"target": "control-plane", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-disabled"} + - {"target": "control-plane", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-disabled", "traffic-flow-tests": "1,2,3"} - {"target": "control-plane-helm","ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-disabled", "dns-name-resolver": "enable-dns-name-resolver"} - {"target": "control-plane-helm","ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "dns-name-resolver": "enable-dns-name-resolver"} - {"target": "control-plane", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "dns-name-resolver": "enable-dns-name-resolver"} @@ -437,6 +439,7 @@ jobs: - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-disabled"} - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} + - {"target": "traffic-flow-test-only","ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "traffic-flow-tests": "1-24"} - {"target": "tools", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} needs: [ build-pr ] env: @@ -446,13 +449,13 @@ jobs: OVN_EMPTY_LB_EVENTS: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' }}" OVN_HA: "${{ matrix.ha == 'HA' }}" OVN_DISABLE_SNAT_MULTIPLE_GWS: "${{ matrix.disable-snat-multiple-gws == 'noSnatGW' }}" - KIND_INSTALL_METALLB: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' }}" + KIND_INSTALL_METALLB: "${{ matrix.target == 'control-plane' || matrix.target == 'control-plane-helm' || matrix.target == 'network-segmentation' }}" OVN_GATEWAY_MODE: "${{ matrix.gateway-mode }}" OVN_SECOND_BRIDGE: "${{ matrix.second-bridge == '2br' }}" KIND_IPV4_SUPPORT: "${{ matrix.ipfamily == 'IPv4' || matrix.ipfamily == 'dualstack' }}" KIND_IPV6_SUPPORT: "${{ matrix.ipfamily == 'IPv6' || matrix.ipfamily == 'dualstack' }}" - ENABLE_MULTI_NET: "${{ matrix.target == 'multi-homing' || matrix.target == 'kv-live-migration' || matrix.target == 'network-segmentation' || matrix.target == 'tools' || matrix.target == 'multi-homing-helm' }}" - ENABLE_NETWORK_SEGMENTATION: "${{ matrix.target == 'network-segmentation' || matrix.target == 'tools' || matrix.target == 'kv-live-migration'}}" + ENABLE_MULTI_NET: "${{ matrix.target == 'multi-homing' || matrix.target == 'kv-live-migration' || matrix.target == 'network-segmentation' || matrix.target == 'tools' || matrix.target == 'multi-homing-helm' || matrix.target == 'traffic-flow-test-only' }}" + ENABLE_NETWORK_SEGMENTATION: "${{ matrix.target == 'network-segmentation' || matrix.target == 'tools' || matrix.target == 'kv-live-migration' || matrix.target == 'traffic-flow-test-only' }}" DISABLE_UDN_HOST_ISOLATION: "true" KIND_INSTALL_KUBEVIRT: "${{ matrix.target == 'kv-live-migration' }}" OVN_COMPACT_MODE: "${{ matrix.target == 'compact-mode' }}" @@ -463,6 +466,7 @@ jobs: OVN_DISABLE_FORWARDING: "${{ matrix.forwarding == 'disable-forwarding' }}" USE_HELM: "${{ matrix.target == 'control-plane-helm' || matrix.target == 'multi-homing-helm' }}" OVN_ENABLE_DNSNAMERESOLVER: "${{ matrix.dns-name-resolver == 'enable-dns-name-resolver' }}" + TRAFFIC_FLOW_TESTS: "${{ matrix.traffic-flow-tests }}" steps: - name: Install VRF kernel module @@ -482,6 +486,11 @@ jobs: msbuild mysql-server-core-* php-* php7* \ powershell temurin-* zulu-* + - name: Setup /mnt/runner directory + run: | + sudo mkdir -pv /mnt/runner + sudo chown runner:runner /mnt/runner + - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -523,6 +532,11 @@ jobs: export OVN_IMAGE="ovn-daemonset-fedora:pr" make -C test install-kind + - name: traffic-flow-tests setup + timeout-minutes: 5 + if: env.TRAFFIC_FLOW_TESTS != '' + run: make -C test traffic-flow-tests WHAT="setup" + - name: Runner Diagnostics uses: ./.github/actions/diagnostics @@ -554,6 +568,10 @@ jobs: elif [ "${{ matrix.target }}" == "tools" ]; then make -C go-controller build make -C test tools + elif [ "${{ matrix.target }}" == "traffic-flow-test-only" ]; then + # Traffic Flow Tests can be ran as part of a target, as an additional + # set of test, set via TRAFFIC_FLOW_TESTS. See below. + : else make -C test ${{ matrix.target }} if [ "${{ matrix.ipfamily }}" != "ipv6" ]; then @@ -561,6 +579,11 @@ jobs: fi fi + # If target also specified traffic flow tests to run, do so now + if [ -n "${TRAFFIC_FLOW_TESTS}" ]; then + make -C test traffic-flow-tests WHAT="run" + fi + - name: Runner Diagnostics uses: ./.github/actions/diagnostics @@ -568,7 +591,10 @@ jobs: if: always() run: | mkdir -p /tmp/kind/logs - kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug /tmp/kind/logs + kind export logs --name ${KIND_CLUSTER_NAME} --verbosity 4 /tmp/kind/logs + if [ -n "${TRAFFIC_FLOW_TESTS}" ]; then + mv -v /tmp/{,kind/logs/}traffic_flow_test_result.json ||: + fi - name: Upload kind logs if: always() @@ -580,7 +606,7 @@ jobs: e2e-dual-conversion: name: e2e-dual-conversion if: github.event_name != 'schedule' - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 60 strategy: fail-fast: false @@ -675,7 +701,7 @@ jobs: if: always() run: | mkdir -p /tmp/kind/logs - kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug /tmp/kind/logs + kind export logs --name ${KIND_CLUSTER_NAME} --verbosity 4 /tmp/kind/logs - name: Upload kind logs if: always() diff --git a/contrib/kind-common b/contrib/kind-common index ff96699030..8fb327d936 100644 --- a/contrib/kind-common +++ b/contrib/kind-common @@ -122,20 +122,18 @@ install_ingress() { METALLB_DIR="/tmp/metallb" install_metallb() { - local metallb_version=v0.14.8 + # TODO: Pin to a version tag bigger than v0.14.8 when released + # so we can have a metallb dev-env that support dual stack + # and use a proper tag instead of a commit id. + local metallb_version=55f648102b918699da610f20e8d650e76c7561fc mkdir -p /tmp/metallb local builddir builddir=$(mktemp -d "${METALLB_DIR}/XXXXXX") pushd "${builddir}" - git clone https://github.com/metallb/metallb.git -b $metallb_version + git clone https://github.com/metallb/metallb.git cd metallb - # Use global IP next hops in IPv6 - if [ "$KIND_IPV6_SUPPORT" == true ]; then - sed -i '/address-family PROTOCOL unicast/a \ - neighbor NODE0_IP route-map IPV6GLOBAL in\n neighbor NODE1_IP route-map IPV6GLOBAL in\n neighbor NODE2_IP route-map IPV6GLOBAL in' dev-env/bgp/frr/bgpd.conf.tmpl - printf "route-map IPV6GLOBAL permit 10\n set ipv6 next-hop prefer-global" >> dev-env/bgp/frr/bgpd.conf.tmpl - fi + git checkout $metallb_version pip install -r dev-env/requirements.txt local ip_family ipv6_network diff --git a/contrib/kind-dual-stack-conversion.sh b/contrib/kind-dual-stack-conversion.sh index d68739381a..b035ee4b6f 100755 --- a/contrib/kind-dual-stack-conversion.sh +++ b/contrib/kind-dual-stack-conversion.sh @@ -33,7 +33,7 @@ convert_cni() { # restart ovnkube-master # FIXME: kubectl rollout restart deployment leaves the old pod hanging # as workaround we delete the master directly. When deployed with - # OVN_INTERCONNECT_ENABLE=true, the db and ncm pods need that too. + # OVN_INTERCONNECT_ENABLE=true, the db and cm pods need that too. # Depending on how kind was deployed, the pods have different labels. kubectl -n ovn-kubernetes delete pod -l name=ovnkube-db ||: kubectl -n ovn-kubernetes delete pod -l name=ovnkube-zone-controller ||: diff --git a/go-controller/.mockery.yaml b/go-controller/.mockery.yaml index c931199231..5d7cbc0e95 100644 --- a/go-controller/.mockery.yaml +++ b/go-controller/.mockery.yaml @@ -51,6 +51,9 @@ packages: github.com/containernetworking/plugins/pkg/ns: interfaces: NetNS: + github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1: + interfaces: + NetworkAttachmentDefinitionInformer: github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1: interfaces: NetworkAttachmentDefinitionLister: diff --git a/go-controller/Makefile b/go-controller/Makefile index 5465e5f2e7..829c37eb4e 100644 --- a/go-controller/Makefile +++ b/go-controller/Makefile @@ -89,7 +89,7 @@ clean: rm -f ./pkg/sbdb/ovn-sb.ovsschema rm -f ./pkg/vswitchd/vswitch.ovsschema -.PHONY: lint gofmt +.PHONY: lint gofmt verify-go-mod-vendor lint: ifeq ($(CONTAINER_RUNNABLE), 0) @@ -105,6 +105,13 @@ else @./hack/verify-gofmt.sh endif +verify-go-mod-vendor: +ifeq ($(CONTAINER_RUNNABLE), 0) + @GOPATH=${GOPATH} ./hack/verify-go-mod-vendor.sh +else + @./hack/verify-go-mod-vendor.sh +endif + pkg/nbdb/ovn-nb.ovsschema: curl -sSL https://raw.githubusercontent.com/ovn-org/ovn/$(OVN_SCHEMA_VERSION)/ovn-nb.ovsschema -o $@ diff --git a/go-controller/cmd/ovnkube/ovnkube.go b/go-controller/cmd/ovnkube/ovnkube.go index aab9e51509..60b11c7f9a 100644 --- a/go-controller/cmd/ovnkube/ovnkube.go +++ b/go-controller/cmd/ovnkube/ovnkube.go @@ -23,10 +23,10 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controllermanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" - controllerManager "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-controller-manager" ovnnode "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -343,7 +343,7 @@ func startOvnKube(ctx *cli.Context, cancel context.CancelFunc) error { case runMode.ovnkubeController: metrics.RegisterOVNKubeControllerBase() haConfig = &config.MasterHA - name = networkControllerManagerLockName() + name = controllerManagerLockName() case runMode.clusterManager: metrics.RegisterClusterManagerBase() haConfig = &config.ClusterMgrHA @@ -505,7 +505,7 @@ func runOvnKube(ctx context.Context, runMode *ovnkubeRunMode, ovnClientset *util return } - networkControllerManager, err := controllerManager.NewNetworkControllerManager( + controllerManager, err := controllermanager.NewControllerManager( ovnClientset, watchFactory, libovsdbOvnNBClient, @@ -517,7 +517,7 @@ func runOvnKube(ctx context.Context, runMode *ovnkubeRunMode, ovnClientset *util return } - err = networkControllerManager.Start(ctx) + err = controllerManager.Start(ctx) if err != nil { controllerErr = fmt.Errorf("failed to start network controller: %w", err) return @@ -527,7 +527,7 @@ func runOvnKube(ctx context.Context, runMode *ovnkubeRunMode, ovnClientset *util metrics.MetricOVNKubeControllerReadyDuration.Set(time.Since(startTime).Seconds()) <-ctx.Done() - networkControllerManager.Stop() + controllerManager.Stop() }() } @@ -545,7 +545,7 @@ func runOvnKube(ctx context.Context, runMode *ovnkubeRunMode, ovnClientset *util // register ovnkube node specific prometheus metrics exported by the node metrics.RegisterNodeMetrics(ctx.Done()) - nodeNetworkControllerManager, err := controllerManager.NewNodeNetworkControllerManager( + nodeControllerManager, err := controllermanager.NewNodeControllerManager( ovnClientset, watchFactory, runMode.identity, @@ -557,7 +557,7 @@ func runOvnKube(ctx context.Context, runMode *ovnkubeRunMode, ovnClientset *util return } - err = nodeNetworkControllerManager.Start(ctx) + err = nodeControllerManager.Start(ctx) if err != nil { nodeErr = fmt.Errorf("failed to start node network controller: %w", err) return @@ -567,7 +567,7 @@ func runOvnKube(ctx context.Context, runMode *ovnkubeRunMode, ovnClientset *util metrics.MetricNodeReadyDuration.Set(time.Since(startTime).Seconds()) <-ctx.Done() - nodeNetworkControllerManager.Stop() + nodeControllerManager.Stop() }() } @@ -656,7 +656,7 @@ func (p ovnkubeMetricsProvider) NewLeaderMetric() leaderelection.LeaderMetric { return &leaderMetrics{p.runMode} } -func networkControllerManagerLockName() string { +func controllerManagerLockName() string { // keep the same old lock name unless we are owners of a specific zone name := "ovn-kubernetes-master" if config.Default.Zone != types.OvnDefaultZone { diff --git a/go-controller/hack/test-go.sh b/go-controller/hack/test-go.sh index fcd094999b..1cb02e3416 100755 --- a/go-controller/hack/test-go.sh +++ b/go-controller/hack/test-go.sh @@ -72,7 +72,7 @@ function testrun { } # These packages requires root for network namespace manipulation in unit tests -root_pkgs=("github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-controller-manager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iptables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/rulemanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/vrfmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/controllers/egressip") +root_pkgs=("github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controllermanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iptables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/rulemanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/vrfmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/controllers/egressip") # These packages are big and require more than the 10m default to run the unit tests big_pkgs=("github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn") diff --git a/go-controller/hack/verify-go-mod-vendor.sh b/go-controller/hack/verify-go-mod-vendor.sh new file mode 100755 index 0000000000..39ce9104bc --- /dev/null +++ b/go-controller/hack/verify-go-mod-vendor.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -o errexit # Nozero exit code of any of the commands below will fail the test. +set -o nounset +set -o pipefail + +HERE=$(dirname "$(readlink --canonicalize "$BASH_SOURCE")") +ROOT=$(readlink --canonicalize "$HERE/..") + +echo "Checking that go mod / sum and vendor/ is correct" +cd "$ROOT/" +go mod tidy +go mod vendor +cd - +CHANGES=$(git status --porcelain) +if [ -n "$CHANGES" ] ; then + echo "ERROR: detected go mod / sum or vendor inconsistency after 'go mod tidy; go mod vendor':" + echo "$CHANGES" + exit 1 +fi diff --git a/go-controller/pkg/clustermanager/clustermanager.go b/go-controller/pkg/clustermanager/clustermanager.go index f97659994a..fcb03bfb93 100644 --- a/go-controller/pkg/clustermanager/clustermanager.go +++ b/go-controller/pkg/clustermanager/clustermanager.go @@ -19,6 +19,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/unidling" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/healthcheck" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -54,6 +55,9 @@ type ClusterManager struct { // used for leader election identity string statusManager *status_manager.StatusManager + + // networkManager creates and deletes network controllers + networkManager networkmanager.Controller } // NewClusterManager creates a new cluster manager to manage the cluster nodes. @@ -77,11 +81,19 @@ func NewClusterManager(ovnClient *util.OVNClusterManagerClientset, wf *factory.W statusManager: status_manager.NewStatusManager(wf, ovnClient), } + cm.networkManager = networkmanager.Default() if config.OVNKubernetesFeature.EnableMultiNetwork { - cm.secondaryNetClusterManager, err = newSecondaryNetworkClusterManager(ovnClient, wf, recorder) + cm.secondaryNetClusterManager, err = newSecondaryNetworkClusterManager(ovnClient, wf, cm.networkManager.Interface(), recorder) + if err != nil { + return nil, err + } + + cm.networkManager, err = networkmanager.NewForCluster(cm.secondaryNetClusterManager, wf, recorder) if err != nil { return nil, err } + cm.secondaryNetClusterManager.networkManager = cm.networkManager.Interface() + } if config.OVNKubernetesFeature.EnableEgressIP { @@ -114,7 +126,7 @@ func NewClusterManager(ovnClient *util.OVNClusterManagerClientset, wf *factory.W } } if util.IsNetworkSegmentationSupportEnabled() { - cm.endpointSliceMirrorController, err = endpointslicemirror.NewController(ovnClient, wf, cm.secondaryNetClusterManager.nadController) + cm.endpointSliceMirrorController, err = endpointslicemirror.NewController(ovnClient, wf, cm.networkManager.Interface()) if err != nil { return nil, err } @@ -156,8 +168,12 @@ func (cm *ClusterManager) Start(ctx context.Context) error { return err } - // Start secondary CM first so that NAD controller initializes before other controllers - if config.OVNKubernetesFeature.EnableMultiNetwork { + // Start networkManager before other controllers + if err := cm.networkManager.Start(); err != nil { + return err + } + + if cm.secondaryNetClusterManager != nil { if err := cm.secondaryNetClusterManager.Start(); err != nil { return err } diff --git a/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller.go b/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller.go index 7b0e17185c..0e077449be 100644 --- a/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller.go +++ b/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller.go @@ -22,7 +22,7 @@ import ( "k8s.io/klog/v2" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -43,7 +43,7 @@ type Controller struct { endpointSlicesSynced cache.InformerSynced podLister corelisters.PodLister podsSynced cache.InformerSynced - nadController *networkAttachDefController.NetAttachDefinitionController + networkManager networkmanager.Interface cancel context.CancelFunc } @@ -111,14 +111,16 @@ func (c *Controller) onEndpointSliceAdd(obj interface{}) { func NewController( ovnClient *util.OVNClusterManagerClientset, - wf *factory.WatchFactory, nadController *networkAttachDefController.NetAttachDefinitionController) (*Controller, error) { + wf *factory.WatchFactory, + networkManager networkmanager.Interface, +) (*Controller, error) { wg := &sync.WaitGroup{} c := &Controller{ - kubeClient: ovnClient.KubeClient, - wg: wg, - name: types.EndpointSliceMirrorControllerName, - nadController: nadController, + kubeClient: ovnClient.KubeClient, + wg: wg, + name: types.EndpointSliceMirrorControllerName, + networkManager: networkManager, } c.queue = workqueue.NewTypedRateLimitingQueueWithConfig( @@ -244,7 +246,7 @@ func (c *Controller) syncDefaultEndpointSlice(ctx context.Context, key string) e return err } - namespacePrimaryNetwork, err := c.nadController.GetActiveNetworkForNamespace(namespace) + namespacePrimaryNetwork, err := c.networkManager.GetActiveNetworkForNamespace(namespace) if err != nil { return err } diff --git a/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller_test.go b/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller_test.go index 5f31e312ae..953c33bfd1 100644 --- a/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller_test.go +++ b/go-controller/pkg/clustermanager/endpointslicemirror/endpointslice_mirror_controller_test.go @@ -17,20 +17,19 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" - kubetest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" - fakenad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" + testnm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) var _ = ginkgo.Describe("Cluster manager EndpointSlice mirror controller", func() { var ( - app *cli.App - controller *Controller - fakeClient *util.OVNClusterManagerClientset - nadController *nad.NetAttachDefinitionController + app *cli.App + controller *Controller + fakeClient *util.OVNClusterManagerClientset + networkManager networkmanager.Controller ) start := func(objects ...runtime.Object) { @@ -40,15 +39,15 @@ var _ = ginkgo.Describe("Cluster manager EndpointSlice mirror controller", func( fakeClient = util.GetOVNClientset(objects...).GetClusterManagerClientset() wf, err := factory.NewClusterManagerWatchFactory(fakeClient) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - nadController, err = nad.NewNetAttachDefinitionController("test", &fakenad.FakeNetworkControllerManager{}, wf, nil) + networkManager, err = networkmanager.NewForCluster(&testnm.FakeControllerManager{}, wf, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - controller, err = NewController(fakeClient, wf, nadController) + controller, err = NewController(fakeClient, wf, networkManager.Interface()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = wf.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = nadController.Start() + err = networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = controller.Start(context.Background(), 1) @@ -70,8 +69,8 @@ var _ = ginkgo.Describe("Cluster manager EndpointSlice mirror controller", func( if controller != nil { controller.Stop() } - if nadController != nil { - nadController.Stop() + if networkManager != nil { + networkManager.Stop() } }) @@ -108,7 +107,7 @@ var _ = ginkgo.Describe("Cluster manager EndpointSlice mirror controller", func( }, }, } - staleEndpointSlice := kubetest.MirrorEndpointSlice(&defaultEndpointSlice, "l3-network", false) + staleEndpointSlice := testing.MirrorEndpointSlice(&defaultEndpointSlice, "l3-network", false) staleEndpointSlice.Labels[types.LabelSourceEndpointSlice] = "non-existing-endpointslice" objs := []runtime.Object{ @@ -314,7 +313,7 @@ var _ = ginkgo.Describe("Cluster manager EndpointSlice mirror controller", func( }, }, } - mirroredEndpointSlice := kubetest.MirrorEndpointSlice(&defaultEndpointSlice, "l3-network", false) + mirroredEndpointSlice := testing.MirrorEndpointSlice(&defaultEndpointSlice, "l3-network", false) objs := []runtime.Object{ &v1.PodList{ Items: []v1.Pod{ diff --git a/go-controller/pkg/clustermanager/fake_cluster_manager_test.go b/go-controller/pkg/clustermanager/fake_cluster_manager_test.go index 0d3bde22d2..49134d0c7d 100644 --- a/go-controller/pkg/clustermanager/fake_cluster_manager_test.go +++ b/go-controller/pkg/clustermanager/fake_cluster_manager_test.go @@ -16,7 +16,7 @@ import ( egresssvc "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1" egresssvcfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/clientset/versioned/fake" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/healthcheck" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "k8s.io/apimachinery/pkg/runtime" @@ -90,8 +90,7 @@ func (o *FakeClusterManager) init() { gomega.Expect(err).ToNot(gomega.HaveOccurred()) } if util.IsNetworkSegmentationSupportEnabled() { - nadController := &nad.NetAttachDefinitionController{} - o.epsMirror, err = endpointslicemirror.NewController(o.fakeClient, o.watcher, nadController) + o.epsMirror, err = endpointslicemirror.NewController(o.fakeClient, o.watcher, networkmanager.Default().Interface()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = o.epsMirror.Start(context.TODO(), 1) diff --git a/go-controller/pkg/clustermanager/network_cluster_controller.go b/go-controller/pkg/clustermanager/network_cluster_controller.go index 0e06785e94..3c5b8491a6 100644 --- a/go-controller/pkg/clustermanager/network_cluster_controller.go +++ b/go-controller/pkg/clustermanager/network_cluster_controller.go @@ -17,7 +17,6 @@ import ( ipamclaimsapi "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/id" - idallocator "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/id" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/ip/subnet" annotationalloc "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/pod" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/node" @@ -25,7 +24,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/persistentips" objretry "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -61,11 +60,11 @@ type networkClusterController struct { tunnelIDAllocator id.Allocator podAllocator *pod.PodAllocator nodeAllocator *node.NodeAllocator - networkIDAllocator idallocator.NamedAllocator + networkIDAllocator id.NamedAllocator ipamClaimReconciler *persistentips.IPAMClaimReconciler subnetAllocator subnet.Allocator - nadController *networkAttachDefController.NetAttachDefinitionController + networkManager networkmanager.Interface // event recorder used to post events to k8s recorder record.EventRecorder @@ -79,12 +78,18 @@ type networkClusterController struct { // To avoid changing that error report with every update, we store reported error node. reportedErrorNode string - util.NetInfo + util.ReconcilableNetInfo } -func newNetworkClusterController(networkIDAllocator idallocator.NamedAllocator, netInfo util.NetInfo, - ovnClient *util.OVNClusterManagerClientset, wf *factory.WatchFactory, recorder record.EventRecorder, - nadController *networkAttachDefController.NetAttachDefinitionController, errorReporter NetworkStatusReporter) *networkClusterController { +func newNetworkClusterController( + networkIDAllocator id.NamedAllocator, + netInfo util.NetInfo, + ovnClient *util.OVNClusterManagerClientset, + wf *factory.WatchFactory, + recorder record.EventRecorder, + networkManager networkmanager.Interface, + errorReporter NetworkStatusReporter, +) *networkClusterController { kube := &kube.KubeOVN{ Kube: kube.Kube{ KClient: ovnClient.KubeClient, @@ -95,17 +100,17 @@ func newNetworkClusterController(networkIDAllocator idallocator.NamedAllocator, wg := &sync.WaitGroup{} ncc := &networkClusterController{ - NetInfo: netInfo, - watchFactory: wf, - kube: kube, - stopChan: make(chan struct{}), - wg: wg, - networkIDAllocator: networkIDAllocator, - recorder: recorder, - nadController: nadController, - statusReporter: errorReporter, - nodeErrors: make(map[string]string), - nodeErrorsLock: sync.Mutex{}, + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), + watchFactory: wf, + kube: kube, + stopChan: make(chan struct{}), + wg: wg, + networkIDAllocator: networkIDAllocator, + recorder: recorder, + networkManager: networkManager, + statusReporter: errorReporter, + nodeErrors: make(map[string]string), + nodeErrorsLock: sync.Mutex{}, } return ncc @@ -114,7 +119,7 @@ func newNetworkClusterController(networkIDAllocator idallocator.NamedAllocator, func newDefaultNetworkClusterController(netInfo util.NetInfo, ovnClient *util.OVNClusterManagerClientset, wf *factory.WatchFactory, recorder record.EventRecorder) *networkClusterController { // use an allocator that can only allocate a single network ID for the // defaiult network - networkIDAllocator := idallocator.NewIDAllocator(types.DefaultNetworkName, 1) + networkIDAllocator := id.NewIDAllocator(types.DefaultNetworkName, 1) // Reserve the id 0 for the default network. err := networkIDAllocator.ReserveID(types.DefaultNetworkName, defaultNetworkID) if err != nil { @@ -122,7 +127,7 @@ func newDefaultNetworkClusterController(netInfo util.NetInfo, ovnClient *util.OV } namedIDAllocator := networkIDAllocator.ForName(types.DefaultNetworkName) - return newNetworkClusterController(namedIDAllocator, netInfo, ovnClient, wf, recorder, nil, nil) + return newNetworkClusterController(namedIDAllocator, netInfo, ovnClient, wf, recorder, networkmanager.Default().Interface(), nil) } func (ncc *networkClusterController) hasPodAllocation() bool { @@ -156,8 +161,8 @@ func (ncc *networkClusterController) hasNodeAllocation() bool { func (ncc *networkClusterController) allowPersistentIPs() bool { return config.OVNKubernetesFeature.EnablePersistentIPs && - util.DoesNetworkRequireIPAM(ncc.NetInfo) && - util.AllowsPersistentIPs(ncc.NetInfo) + util.DoesNetworkRequireIPAM(ncc.GetNetInfo()) && + util.AllowsPersistentIPs(ncc.GetNetInfo()) } func (ncc *networkClusterController) init() error { @@ -171,11 +176,8 @@ func (ncc *networkClusterController) init() error { return err } - if util.DoesNetworkRequireTunnelIDs(ncc.NetInfo) { + if util.DoesNetworkRequireTunnelIDs(ncc.GetNetInfo()) { ncc.tunnelIDAllocator = id.NewIDAllocator(ncc.GetNetworkName(), types.MaxLogicalPortTunnelKey) - if err != nil { - return fmt.Errorf("failed to create new id allocator for network %s: %w", ncc.GetNetworkName(), err) - } // Reserve the id 0. We don't want to assign this id to any of the pods or nodes. if err = ncc.tunnelIDAllocator.ReserveID("zero", util.NoID); err != nil { return err @@ -210,7 +212,7 @@ func (ncc *networkClusterController) init() error { if ncc.hasNodeAllocation() { ncc.retryNodes = ncc.newRetryFramework(factory.NodeType, true) - ncc.nodeAllocator = node.NewNodeAllocator(networkID, ncc.NetInfo, ncc.watchFactory.NodeCoreInformer().Lister(), ncc.kube, ncc.tunnelIDAllocator) + ncc.nodeAllocator = node.NewNodeAllocator(networkID, ncc.GetNetInfo(), ncc.watchFactory.NodeCoreInformer().Lister(), ncc.kube, ncc.tunnelIDAllocator) err := ncc.nodeAllocator.Init() if err != nil { return fmt.Errorf("failed to initialize host subnet ip allocator: %w", err) @@ -219,9 +221,9 @@ func (ncc *networkClusterController) init() error { if ncc.hasPodAllocation() { ncc.retryPods = ncc.newRetryFramework(factory.PodType, true) - ipAllocator, err := newIPAllocatorForNetwork(ncc.NetInfo) + ipAllocator, err := newIPAllocatorForNetwork(ncc.GetNetInfo()) if err != nil { - return fmt.Errorf("could not initialize the IP allocator for network %q: %w", ncc.NetInfo.GetNetworkName(), err) + return fmt.Errorf("could not initialize the IP allocator for network %q: %w", ncc.GetNetworkName(), err) } ncc.subnetAllocator = ipAllocator @@ -234,21 +236,28 @@ func (ncc *networkClusterController) init() error { ncc.retryIPAMClaims = ncc.newRetryFramework(factory.IPAMClaimsType, true) ncc.ipamClaimReconciler = persistentips.NewIPAMClaimReconciler( ncc.kube, - ncc.NetInfo, + ncc.GetNetInfo(), ncc.watchFactory.IPAMClaimsInformer().Lister(), ) ipamClaimsReconciler = ncc.ipamClaimReconciler } podAllocationAnnotator = annotationalloc.NewPodAnnotationAllocator( - ncc.NetInfo, + ncc.GetNetInfo(), ncc.watchFactory.PodCoreInformer().Lister(), ncc.kube, ipamClaimsReconciler, ) - ncc.podAllocator = pod.NewPodAllocator(ncc.NetInfo, podAllocationAnnotator, ipAllocator, - ipamClaimsReconciler, ncc.nadController, ncc.recorder, ncc.tunnelIDAllocator) + ncc.podAllocator = pod.NewPodAllocator( + ncc.GetNetInfo(), + podAllocationAnnotator, + ipAllocator, + ipamClaimsReconciler, + ncc.networkManager, + ncc.recorder, + ncc.tunnelIDAllocator, + ) if err := ncc.podAllocator.Init(); err != nil { return fmt.Errorf("failed to initialize pod ip allocator: %w", err) } @@ -309,7 +318,7 @@ func (ncc *networkClusterController) updateNetworkStatus(nodeName string, handle }) } - netName := ncc.NetInfo.GetNetworkName() + netName := ncc.GetNetworkName() if err := ncc.statusReporter(netName, "NetworkClusterController", condition, events...); err != nil { return fmt.Errorf("failed to report network status: %w", err) } @@ -324,7 +333,7 @@ func (ncc *networkClusterController) resetStatus() error { if ncc.statusReporter == nil { return nil } - netName := ncc.NetInfo.GetNetworkName() + netName := ncc.GetNetworkName() return ncc.statusReporter(netName, "NetworkClusterController", getNetworkAllocationUDNCondition("")) } @@ -435,6 +444,15 @@ func (ncc *networkClusterController) Cleanup() error { return nil } +func (ncc *networkClusterController) Reconcile(netInfo util.NetInfo) error { + // update network informaiton, point of no return + err := util.ReconcileNetInfo(ncc.ReconcilableNetInfo, netInfo) + if err != nil { + klog.Errorf("Failed to reconcile network %s: %v", ncc.GetNetworkName(), err) + } + return nil +} + // networkClusterControllerEventHandler object handles the events // from retry framework. type networkClusterControllerEventHandler struct { @@ -557,7 +575,7 @@ func (h *networkClusterControllerEventHandler) DeleteResource(obj, cachedObj int return fmt.Errorf("could not cast obj of type %T to *ipamclaimsapi.IPAMClaim", obj) } - ipAllocator := h.ncc.subnetAllocator.ForSubnet(h.ncc.NetInfo.GetNetworkName()) + ipAllocator := h.ncc.subnetAllocator.ForSubnet(h.ncc.GetNetworkName()) err := h.ncc.ipamClaimReconciler.Reconcile(ipamClaim, nil, ipAllocator) if err != nil && !errors.Is(err, persistentips.ErrIgnoredIPAMClaim) { return fmt.Errorf("error deleting IPAMClaim: %w", err) @@ -585,7 +603,7 @@ func (h *networkClusterControllerEventHandler) SyncFunc(objs []interface{}) erro syncFunc = func(claims []interface{}) error { return h.ncc.ipamClaimReconciler.Sync( claims, - h.ncc.subnetAllocator.ForSubnet(h.ncc.NetInfo.GetNetworkName()), + h.ncc.subnetAllocator.ForSubnet(h.ncc.GetNetworkName()), ) } diff --git a/go-controller/pkg/clustermanager/pod/allocator.go b/go-controller/pkg/clustermanager/pod/allocator.go index 24b9591aff..01af49f594 100644 --- a/go-controller/pkg/clustermanager/pod/allocator.go +++ b/go-controller/pkg/clustermanager/pod/allocator.go @@ -18,7 +18,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/ip/subnet" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/allocator/pod" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/persistentips" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -41,7 +41,7 @@ type PodAllocator struct { ipamClaimsReconciler persistentips.PersistentAllocations - nadController nad.NADController + networkManager networkmanager.Interface // event recorder used to post events to k8s recorder record.EventRecorder @@ -58,7 +58,7 @@ func NewPodAllocator( podAnnotationAllocator *pod.PodAnnotationAllocator, ipAllocator subnet.Allocator, claimsReconciler persistentips.PersistentAllocations, - nadController nad.NADController, + networkManager networkmanager.Interface, recorder record.EventRecorder, idAllocator id.Allocator, ) *PodAllocator { @@ -67,7 +67,7 @@ func NewPodAllocator( releasedPods: map[string]sets.Set[string]{}, releasedPodsMutex: sync.Mutex{}, podAnnotationAllocator: podAnnotationAllocator, - nadController: nadController, + networkManager: networkManager, recorder: recorder, idAllocator: idAllocator, } @@ -95,10 +95,10 @@ func (a *PodAllocator) Init() error { return nil } -// getActiveNetworkForNamespace returns the active network for the given pod's namespace +// getActiveNetworkForPod returns the active network for the given pod's namespace // and is a wrapper around GetActiveNetworkForNamespace func (a *PodAllocator) getActiveNetworkForPod(pod *corev1.Pod) (util.NetInfo, error) { - activeNetwork, err := a.nadController.GetActiveNetworkForNamespace(pod.Namespace) + activeNetwork, err := a.networkManager.GetActiveNetworkForNamespace(pod.Namespace) if err != nil { if util.IsUnprocessedActiveNetworkError(err) { a.recordPodErrorEvent(pod, err) diff --git a/go-controller/pkg/clustermanager/pod/allocator_test.go b/go-controller/pkg/clustermanager/pod/allocator_test.go index 14a1a947f2..ec59af36ed 100644 --- a/go-controller/pkg/clustermanager/pod/allocator_test.go +++ b/go-controller/pkg/clustermanager/pod/allocator_test.go @@ -18,7 +18,6 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/persistentips" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -37,6 +36,7 @@ import ( kubemocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube/mocks" v1mocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/client-go/listers/core/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/networkmanager" ) type testPod struct { @@ -558,7 +558,9 @@ func TestPodAllocator_reconcileForNAD(t *testing.T) { if err != nil { t.Fatalf("Invalid netConf") } - netInfo.AddNADs("namespace/nad") + mutableNetInfo := util.NewMutableNetInfo(netInfo) + mutableNetInfo.AddNADs("namespace/nad") + netInfo = mutableNetInfo var ipamClaimsReconciler persistentips.PersistentAllocations if tt.ipam && tt.args.ipamClaim != nil { @@ -592,7 +594,7 @@ func TestPodAllocator_reconcileForNAD(t *testing.T) { } } - nadController := &nad.FakeNADController{PrimaryNetworks: nadNetworks} + fakeNetworkManager := &networkmanager.FakeNetworkManager{PrimaryNetworks: nadNetworks} fakeRecorder := record.NewFakeRecorder(10) @@ -607,8 +609,8 @@ func TestPodAllocator_reconcileForNAD(t *testing.T) { releasedPods: map[string]sets.Set[string]{}, releasedPodsMutex: sync.Mutex{}, ipamClaimsReconciler: ipamClaimsReconciler, + networkManager: fakeNetworkManager, recorder: fakeRecorder, - nadController: nadController, } var old, new *corev1.Pod diff --git a/go-controller/pkg/clustermanager/secondary_network_cluster_manager.go b/go-controller/pkg/clustermanager/secondary_network_cluster_manager.go index 36201b2df9..5bc7448905 100644 --- a/go-controller/pkg/clustermanager/secondary_network_cluster_manager.go +++ b/go-controller/pkg/clustermanager/secondary_network_cluster_manager.go @@ -11,7 +11,7 @@ import ( ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -22,13 +22,13 @@ const ( ) // secondaryNetworkClusterManager object manages the multi net-attach-def controllers. -// It implements networkAttachDefController.NetworkControllerManager and can be used -// by NetAttachDefinitionController to add and delete NADs. +// It implements networkmanager.ControllerManager interface and can be used +// by network manager to create and delete network controllers. type secondaryNetworkClusterManager struct { - // net-attach-def controller handle net-attach-def and create/delete network controllers - nadController *nad.NetAttachDefinitionController - ovnClient *util.OVNClusterManagerClientset - watchFactory *factory.WatchFactory + // networkManager creates and deletes network controllers + networkManager networkmanager.Interface + ovnClient *util.OVNClusterManagerClientset + watchFactory *factory.WatchFactory // networkIDAllocator is used to allocate a unique ID for each secondary layer3 network networkIDAllocator id.Allocator @@ -38,8 +38,12 @@ type secondaryNetworkClusterManager struct { errorReporter NetworkStatusReporter } -func newSecondaryNetworkClusterManager(ovnClient *util.OVNClusterManagerClientset, wf *factory.WatchFactory, - recorder record.EventRecorder) (*secondaryNetworkClusterManager, error) { +func newSecondaryNetworkClusterManager( + ovnClient *util.OVNClusterManagerClientset, + wf *factory.WatchFactory, + networkManager networkmanager.Interface, + recorder record.EventRecorder, +) (*secondaryNetworkClusterManager, error) { klog.Infof("Creating secondary network cluster manager") networkIDAllocator := id.NewIDAllocator("NetworkIDs", maxSecondaryNetworkIDs) // Reserve the id 0 for the default network. @@ -50,13 +54,9 @@ func newSecondaryNetworkClusterManager(ovnClient *util.OVNClusterManagerClientse ovnClient: ovnClient, watchFactory: wf, networkIDAllocator: networkIDAllocator, + networkManager: networkManager, recorder: recorder, } - var err error - sncm.nadController, err = nad.NewNetAttachDefinitionController("cluster-manager", sncm, wf, recorder) - if err != nil { - return nil, err - } return sncm, nil } @@ -68,13 +68,7 @@ func (sncm *secondaryNetworkClusterManager) SetNetworkStatusReporter(errorReport // needed logical entities func (sncm *secondaryNetworkClusterManager) Start() error { klog.Infof("Starting secondary network cluster manager") - - err := sncm.init() - if err != nil { - return err - } - - return sncm.nadController.Start() + return sncm.init() } func (sncm *secondaryNetworkClusterManager) init() error { @@ -103,22 +97,31 @@ func (sncm *secondaryNetworkClusterManager) init() error { func (sncm *secondaryNetworkClusterManager) Stop() { klog.Infof("Stopping secondary network cluster manager") - sncm.nadController.Stop() } -// NewNetworkController implements the networkAttachDefController.NetworkControllerManager -// interface function. This function is called by the net-attach-def controller when -// a secondary network is created. -func (sncm *secondaryNetworkClusterManager) NewNetworkController(nInfo util.NetInfo) (nad.NetworkController, error) { +func (sncm *secondaryNetworkClusterManager) GetDefaultNetworkController() networkmanager.ReconcilableNetworkController { + return nil +} + +// NewNetworkController implements the networkmanager.ControllerManager +// interface called by network manager to create or delete a network controller. +func (sncm *secondaryNetworkClusterManager) NewNetworkController(nInfo util.NetInfo) (networkmanager.NetworkController, error) { if !sncm.isTopologyManaged(nInfo) { - return nil, nad.ErrNetworkControllerTopologyNotManaged + return nil, networkmanager.ErrNetworkControllerTopologyNotManaged } klog.Infof("Creating new network controller for network %s of topology %s", nInfo.GetNetworkName(), nInfo.TopologyType()) namedIDAllocator := sncm.networkIDAllocator.ForName(nInfo.GetNetworkName()) - sncc := newNetworkClusterController(namedIDAllocator, nInfo, sncm.ovnClient, sncm.watchFactory, sncm.recorder, - sncm.nadController, sncm.errorReporter) + sncc := newNetworkClusterController( + namedIDAllocator, + nInfo, + sncm.ovnClient, + sncm.watchFactory, + sncm.recorder, + sncm.networkManager, + sncm.errorReporter, + ) return sncc, nil } @@ -139,15 +142,16 @@ func (sncm *secondaryNetworkClusterManager) isTopologyManaged(nInfo util.NetInfo return false } -// CleanupDeletedNetworks implements the networkAttachDefController.NetworkControllerManager -// interface function. -func (sncm *secondaryNetworkClusterManager) CleanupDeletedNetworks(validNetworks ...util.BasicNetInfo) error { +// CleanupStaleNetworks cleans of stale data from the OVN database +// corresponding to networks not included in validNetworks, which are considered +// stale. +func (sncm *secondaryNetworkClusterManager) CleanupStaleNetworks(validNetworks ...util.NetInfo) error { existingNetworksMap := map[string]struct{}{} for _, network := range validNetworks { existingNetworksMap[network.GetNetworkName()] = struct{}{} } - staleNetworkControllers := map[string]nad.NetworkController{} + staleNetworkControllers := map[string]networkmanager.NetworkController{} existingNodes, err := sncm.watchFactory.GetNodes() if err != nil { return err @@ -195,11 +199,18 @@ func (sncm *secondaryNetworkClusterManager) CleanupDeletedNetworks(validNetworks } // newDummyNetworkController creates a dummy network controller used to clean up specific network -func (sncm *secondaryNetworkClusterManager) newDummyLayer3NetworkController(netName string) (nad.NetworkController, error) { +func (sncm *secondaryNetworkClusterManager) newDummyLayer3NetworkController(netName string) (networkmanager.NetworkController, error) { netInfo, _ := util.NewNetInfo(&ovncnitypes.NetConf{NetConf: types.NetConf{Name: netName}, Topology: ovntypes.Layer3Topology}) namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) - nc := newNetworkClusterController(namedIDAllocator, netInfo, sncm.ovnClient, sncm.watchFactory, sncm.recorder, - sncm.nadController, nil) + nc := newNetworkClusterController( + namedIDAllocator, + netInfo, + sncm.ovnClient, + sncm.watchFactory, + sncm.recorder, + sncm.networkManager, + nil, + ) err := nc.init() return nc, err } diff --git a/go-controller/pkg/clustermanager/secondary_network_unit_test.go b/go-controller/pkg/clustermanager/secondary_network_unit_test.go index f1c98c643b..21a80a26a0 100644 --- a/go-controller/pkg/clustermanager/secondary_network_unit_test.go +++ b/go-controller/pkg/clustermanager/secondary_network_unit_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "net" - "strings" "sync" "github.com/containernetworking/cni/pkg/types" @@ -24,7 +23,8 @@ import ( ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -75,7 +75,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { err = f.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) netInfo, err := util.NewNetInfo(&ovncnitypes.NetConf{NetConf: types.NetConf{Name: "blue"}, Topology: ovntypes.Layer3Topology, Subnets: "192.168.0.0/16/24"}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -144,7 +144,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { err = f.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) config.OVNKubernetesFeature.EnableInterconnect = false @@ -182,7 +182,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(f.Start()).NotTo(gomega.HaveOccurred()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) @@ -192,7 +192,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm.ovnClient, sncm.watchFactory, sncm.recorder, - sncm.nadController, + sncm.networkManager, nil, ) gomega.Expect(nc.init()).To(gomega.Succeed()) @@ -200,8 +200,9 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { namedSubnetAllocator := nc.subnetAllocator.ForSubnet(netInfo.GetNetworkName()) - firstAllocatableIPs := "192.168.200.1/24,fd12:1234::1/64" - allocatableIPs, err := util.ParseIPNets(strings.Split(firstAllocatableIPs, ",")) + firstAllocatableIPs := []string{"192.168.200.1/24", "fd12:1234::1/64"} + allocatableIPs, err := util.ParseIPNets(firstAllocatableIPs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(namedSubnetAllocator.AllocateIPs(allocatableIPs)).To(gomega.Succeed()) return nil } @@ -242,7 +243,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { err = f.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) _, err = sncm.NewNetworkController(netInfo) @@ -380,7 +381,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { gomega.Eventually(checkNodeAnnotations).ShouldNot(gomega.HaveOccurred()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = sncm.init() @@ -402,13 +403,13 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm.ovnClient, sncm.watchFactory, sncm.recorder, - sncm.nadController, + sncm.networkManager, nil, ) err = oc.init() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = sncm.CleanupDeletedNetworks(oc) + err = sncm.CleanupStaleNetworks(oc) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Clean up the red network @@ -420,7 +421,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { // Now call CleanupDeletedNetworks() with empty nad controllers. // Blue network should also be cleared. - err = sncm.CleanupDeletedNetworks() + err = sncm.CleanupStaleNetworks() gomega.Expect(err).NotTo(gomega.HaveOccurred()) expectBlueCleanup = true @@ -487,7 +488,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(f.Start()).To(gomega.Succeed()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) @@ -497,7 +498,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm.ovnClient, sncm.watchFactory, sncm.recorder, - sncm.nadController, + sncm.networkManager, nil, ) gomega.Expect(nc.init()).To(gomega.Succeed()) @@ -542,7 +543,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(f.Start()).To(gomega.Succeed()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) @@ -552,7 +553,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm.ovnClient, sncm.watchFactory, sncm.recorder, - sncm.nadController, + sncm.networkManager, nil, ) gomega.Expect(nc.init()).To(gomega.Succeed()) @@ -593,7 +594,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(f.Start()).To(gomega.Succeed()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) @@ -603,7 +604,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm.ovnClient, sncm.watchFactory, sncm.recorder, - sncm.nadController, + sncm.networkManager, nil, ) gomega.Expect(nc.init()).To(gomega.Succeed()) @@ -664,7 +665,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(f.Start()).To(gomega.Succeed()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) @@ -674,7 +675,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm.ovnClient, sncm.watchFactory, sncm.recorder, - sncm.nadController, + sncm.networkManager, nil, ) gomega.Expect(nc.init()).To(gomega.Succeed()) @@ -738,7 +739,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(f.Start()).To(gomega.Succeed()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) @@ -748,7 +749,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm.ovnClient, sncm.watchFactory, sncm.recorder, - sncm.nadController, + sncm.networkManager, nil, ) gomega.Expect(nc.init()).To(gomega.Succeed()) @@ -806,7 +807,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(f.Start()).NotTo(gomega.HaveOccurred()) - sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, recorder) + sncm, err := newSecondaryNetworkClusterManager(fakeClient, f, networkmanager.Default().Interface(), recorder) gomega.Expect(err).NotTo(gomega.HaveOccurred()) namedIDAllocator := sncm.networkIDAllocator.ForName(netInfo.GetNetworkName()) @@ -816,7 +817,7 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { sncm.ovnClient, sncm.watchFactory, sncm.recorder, - sncm.nadController, + sncm.networkManager, nil, ) gomega.Expect(nc.init()).To(gomega.Succeed()) @@ -837,8 +838,9 @@ var _ = ginkgo.Describe("Cluster Controller Manager", func() { ) } - firstAllocatableIPs := "192.168.200.3/24,fd12:1234::3/64" - allocatableIPs, err := util.ParseIPNets(strings.Split(firstAllocatableIPs, ",")) + firstAllocatableIPs := []string{"192.168.200.3/24", "fd12:1234::3/64"} + allocatableIPs, err := util.ParseIPNets(firstAllocatableIPs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(namedSubnetAllocator.AllocateIPs(allocatableIPs)).To(gomega.Succeed()) return nil } diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go index 3dd14f1902..bd3fb29d75 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go @@ -179,7 +179,7 @@ var _ = Describe("User Defined Network Controller", func() { mutatedNAD := expectedNAD.DeepCopy() mutatedNAD.Spec.Config = "MUTATED" - mutatedNAD, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(udn.Namespace).Update(context.Background(), mutatedNAD, metav1.UpdateOptions{}) + _, err := cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(udn.Namespace).Update(context.Background(), mutatedNAD, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) Eventually(func() *netv1.NetworkAttachmentDefinition { @@ -349,7 +349,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) return udn.Finalizers }).Should(BeEmpty(), "should remove finalizer on UDN following deletion and not being used") - nad, err = cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(nad.Namespace).Get(context.Background(), nad.Name, metav1.GetOptions{}) + _, err = cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(nad.Namespace).Get(context.Background(), nad.Name, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) Expect(kerrors.IsNotFound(err)).To(BeTrue()) }) @@ -478,7 +478,7 @@ var _ = Describe("User Defined Network Controller", func() { newNsNames := []string{"black", "gray"} for _, nsName := range newNsNames { ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName, Labels: newNsLabel}} - ns, err := cs.KubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) + _, err := cs.KubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) } @@ -487,6 +487,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) cudn.Spec.NamespaceSelector.MatchExpressions[0].Values = append(cudn.Spec.NamespaceSelector.MatchExpressions[0].Values, newNsLabelValue) cudn, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Update(context.Background(), cudn, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) Expect(cudn.Spec.NamespaceSelector.MatchExpressions).To(Equal([]metav1.LabelSelectorRequirement{{ Key: testLabelKey, Operator: metav1.LabelSelectorOpIn, @@ -518,6 +519,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(err).NotTo(HaveOccurred()) cudn.Spec.NamespaceSelector.MatchExpressions[0].Values = []string{""} cudn, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Update(context.Background(), cudn, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) Expect(cudn.Spec.NamespaceSelector.MatchExpressions).To(Equal([]metav1.LabelSelectorRequirement{{ Key: testLabelKey, Operator: metav1.LabelSelectorOpIn, Values: []string{""}, }})) @@ -544,7 +546,7 @@ var _ = Describe("User Defined Network Controller", func() { var testPods []corev1.Pod for _, nsName := range connectedNsNames { pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("pod-0"), + Name: "pod-0", Namespace: nsName, Annotations: map[string]string{util.OvnPodAnnotationName: `{"default": {"role":"primary"}, "` + nsName + `/` + cudnName + `": {"role": "secondary"}}`}}, } @@ -610,7 +612,7 @@ var _ = Describe("User Defined Network Controller", func() { for _, nsName := range newNsNames { ns := testNamespace(nsName) ns.Labels = map[string]string{testLabelKey: testLabelValue} - ns, err := cs.KubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) + _, err := cs.KubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) } @@ -663,7 +665,7 @@ var _ = Describe("User Defined Network Controller", func() { By("remove label from few connected namespaces") for _, nsName := range staleNADNsNames { - p := fmt.Sprintf(`[{"op": "replace", "path": "./metadata/labels", "value": {}}]`) + p := `[{"op": "replace", "path": "./metadata/labels", "value": {}}]` ns, err := cs.KubeClient.CoreV1().Namespaces().Patch(context.Background(), nsName, types.JSONPatchType, []byte(p), metav1.PatchOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(ns.Labels).To(BeEmpty()) @@ -776,7 +778,7 @@ var _ = Describe("User Defined Network Controller", func() { _, err := c.syncUserDefinedNetwork(udn) Expect(err).ToNot(HaveOccurred()) - nad, err = cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(udn.Namespace).Get(context.Background(), nad.Name, metav1.GetOptions{}) + _, err = cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(udn.Namespace).Get(context.Background(), nad.Name, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) Expect(kerrors.IsNotFound(err)).To(BeTrue()) }) @@ -847,7 +849,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(udn.Finalizers).To(BeEmpty()) - nad, err = cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(udn.Namespace).Get(context.Background(), nad.Name, metav1.GetOptions{}) + _, err = cs.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(udn.Namespace).Get(context.Background(), nad.Name, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) Expect(kerrors.IsNotFound(err)).To(BeTrue()) }) @@ -1241,6 +1243,8 @@ func assertConditionReportNetworkInUse(conditions []metav1.Condition, messageNAD } func assertUserDefinedNetworkStatus(udnClient udnclient.Interface, udn *udnv1.UserDefinedNetwork, expectedStatus *udnv1.UserDefinedNetworkStatus) { + GinkgoHelper() + actualUDN, err := udnClient.K8sV1().UserDefinedNetworks(udn.Namespace).Get(context.Background(), udn.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1255,6 +1259,8 @@ func assertFinalizersPresent( udn *udnv1.UserDefinedNetwork, pods ...*corev1.Pod, ) { + GinkgoHelper() + var podNames []string for _, pod := range pods { podNames = append(podNames, pod.Namespace+"/"+pod.Name) diff --git a/go-controller/pkg/cni/cni.go b/go-controller/pkg/cni/cni.go index cb615ff501..10c09897d6 100644 --- a/go-controller/pkg/cni/cni.go +++ b/go-controller/pkg/cni/cni.go @@ -19,7 +19,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kubevirt" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -106,12 +106,16 @@ func (pr *PodRequest) checkOrUpdatePodUID(pod *kapi.Pod) error { return nil } -func (pr *PodRequest) cmdAdd(kubeAuth *KubeAPIAuth, clientset *ClientSet, - nadController *nad.NetAttachDefinitionController) (*Response, error) { - return pr.cmdAddWithGetCNIResultFunc(kubeAuth, clientset, getCNIResult, nadController) +func (pr *PodRequest) cmdAdd(kubeAuth *KubeAPIAuth, clientset *ClientSet, networkManager networkmanager.Interface) (*Response, error) { + return pr.cmdAddWithGetCNIResultFunc(kubeAuth, clientset, getCNIResult, networkManager) } -func (pr *PodRequest) cmdAddWithGetCNIResultFunc(kubeAuth *KubeAPIAuth, clientset *ClientSet, - getCNIResultFn getCNIResultFunc, nadController nad.NADController) (*Response, error) { + +func (pr *PodRequest) cmdAddWithGetCNIResultFunc( + kubeAuth *KubeAPIAuth, + clientset *ClientSet, + getCNIResultFn getCNIResultFunc, + networkManager networkmanager.Interface, +) (*Response, error) { namespace := pr.PodNamespace podName := pr.PodName if namespace == "" || podName == "" { @@ -144,7 +148,7 @@ func (pr *PodRequest) cmdAddWithGetCNIResultFunc(kubeAuth *KubeAPIAuth, clientse // Get the IP address and MAC address of the pod // for DPU, ensure connection-details is present - primaryUDN := udn.NewPrimaryNetwork(nadController) + primaryUDN := udn.NewPrimaryNetwork(networkManager) if util.IsNetworkSegmentationSupportEnabled() { annotCondFn = primaryUDN.WaitForPrimaryAnnotationFn(namespace, annotCondFn) } @@ -294,7 +298,12 @@ func (pr *PodRequest) cmdCheck() error { // Argument '*PodRequest' encapsulates all the necessary information // kclient is passed in so that clientset can be reused from the server // Return value is the actual bytes to be sent back without further processing. -func HandlePodRequest(request *PodRequest, clientset *ClientSet, kubeAuth *KubeAPIAuth, nadController *nad.NetAttachDefinitionController) ([]byte, error) { +func HandlePodRequest( + request *PodRequest, + clientset *ClientSet, + kubeAuth *KubeAPIAuth, + networkManager networkmanager.Interface, +) ([]byte, error) { var result, resultForLogging []byte var response *Response var err, err1 error @@ -302,7 +311,7 @@ func HandlePodRequest(request *PodRequest, clientset *ClientSet, kubeAuth *KubeA klog.Infof("%s %s starting CNI request %+v", request, request.Command, request) switch request.Command { case CNIAdd: - response, err = request.cmdAdd(kubeAuth, clientset, nadController) + response, err = request.cmdAdd(kubeAuth, clientset, networkManager) case CNIDel: response, err = request.cmdDel(clientset) case CNICheck: diff --git a/go-controller/pkg/cni/cni_test.go b/go-controller/pkg/cni/cni_test.go index 78c97270d4..fcb17ff0fb 100644 --- a/go-controller/pkg/cni/cni_test.go +++ b/go-controller/pkg/cni/cni_test.go @@ -21,9 +21,10 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" v1nadmocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" v1mocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/client-go/listers/core/v1" - ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" + testnm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/networkmanager" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -57,7 +58,6 @@ var _ = Describe("Network Segmentation", func() { } prInterfaceOpsStub = &podRequestInterfaceOpsStub{} enableMultiNetwork, enableNetworkSegmentation bool - nadController *ovntest.FakeNADController ) BeforeEach(func() { @@ -126,7 +126,7 @@ var _ = Describe("Network Segmentation", func() { }) It("should not fail at cmdAdd", func() { podNamespaceLister.On("Get", pr.PodName).Return(pod, nil) - Expect(pr.cmdAddWithGetCNIResultFunc(kubeAuth, clientSet, getCNIResultStub, nil)).NotTo(BeNil()) + Expect(pr.cmdAddWithGetCNIResultFunc(kubeAuth, clientSet, getCNIResultStub, networkmanager.Default().Interface())).NotTo(BeNil()) Expect(obtainedPodIterfaceInfos).ToNot(BeEmpty()) }) It("should not fail at cmdDel", func() { @@ -153,13 +153,10 @@ var _ = Describe("Network Segmentation", func() { }, }, } - nadController = &ovntest.FakeNADController{ - PrimaryNetworks: make(map[string]util.NetInfo), - } }) It("should not fail at cmdAdd", func() { podNamespaceLister.On("Get", pr.PodName).Return(pod, nil) - Expect(pr.cmdAddWithGetCNIResultFunc(kubeAuth, clientSet, getCNIResultStub, nadController)).NotTo(BeNil()) + Expect(pr.cmdAddWithGetCNIResultFunc(kubeAuth, clientSet, getCNIResultStub, networkmanager.Default().Interface())).NotTo(BeNil()) Expect(obtainedPodIterfaceInfos).ToNot(BeEmpty()) }) It("should not fail at cmdDel", func() { @@ -176,6 +173,8 @@ var _ = Describe("Network Segmentation", func() { namespace = "foo-ns" ) + var fakeNetworkManager *testnm.FakeNetworkManager + dummyGetCNIResult := func(request *PodRequest, getter PodInfoGetter, podInterfaceInfo *PodInterfaceInfo) (*current.Result, error) { var gatewayIP net.IP if len(podInterfaceInfo.Gateways) > 0 { @@ -233,16 +232,16 @@ var _ = Describe("Network Segmentation", func() { nadLister.On("NetworkAttachmentDefinitions", "foo-ns").Return(nadNamespaceLister) nadNetwork, err := util.ParseNADInfo(nad) Expect(err).NotTo(HaveOccurred()) - nadController = &ovntest.FakeNADController{ + fakeNetworkManager = &testnm.FakeNetworkManager{ PrimaryNetworks: make(map[string]util.NetInfo), } - nadController.PrimaryNetworks[nad.Namespace] = nadNetwork + fakeNetworkManager.PrimaryNetworks[nad.Namespace] = nadNetwork getCNIResultStub = dummyGetCNIResult }) It("should return the information of both the default net and the primary UDN in the result", func() { podNamespaceLister.On("Get", pr.PodName).Return(pod, nil) - response, err := pr.cmdAddWithGetCNIResultFunc(kubeAuth, clientSet, getCNIResultStub, nadController) + response, err := pr.cmdAddWithGetCNIResultFunc(kubeAuth, clientSet, getCNIResultStub, fakeNetworkManager) Expect(err).NotTo(HaveOccurred()) // for every interface added, we return 2 interfaces; the host side of the // veth, then the pod side of the veth. diff --git a/go-controller/pkg/cni/cniserver.go b/go-controller/pkg/cni/cniserver.go index bf30f4d207..776c688dfb 100644 --- a/go-controller/pkg/cni/cniserver.go +++ b/go-controller/pkg/cni/cniserver.go @@ -18,7 +18,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -49,8 +49,11 @@ import ( // started. // NewCNIServer creates and returns a new Server object which will listen on a socket in the given path -func NewCNIServer(factory factory.NodeWatchFactory, kclient kubernetes.Interface, - nadController *nad.NetAttachDefinitionController) (*Server, error) { +func NewCNIServer( + factory factory.NodeWatchFactory, + kclient kubernetes.Interface, + networkManager networkmanager.Interface, +) (*Server, error) { if config.OvnKubeNode.Mode == types.NodeModeDPU { return nil, fmt.Errorf("unsupported ovnkube-node mode for CNI server: %s", config.OvnKubeNode.Mode) } @@ -72,10 +75,7 @@ func NewCNIServer(factory factory.NodeWatchFactory, kclient kubernetes.Interface KubeAPITokenFile: config.Kubernetes.TokenFile, }, handlePodRequestFunc: HandlePodRequest, - } - - if util.IsNetworkSegmentationSupportEnabled() { - s.nadController = nadController + networkManager: networkManager, } if len(config.Kubernetes.CAData) > 0 { @@ -221,7 +221,7 @@ func (s *Server) handleCNIRequest(r *http.Request) ([]byte, error) { } defer req.cancel() - result, err := s.handlePodRequestFunc(req, s.clientSet, s.kubeAuth, s.nadController) + result, err := s.handlePodRequestFunc(req, s.clientSet, s.kubeAuth, s.networkManager) if err != nil { // Prefix error with request information for easier debugging return nil, fmt.Errorf("%s %v", req, err) diff --git a/go-controller/pkg/cni/cniserver_test.go b/go-controller/pkg/cni/cniserver_test.go index 71dd2f4ede..c1eebcad65 100644 --- a/go-controller/pkg/cni/cniserver_test.go +++ b/go-controller/pkg/cni/cniserver_test.go @@ -23,7 +23,7 @@ import ( nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -33,7 +33,7 @@ func clientDoCNI(t *testing.T, client *http.Client, req *Request) ([]byte, int) t.Fatalf("failed to marshal CNI request %v: %v", req, err) } - url := fmt.Sprintf("http://dummy/") + url := "http://dummy/" resp, err := client.Post(url, "application/json", bytes.NewReader(data)) if err != nil { t.Fatalf("failed to send CNI request: %v", err) @@ -49,7 +49,7 @@ func clientDoCNI(t *testing.T, client *http.Client, req *Request) ([]byte, int) var expectedResult cnitypes.Result -func serverHandleCNI(request *PodRequest, clientset *ClientSet, kubeAuth *KubeAPIAuth, nadController *nad.NetAttachDefinitionController) ([]byte, error) { +func serverHandleCNI(request *PodRequest, clientset *ClientSet, kubeAuth *KubeAPIAuth, networkManager networkmanager.Interface) ([]byte, error) { if request.Command == CNIAdd { return json.Marshal(&expectedResult) } else if request.Command == CNIDel || request.Command == CNIUpdate || request.Command == CNICheck { @@ -91,7 +91,7 @@ func TestCNIServer(t *testing.T) { t.Fatalf("failed to start watch factory: %v", err) } - s, err := NewCNIServer(wf, fakeClient, nil) + s, err := NewCNIServer(wf, fakeClient, networkmanager.Default().Interface()) if err != nil { t.Fatalf("error creating CNI server: %v", err) } diff --git a/go-controller/pkg/cni/helper_linux.go b/go-controller/pkg/cni/helper_linux.go index 9f160a4b52..e7fe81128a 100644 --- a/go-controller/pkg/cni/helper_linux.go +++ b/go-controller/pkg/cni/helper_linux.go @@ -519,6 +519,7 @@ func ConfigureOVS(ctx context.Context, namespace, podName, hostIfaceName string, } else { dpdkArgs := []string{"type=dpdk"} ovsArgs = append(ovsArgs, dpdkArgs...) + ovsArgs = append(ovsArgs, fmt.Sprintf("mtu_request=%v", ifInfo.MTU)) } } diff --git a/go-controller/pkg/cni/types.go b/go-controller/pkg/cni/types.go index edff41df08..feb539c8c2 100644 --- a/go-controller/pkg/cni/types.go +++ b/go-controller/pkg/cni/types.go @@ -12,7 +12,7 @@ import ( "k8s.io/klog/v2" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" @@ -168,7 +168,7 @@ type PodRequest struct { deviceInfo nadapi.DeviceInfo } -type podRequestFunc func(request *PodRequest, clientset *ClientSet, kubeAuth *KubeAPIAuth, nadController *nad.NetAttachDefinitionController) ([]byte, error) +type podRequestFunc func(request *PodRequest, clientset *ClientSet, kubeAuth *KubeAPIAuth, networkManager networkmanager.Interface) ([]byte, error) type getCNIResultFunc func(request *PodRequest, getter PodInfoGetter, podInterfaceInfo *PodInterfaceInfo) (*current.Result, error) type PodInfoGetter interface { @@ -195,5 +195,5 @@ type Server struct { handlePodRequestFunc podRequestFunc clientSet *ClientSet kubeAuth *KubeAPIAuth - nadController *nad.NetAttachDefinitionController + networkManager networkmanager.Interface } diff --git a/go-controller/pkg/cni/udn/primary_network.go b/go-controller/pkg/cni/udn/primary_network.go index f6718a7ac3..f9ff93d6f9 100644 --- a/go-controller/pkg/cni/udn/primary_network.go +++ b/go-controller/pkg/cni/udn/primary_network.go @@ -5,7 +5,7 @@ import ( "k8s.io/klog/v2" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -14,14 +14,14 @@ import ( type podAnnotWaitCond = func(map[string]string, string) (*util.PodAnnotation, bool) type UserDefinedPrimaryNetwork struct { - nadController nad.NADController - annotation *util.PodAnnotation - activeNetwork util.NetInfo + networkManager networkmanager.Interface + annotation *util.PodAnnotation + activeNetwork util.NetInfo } -func NewPrimaryNetwork(nadController nad.NADController) *UserDefinedPrimaryNetwork { +func NewPrimaryNetwork(networkManager networkmanager.Interface) *UserDefinedPrimaryNetwork { return &UserDefinedPrimaryNetwork{ - nadController: nadController, + networkManager: networkManager, } } @@ -123,7 +123,7 @@ func (p *UserDefinedPrimaryNetwork) ensureActiveNetwork(namespace string) error if p.activeNetwork != nil { return nil } - activeNetwork, err := p.nadController.GetActiveNetworkForNamespace(namespace) + activeNetwork, err := p.networkManager.GetActiveNetworkForNamespace(namespace) if err != nil { return err } diff --git a/go-controller/pkg/cni/udn/primary_network_test.go b/go-controller/pkg/cni/udn/primary_network_test.go index 08cb692fa4..a42ae242b5 100644 --- a/go-controller/pkg/cni/udn/primary_network_test.go +++ b/go-controller/pkg/cni/udn/primary_network_test.go @@ -10,7 +10,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" v1nadmocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/networkmanager" types "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -188,20 +188,22 @@ func TestWaitForPrimaryAnnotationFn(t *testing.T) { return tt.annotationFromFn, tt.isReadyFromFn } - nadController := &nad.FakeNADController{ + fakeNetworkManager := &networkmanager.FakeNetworkManager{ PrimaryNetworks: map[string]util.NetInfo{}, } for _, nad := range tt.nads { nadNetwork, _ := util.ParseNADInfo(nad) - nadNetwork.SetNADs(util.GetNADName(nad.Namespace, nad.Name)) + mutableNetInfo := util.NewMutableNetInfo(nadNetwork) + mutableNetInfo.SetNADs(util.GetNADName(nad.Namespace, nad.Name)) + nadNetwork = mutableNetInfo if nadNetwork.IsPrimaryNetwork() { - if _, loaded := nadController.PrimaryNetworks[nad.Namespace]; !loaded { - nadController.PrimaryNetworks[nad.Namespace] = nadNetwork + if _, loaded := fakeNetworkManager.PrimaryNetworks[nad.Namespace]; !loaded { + fakeNetworkManager.PrimaryNetworks[nad.Namespace] = nadNetwork } } } - userDefinedPrimaryNetwork := NewPrimaryNetwork(nadController) + userDefinedPrimaryNetwork := NewPrimaryNetwork(fakeNetworkManager) obtainedAnnotation, obtainedIsReady := userDefinedPrimaryNetwork.WaitForPrimaryAnnotationFn(tt.namespace, waitCond)(tt.annotations, tt.nadName) obtainedFound := userDefinedPrimaryNetwork.Found() obtainedNetworkName := userDefinedPrimaryNetwork.NetworkName() diff --git a/go-controller/pkg/config/config.go b/go-controller/pkg/config/config.go index dab8145de3..3a4f521826 100644 --- a/go-controller/pkg/config/config.go +++ b/go-controller/pkg/config/config.go @@ -232,9 +232,11 @@ type DefaultConfig struct { // EncapType value defines the encapsulation protocol to use to transmit packets between // hypervisors. By default the value is 'geneve' EncapType string `gcfg:"encap-type"` - // The IP address of the encapsulation endpoint. If not specified, the IP address the - // NodeName resolves to will be used + // Configured IP address of the encapsulation endpoint. EncapIP string `gcfg:"encap-ip"` + // Effective encap IP. It may be different from EncapIP if EncapIP meant to be + // the node's primary IP which can be updated when node's primary IP changes. + EffectiveEncapIP string // The UDP Port of the encapsulation endpoint. If not specified, the IP default port // of 6081 will be used EncapPort uint `gcfg:"encap-port"` diff --git a/go-controller/pkg/network-controller-manager/network_controller_manager.go b/go-controller/pkg/controllermanager/controller_manager.go similarity index 85% rename from go-controller/pkg/network-controller-manager/network_controller_manager.go rename to go-controller/pkg/controllermanager/controller_manager.go index fc041db752..d3fd104d75 100644 --- a/go-controller/pkg/network-controller-manager/network_controller_manager.go +++ b/go-controller/pkg/controllermanager/controller_manager.go @@ -1,4 +1,4 @@ -package networkControllerManager +package controllermanager import ( "context" @@ -17,7 +17,7 @@ import ( libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/observability" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" @@ -31,8 +31,8 @@ import ( "k8s.io/klog/v2" ) -// NetworkControllerManager structure is the object manages all controllers for all networks -type NetworkControllerManager struct { +// ControllerManager structure is the object manages all controllers +type ControllerManager struct { client clientset.Interface kube *kube.KubeOVN watchFactory *factory.WatchFactory @@ -53,15 +53,16 @@ type NetworkControllerManager struct { stopChan chan struct{} wg *sync.WaitGroup portCache *ovn.PortCache - defaultNetworkController nad.BaseNetworkController + defaultNetworkController networkmanager.BaseNetworkController + + // networkManager creates and deletes network controllers + networkManager networkmanager.Controller - // net-attach-def controller handle net-attach-def and create/delete network controllers - nadController *nad.NetAttachDefinitionController // eIPController programs OVN to support EgressIP eIPController *ovn.EgressIPController } -func (cm *NetworkControllerManager) NewNetworkController(nInfo util.NetInfo) (nad.NetworkController, error) { +func (cm *ControllerManager) NewNetworkController(nInfo util.NetInfo) (networkmanager.NetworkController, error) { cnci, err := cm.newCommonNetworkControllerInfo() if err != nil { return nil, fmt.Errorf("failed to create network controller info %w", err) @@ -69,17 +70,17 @@ func (cm *NetworkControllerManager) NewNetworkController(nInfo util.NetInfo) (na topoType := nInfo.TopologyType() switch topoType { case ovntypes.Layer3Topology: - return ovn.NewSecondaryLayer3NetworkController(cnci, nInfo, cm.nadController, cm.eIPController, cm.portCache) + return ovn.NewSecondaryLayer3NetworkController(cnci, nInfo, cm.networkManager.Interface(), cm.eIPController, cm.portCache) case ovntypes.Layer2Topology: - return ovn.NewSecondaryLayer2NetworkController(cnci, nInfo, cm.nadController) + return ovn.NewSecondaryLayer2NetworkController(cnci, nInfo, cm.networkManager.Interface(), cm.eIPController, cm.portCache) case ovntypes.LocalnetTopology: - return ovn.NewSecondaryLocalnetNetworkController(cnci, nInfo, cm.nadController), nil + return ovn.NewSecondaryLocalnetNetworkController(cnci, nInfo, cm.networkManager.Interface()), nil } return nil, fmt.Errorf("topology type %s not supported", topoType) } // newDummyNetworkController creates a dummy network controller used to clean up specific network -func (cm *NetworkControllerManager) newDummyNetworkController(topoType, netName string) (nad.NetworkController, error) { +func (cm *ControllerManager) newDummyNetworkController(topoType, netName string) (networkmanager.NetworkController, error) { cnci, err := cm.newCommonNetworkControllerInfo() if err != nil { return nil, fmt.Errorf("failed to create network controller info %w", err) @@ -87,11 +88,11 @@ func (cm *NetworkControllerManager) newDummyNetworkController(topoType, netName netInfo, _ := util.NewNetInfo(&ovncnitypes.NetConf{NetConf: types.NetConf{Name: netName}, Topology: topoType}) switch topoType { case ovntypes.Layer3Topology: - return ovn.NewSecondaryLayer3NetworkController(cnci, netInfo, cm.nadController, cm.eIPController, cm.portCache) + return ovn.NewSecondaryLayer3NetworkController(cnci, netInfo, cm.networkManager.Interface(), cm.eIPController, cm.portCache) case ovntypes.Layer2Topology: - return ovn.NewSecondaryLayer2NetworkController(cnci, netInfo, cm.nadController) + return ovn.NewSecondaryLayer2NetworkController(cnci, netInfo, cm.networkManager.Interface(), cm.eIPController, cm.portCache) case ovntypes.LocalnetTopology: - return ovn.NewSecondaryLocalnetNetworkController(cnci, netInfo, cm.nadController), nil + return ovn.NewSecondaryLocalnetNetworkController(cnci, netInfo, cm.networkManager.Interface()), nil } return nil, fmt.Errorf("topology type %s not supported", topoType) } @@ -125,7 +126,11 @@ func findAllSecondaryNetworkLogicalEntities(nbClient libovsdbclient.Client) ([]* return nodeSwitches, clusterRouters, nil } -func (cm *NetworkControllerManager) CleanupDeletedNetworks(validNetworks ...util.BasicNetInfo) error { +func (cm *ControllerManager) GetDefaultNetworkController() networkmanager.ReconcilableNetworkController { + return cm.defaultNetworkController +} + +func (cm *ControllerManager) CleanupStaleNetworks(validNetworks ...util.NetInfo) error { existingNetworksMap := map[string]string{} for _, network := range validNetworks { existingNetworksMap[network.GetNetworkName()] = network.TopologyType() @@ -137,7 +142,7 @@ func (cm *NetworkControllerManager) CleanupDeletedNetworks(validNetworks ...util return err } - staleNetworkControllers := map[string]nad.NetworkController{} + staleNetworkControllers := map[string]networkmanager.NetworkController{} for _, ls := range switches { netName := ls.ExternalIDs[ovntypes.NetworkExternalID] // TopologyExternalID always co-exists with NetworkExternalID @@ -179,13 +184,14 @@ func (cm *NetworkControllerManager) CleanupDeletedNetworks(validNetworks ...util return nil } -// NewNetworkControllerManager creates a new ovnkube controller manager to manage all the controller for all networks -func NewNetworkControllerManager(ovnClient *util.OVNClientset, wf *factory.WatchFactory, +// NewControllerManager creates a new ovnkube controller manager to manage all the controller for all networks +func NewControllerManager(ovnClient *util.OVNClientset, wf *factory.WatchFactory, libovsdbOvnNBClient libovsdbclient.Client, libovsdbOvnSBClient libovsdbclient.Client, - recorder record.EventRecorder, wg *sync.WaitGroup) (*NetworkControllerManager, error) { + recorder record.EventRecorder, wg *sync.WaitGroup) (*ControllerManager, error) { podRecorder := metrics.NewPodRecorder() + stopCh := make(chan struct{}) - cm := &NetworkControllerManager{ + cm := &ControllerManager{ client: ovnClient.KubeClient, kube: &kube.KubeOVN{ Kube: kube.Kube{KClient: ovnClient.KubeClient}, @@ -210,16 +216,18 @@ func NewNetworkControllerManager(ovnClient *util.OVNClientset, wf *factory.Watch } var err error + cm.networkManager = networkmanager.Default() if config.OVNKubernetesFeature.EnableMultiNetwork { - cm.nadController, err = nad.NewNetAttachDefinitionController("network-controller-manager", cm, wf, nil) + cm.networkManager, err = networkmanager.NewForZone(config.Default.Zone, cm, wf) if err != nil { return nil, err } } + return cm, nil } -func (cm *NetworkControllerManager) configureSCTPSupport() error { +func (cm *ControllerManager) configureSCTPSupport() error { hasSCTPSupport, err := util.DetectSCTPSupport() if err != nil { return err @@ -234,7 +242,7 @@ func (cm *NetworkControllerManager) configureSCTPSupport() error { return nil } -func (cm *NetworkControllerManager) configureSvcTemplateSupport() { +func (cm *ControllerManager) configureSvcTemplateSupport() { if !config.OVNKubernetesFeature.EnableServiceTemplateSupport { cm.svcTemplateSupport = false } else if _, _, err := util.RunOVNNbctl("--columns=_uuid", "list", "Chassis_Template_Var"); err != nil { @@ -246,14 +254,14 @@ func (cm *NetworkControllerManager) configureSvcTemplateSupport() { } } -func (cm *NetworkControllerManager) configureMetrics(stopChan <-chan struct{}) { +func (cm *ControllerManager) configureMetrics(stopChan <-chan struct{}) { metrics.RegisterOVNKubeControllerPerformance(cm.nbClient) metrics.RegisterOVNKubeControllerFunctional(stopChan) metrics.RunTimestamp(stopChan, cm.sbClient, cm.nbClient) metrics.MonitorIPSec(cm.nbClient) } -func (cm *NetworkControllerManager) createACLLoggingMeter() error { +func (cm *ControllerManager) createACLLoggingMeter() error { band := &nbdb.MeterBand{ Action: ovntypes.MeterAction, Rate: config.Logging.ACLLoggingRateLimit, @@ -284,19 +292,18 @@ func (cm *NetworkControllerManager) createACLLoggingMeter() error { } // newCommonNetworkControllerInfo creates and returns the common networkController info -func (cm *NetworkControllerManager) newCommonNetworkControllerInfo() (*ovn.CommonNetworkControllerInfo, error) { +func (cm *ControllerManager) newCommonNetworkControllerInfo() (*ovn.CommonNetworkControllerInfo, error) { return ovn.NewCommonNetworkControllerInfo(cm.client, cm.kube, cm.watchFactory, cm.recorder, cm.nbClient, cm.sbClient, cm.podRecorder, cm.SCTPSupport, cm.multicastSupport, cm.svcTemplateSupport) } // initDefaultNetworkController creates the controller for default network -func (cm *NetworkControllerManager) initDefaultNetworkController(nadController *nad.NetAttachDefinitionController, - observManager *observability.Manager) error { +func (cm *ControllerManager) initDefaultNetworkController(observManager *observability.Manager) error { cnci, err := cm.newCommonNetworkControllerInfo() if err != nil { return fmt.Errorf("failed to create common network controller info: %w", err) } - defaultController, err := ovn.NewDefaultNetworkController(cnci, nadController, observManager, cm.portCache, cm.eIPController) + defaultController, err := ovn.NewDefaultNetworkController(cnci, observManager, cm.networkManager.Interface(), cm.eIPController, cm.portCache) if err != nil { return err } @@ -308,7 +315,7 @@ func (cm *NetworkControllerManager) initDefaultNetworkController(nadController * } // Start the ovnkube controller -func (cm *NetworkControllerManager) Start(ctx context.Context) error { +func (cm *ControllerManager) Start(ctx context.Context) error { klog.Info("Starting the ovnkube controller") // Make sure that the ovnkube-controller zone matches with the Northbound db zone. @@ -393,7 +400,7 @@ func (cm *NetworkControllerManager) Start(ctx context.Context) error { cm.podRecorder.Run(cm.sbClient, cm.stopChan) if config.OVNKubernetesFeature.EnableEgressIP { - cm.eIPController = ovn.NewEIPController(cm.nbClient, cm.kube, cm.watchFactory, cm.recorder, cm.portCache, cm.nadController, + cm.eIPController = ovn.NewEIPController(cm.nbClient, cm.kube, cm.watchFactory, cm.recorder, cm.portCache, cm.networkManager.Interface(), addressset.NewOvnAddressSetFactory(cm.nbClient, config.IPv4Mode, config.IPv6Mode), config.IPv4Mode, config.IPv6Mode, zone, ovn.DefaultNetworkControllerName) // FIXME(martinkennelly): remove when EIP controller is fully extracted from from DNC and started here. Ensure SyncLocalNodeZonesCache is re-enabled in EIP controller. if err = cm.eIPController.SyncLocalNodeZonesCache(); err != nil { @@ -401,13 +408,6 @@ func (cm *NetworkControllerManager) Start(ctx context.Context) error { } } - // nadController is nil if multi-network is disabled - if cm.nadController != nil { - if err = cm.nadController.Start(); err != nil { - return fmt.Errorf("failed to start NAD Controller :%v", err) - } - } - var observabilityManager *observability.Manager if config.OVNKubernetesFeature.EnableObservability { observabilityManager = observability.NewManager(cm.nbClient) @@ -431,10 +431,17 @@ func (cm *NetworkControllerManager) Start(ctx context.Context) error { }() } - err = cm.initDefaultNetworkController(cm.nadController, observabilityManager) + err = cm.initDefaultNetworkController(observabilityManager) if err != nil { return fmt.Errorf("failed to init default network controller: %v", err) } + + if cm.networkManager != nil { + if err = cm.networkManager.Start(); err != nil { + return fmt.Errorf("failed to start NAD Controller :%v", err) + } + } + err = cm.defaultNetworkController.Start(ctx) if err != nil { return fmt.Errorf("failed to start default network controller: %v", err) @@ -444,7 +451,7 @@ func (cm *NetworkControllerManager) Start(ctx context.Context) error { } // Stop gracefully stops all managed controllers -func (cm *NetworkControllerManager) Stop() { +func (cm *ControllerManager) Stop() { // stop metric recorders close(cm.stopChan) @@ -454,7 +461,7 @@ func (cm *NetworkControllerManager) Stop() { } // stop the NAD controller - if cm.nadController != nil { - cm.nadController.Stop() + if cm.networkManager != nil { + cm.networkManager.Stop() } } diff --git a/go-controller/pkg/network-controller-manager/network_controller_manager_suite_test.go b/go-controller/pkg/controllermanager/controller_manager_suite_test.go similarity index 85% rename from go-controller/pkg/network-controller-manager/network_controller_manager_suite_test.go rename to go-controller/pkg/controllermanager/controller_manager_suite_test.go index 44ae4d83a6..37ad90ab6c 100644 --- a/go-controller/pkg/network-controller-manager/network_controller_manager_suite_test.go +++ b/go-controller/pkg/controllermanager/controller_manager_suite_test.go @@ -1,4 +1,4 @@ -package networkControllerManager +package controllermanager import ( "testing" diff --git a/go-controller/pkg/network-controller-manager/node_network_controller_manager.go b/go-controller/pkg/controllermanager/node_controller_manager.go similarity index 83% rename from go-controller/pkg/network-controller-manager/node_network_controller_manager.go rename to go-controller/pkg/controllermanager/node_controller_manager.go index 835bf0f382..ef89c90874 100644 --- a/go-controller/pkg/network-controller-manager/node_network_controller_manager.go +++ b/go-controller/pkg/controllermanager/node_controller_manager.go @@ -1,4 +1,4 @@ -package networkControllerManager +package controllermanager import ( "context" @@ -11,7 +11,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iprulemanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager" @@ -26,8 +26,8 @@ import ( kexec "k8s.io/utils/exec" ) -// nodeNetworkControllerManager structure is the object manages all controllers for all networks for ovnkube-node -type nodeNetworkControllerManager struct { +// NodeControllerManager structure is the object manages all controllers for all networks for ovnkube-node +type NodeControllerManager struct { name string ovnNodeClient *util.OVNNodeClientset Kube kube.Interface @@ -38,9 +38,8 @@ type nodeNetworkControllerManager struct { defaultNodeNetworkController *node.DefaultNodeNetworkController - // net-attach-def controller handle net-attach-def and create/delete secondary controllers - // nil in dpu-host mode - nadController *nad.NetAttachDefinitionController + // networkManager creates and deletes secondary network controllers + networkManager networkmanager.Controller // vrf manager that creates and manages vrfs for all UDNs vrfManager *vrfmanager.Controller // route manager that creates and manages routes @@ -50,7 +49,7 @@ type nodeNetworkControllerManager struct { } // NewNetworkController create secondary node network controllers for the given NetInfo -func (ncm *nodeNetworkControllerManager) NewNetworkController(nInfo util.NetInfo) (nad.NetworkController, error) { +func (ncm *NodeControllerManager) NewNetworkController(nInfo util.NetInfo) (networkmanager.NetworkController, error) { topoType := nInfo.TopologyType() switch topoType { case ovntypes.Layer3Topology, ovntypes.Layer2Topology, ovntypes.LocalnetTopology: @@ -60,8 +59,12 @@ func (ncm *nodeNetworkControllerManager) NewNetworkController(nInfo util.NetInfo return nil, fmt.Errorf("topology type %s not supported", topoType) } -// CleanupDeletedNetworks cleans up all stale entities giving list of all existing secondary network controllers -func (ncm *nodeNetworkControllerManager) CleanupDeletedNetworks(validNetworks ...util.BasicNetInfo) error { +func (ncm *NodeControllerManager) GetDefaultNetworkController() networkmanager.ReconcilableNetworkController { + return ncm.defaultNodeNetworkController +} + +// CleanupStaleNetworks cleans up all stale entities giving list of all existing secondary network controllers +func (ncm *NodeControllerManager) CleanupStaleNetworks(validNetworks ...util.NetInfo) error { if !util.IsNetworkSegmentationSupportEnabled() { return nil } @@ -80,7 +83,7 @@ func (ncm *nodeNetworkControllerManager) CleanupDeletedNetworks(validNetworks .. return ncm.vrfManager.Repair(validVRFDevices) } -func (ncm *nodeNetworkControllerManager) getNetworkID(network util.BasicNetInfo) (int, error) { +func (ncm *NodeControllerManager) getNetworkID(network util.NetInfo) (int, error) { nodes, err := ncm.watchFactory.GetNodes() if err != nil { return util.InvalidID, err @@ -93,22 +96,24 @@ func (ncm *nodeNetworkControllerManager) getNetworkID(network util.BasicNetInfo) } // newCommonNetworkControllerInfo creates and returns the base node network controller info -func (ncm *nodeNetworkControllerManager) newCommonNetworkControllerInfo() *node.CommonNodeNetworkControllerInfo { +func (ncm *NodeControllerManager) newCommonNetworkControllerInfo() *node.CommonNodeNetworkControllerInfo { return node.NewCommonNodeNetworkControllerInfo(ncm.ovnNodeClient.KubeClient, ncm.ovnNodeClient.AdminPolicyRouteClient, ncm.watchFactory, ncm.recorder, ncm.name, ncm.routeManager) } -// NAD controller should be started on the node side under the following conditions: +// isNetworkManagerRequiredForNode checks if network manager should be started +// on the node side, which requires any of the following conditions: // (1) dpu mode is enabled when secondary networks feature is enabled // (2) primary user defined networks is enabled (all modes) -func isNodeNADControllerRequired() bool { +func isNetworkManagerRequiredForNode() bool { return (config.OVNKubernetesFeature.EnableMultiNetwork && config.OvnKubeNode.Mode == ovntypes.NodeModeDPU) || - util.IsNetworkSegmentationSupportEnabled() + util.IsNetworkSegmentationSupportEnabled() || + util.IsRouteAdvertisementsEnabled() } -// NewNodeNetworkControllerManager creates a new OVN controller manager to manage all the controller for all networks -func NewNodeNetworkControllerManager(ovnClient *util.OVNClientset, wf factory.NodeWatchFactory, name string, - wg *sync.WaitGroup, eventRecorder record.EventRecorder, routeManager *routemanager.Controller) (*nodeNetworkControllerManager, error) { - ncm := &nodeNetworkControllerManager{ +// NewNodeControllerManager creates a new OVN controller manager to manage all the controller for all networks +func NewNodeControllerManager(ovnClient *util.OVNClientset, wf factory.NodeWatchFactory, name string, + wg *sync.WaitGroup, eventRecorder record.EventRecorder, routeManager *routemanager.Controller) (*NodeControllerManager, error) { + ncm := &NodeControllerManager{ name: name, ovnNodeClient: &util.OVNNodeClientset{KubeClient: ovnClient.KubeClient, AdminPolicyRouteClient: ovnClient.AdminPolicyRouteClient}, Kube: &kube.Kube{KClient: ovnClient.KubeClient}, @@ -121,9 +126,11 @@ func NewNodeNetworkControllerManager(ovnClient *util.OVNClientset, wf factory.No // need to configure OVS interfaces for Pods on secondary networks in the DPU mode // need to start NAD controller on node side for programming gateway pieces for UDNs + // need to start NAD controller on node side for VRF awareness with BGP var err error - if isNodeNADControllerRequired() { - ncm.nadController, err = nad.NewNetAttachDefinitionController("node-network-controller-manager", ncm, wf, nil) + ncm.networkManager = networkmanager.Default() + if isNetworkManagerRequiredForNode() { + ncm.networkManager, err = networkmanager.NewForNode(name, ncm, wf) if err != nil { return nil, err } @@ -136,9 +143,8 @@ func NewNodeNetworkControllerManager(ovnClient *util.OVNClientset, wf factory.No } // initDefaultNodeNetworkController creates the controller for default network -func (ncm *nodeNetworkControllerManager) initDefaultNodeNetworkController() error { - defaultNodeNetworkController, err := node.NewDefaultNodeNetworkController(ncm.newCommonNetworkControllerInfo(), - ncm.nadController) +func (ncm *NodeControllerManager) initDefaultNodeNetworkController() error { + defaultNodeNetworkController, err := node.NewDefaultNodeNetworkController(ncm.newCommonNetworkControllerInfo(), ncm.networkManager.Interface()) if err != nil { return err } @@ -150,7 +156,7 @@ func (ncm *nodeNetworkControllerManager) initDefaultNodeNetworkController() erro } // Start the node network controller manager -func (ncm *nodeNetworkControllerManager) Start(ctx context.Context) (err error) { +func (ncm *NodeControllerManager) Start(ctx context.Context) (err error) { klog.Infof("Starting the node network controller manager, Mode: %s", config.OvnKubeNode.Mode) // Initialize OVS exec runner; find OVS binaries that the CNI code uses. @@ -197,8 +203,8 @@ func (ncm *nodeNetworkControllerManager) Start(ctx context.Context) (err error) return fmt.Errorf("failed to start default node network controller: %v", err) } - if ncm.nadController != nil { - err = ncm.nadController.Start() + if ncm.networkManager != nil { + err = ncm.networkManager.Start() if err != nil { return fmt.Errorf("failed to start NAD controller: %w", err) } @@ -235,7 +241,7 @@ func (ncm *nodeNetworkControllerManager) Start(ctx context.Context) (err error) } // Stop gracefully stops all managed controllers -func (ncm *nodeNetworkControllerManager) Stop() { +func (ncm *NodeControllerManager) Stop() { // stop stale ovs ports cleanup close(ncm.stopChan) @@ -244,15 +250,15 @@ func (ncm *nodeNetworkControllerManager) Stop() { } // stop the NAD controller - if ncm.nadController != nil { - ncm.nadController.Stop() + if ncm.networkManager != nil { + ncm.networkManager.Stop() } } // checkForStaleOVSRepresentorInterfaces checks for stale OVS ports backed by Repreresentor interfaces, // derive iface-id from pod name and namespace then remove any interfaces assoicated with a sandbox that are // not scheduled to the node. -func (ncm *nodeNetworkControllerManager) checkForStaleOVSRepresentorInterfaces() { +func (ncm *NodeControllerManager) checkForStaleOVSRepresentorInterfaces() { // Get all representor interfaces. these are OVS interfaces that have their external_ids:sandbox and vf-netdev-name set. out, stderr, err := util.RunOVSVsctl("--columns=name,external_ids", "--data=bare", "--no-headings", "--format=csv", "find", "Interface", "external_ids:sandbox!=\"\"", "external_ids:vf-netdev-name!=\"\"") diff --git a/go-controller/pkg/network-controller-manager/node_network_controller_manager_test.go b/go-controller/pkg/controllermanager/node_controller_manager_test.go similarity index 88% rename from go-controller/pkg/network-controller-manager/node_network_controller_manager_test.go rename to go-controller/pkg/controllermanager/node_controller_manager_test.go index e63e698bc1..d414833300 100644 --- a/go-controller/pkg/network-controller-manager/node_network_controller_manager_test.go +++ b/go-controller/pkg/controllermanager/node_controller_manager_test.go @@ -1,4 +1,4 @@ -package networkControllerManager +package controllermanager import ( "fmt" @@ -12,6 +12,8 @@ import ( factoryMocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory/mocks" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + nadinformermocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1" + nadlistermocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -22,7 +24,7 @@ import ( ) func genListStalePortsCmd() string { - return fmt.Sprintf("ovs-vsctl --timeout=15 --data=bare --no-headings --columns=name find interface ofport=-1") + return "ovs-vsctl --timeout=15 --data=bare --no-headings --columns=name find interface ofport=-1" } func genDeleteStalePortCmd(ifaces ...string) string { @@ -99,7 +101,7 @@ var _ = Describe("Healthcheck tests", func() { }) Describe("checkForStaleOVSRepresentorInterfaces", func() { - var ncm *nodeNetworkControllerManager + var ncm *NodeControllerManager nodeName := "localNode" routeManager := routemanager.NewController() podList := []*corev1.Pod{ @@ -130,7 +132,7 @@ var _ = Describe("Healthcheck tests", func() { BeforeEach(func() { // setup kube output factoryMock.On("NADInformer").Return(nil) - ncm, err = NewNodeNetworkControllerManager(fakeClient, &factoryMock, nodeName, &sync.WaitGroup{}, nil, routeManager) + ncm, err = NewNodeControllerManager(fakeClient, &factoryMock, nodeName, &sync.WaitGroup{}, nil, routeManager) Expect(err).NotTo(HaveOccurred()) factoryMock.On("GetPods", "").Return(podList, nil) }) @@ -222,12 +224,16 @@ var _ = Describe("Healthcheck tests", func() { nodeList := []*corev1.Node{node} factoryMock.On("GetNode", nodeName).Return(nodeList[0], nil) factoryMock.On("GetNodes").Return(nodeList, nil) - factoryMock.On("NADInformer").Return(nil) factoryMock.On("UserDefinedNetworkInformer").Return(nil) factoryMock.On("ClusterUserDefinedNetworkInformer").Return(nil) factoryMock.On("NamespaceInformer").Return(nil) + nadListerMock := &nadlistermocks.NetworkAttachmentDefinitionLister{} + nadInformerMock := &nadinformermocks.NetworkAttachmentDefinitionInformer{} + nadInformerMock.On("Lister").Return(nadListerMock) + nadInformerMock.On("Informer").Return(nil) + factoryMock.On("NADInformer").Return(nadInformerMock) - ncm, err := NewNodeNetworkControllerManager(fakeClient, &factoryMock, nodeName, &sync.WaitGroup{}, nil, routeManager) + ncm, err := NewNodeControllerManager(fakeClient, &factoryMock, nodeName, &sync.WaitGroup{}, nil, routeManager) Expect(err).NotTo(HaveOccurred()) err = testNS.Do(func(ns.NetNS) error { @@ -243,7 +249,7 @@ var _ = Describe("Healthcheck tests", func() { _, err = util.GetNetLinkOps().LinkByName(validVrfDevice) Expect(err).NotTo(HaveOccurred()) - err = ncm.CleanupDeletedNetworks(NetInfo) + err = ncm.CleanupStaleNetworks(NetInfo) Expect(err).NotTo(HaveOccurred()) // Verify CleanupDeletedNetworks cleans up VRF configuration for diff --git a/go-controller/pkg/libovsdb/ops/db_object_ids.go b/go-controller/pkg/libovsdb/ops/db_object_ids.go index b242d7d3ea..44e68ad9b3 100644 --- a/go-controller/pkg/libovsdb/ops/db_object_ids.go +++ b/go-controller/pkg/libovsdb/ops/db_object_ids.go @@ -147,6 +147,15 @@ func NewDbObjectIDs(idsType *ObjectIDsType, controller string, objectIds map[Ext if controller == "" { panic("NewDbObjectIDs failed: controller should not be empty") } + objectIDs := newDbObjectIDs(idsType, objectIds) + objectIDs.ownerControllerName = controller + return objectIDs +} + +// newDbObjectIDs is used to construct DbObjectIDs, idsType is always required, +// objectIds may be empty, or half-filled for predicate search. +// objectIds keys that are not used by given idsType will cause panic. +func newDbObjectIDs(idsType *ObjectIDsType, objectIds map[ExternalIDKey]string) *DbObjectIDs { externalIDKeys := idsType.GetExternalIDKeys() if externalIDKeys == nil { // can only happen if ObjectIDsType{} is passed @@ -162,13 +171,20 @@ func NewDbObjectIDs(idsType *ObjectIDsType, controller string, objectIds map[Ext objectIds = map[ExternalIDKey]string{} } objectIDs := &DbObjectIDs{ - idsType: idsType, - ownerControllerName: controller, - objectIDs: objectIds, + idsType: idsType, + objectIDs: objectIds, } return objectIDs } +// NewDbObjectIDsAcrossAllContollers is used to construct DbObjectIDs, without filtering out on specific controller, +// idsType is always required, +// objectIds may be empty, or half-filled for predicate search. +// objectIds keys that are not used by given idsType will cause panic. +func NewDbObjectIDsAcrossAllContollers(idsType *ObjectIDsType, objectIds map[ExternalIDKey]string) *DbObjectIDs { + return newDbObjectIDs(idsType, objectIds) +} + // AddIDs creates new DbObjectIDs with the additional extraObjectIds. // If at least one of extraObjectIds keys is not used by the objectIDs.idsType it will cause panic. func (objectIDs *DbObjectIDs) AddIDs(extraObjectIds map[ExternalIDKey]string) *DbObjectIDs { @@ -219,13 +235,14 @@ func (objectIDs *DbObjectIDs) GetObjectID(key ExternalIDKey) string { // - objectIDs.idsType.ownerObjectType // - values from DbObjectIDs.objectIDs are added in order set in ObjectIDsType.externalIDKeys func (objectIDs *DbObjectIDs) GetExternalIDs() map[string]string { - return objectIDs.getExternalIDs(false) + externalIDs := objectIDs.getExternalIDs(false) + externalIDs[OwnerControllerKey.String()] = objectIDs.ownerControllerName + return externalIDs } func (objectIDs *DbObjectIDs) getExternalIDs(allowEmptyKeys bool) map[string]string { externalIDs := map[string]string{ - OwnerControllerKey.String(): objectIDs.ownerControllerName, - OwnerTypeKey.String(): objectIDs.idsType.ownerObjectType, + OwnerTypeKey.String(): objectIDs.idsType.ownerObjectType, } for key, value := range objectIDs.objectIDs { externalIDs[key.String()] = value @@ -308,6 +325,24 @@ func GetNoOwnerPredicate[T hasExternalIDs]() func(item T) bool { // to nil. func GetPredicate[nbdbT hasExternalIDs](objectIDs *DbObjectIDs, f func(item nbdbT) bool) func(item nbdbT) bool { predicateIDs := objectIDs.getExternalIDs(true) + predicateIDs[OwnerControllerKey.String()] = objectIDs.ownerControllerName + return getPredicate(predicateIDs, f) +} + +// GetPredicateAcrossAllControllers returns a predicate to search for db obj of type nbdbT across all controllers. +// Only non-empty ids will be matched (that always includes DbObjectIDs.OwnerTypeKey and DbObjectIDs.ownerControllerName), +// but the other IDs may be empty and will be ignored in the filtering, additional filter function f may be passed, or set +// to nil. +func GetPredicateAcrossAllControllers[nbdbT hasExternalIDs](objectIDs *DbObjectIDs, f func(item nbdbT) bool) func(item nbdbT) bool { + predicateIDs := objectIDs.getExternalIDs(true) + return getPredicate(predicateIDs, f) +} + +// GetPredicate returns a predicate to search for db obj of type nbdbT. +// Only non-empty ids will be matched (that always includes DbObjectIDs.OwnerTypeKey and DbObjectIDs.ownerControllerName), +// but the other IDs may be empty and will be ignored in the filtering, additional filter function f may be passed, or set +// to nil. +func getPredicate[nbdbT hasExternalIDs](predicateIDs map[string]string, f func(item nbdbT) bool) func(item nbdbT) bool { if primaryID, ok := predicateIDs[PrimaryIDKey.String()]; ok { // when primary id is set, other ids are not required predicateIDs = map[string]string{PrimaryIDKey.String(): primaryID} diff --git a/go-controller/pkg/network-attach-def-controller/network_manager.go b/go-controller/pkg/network-attach-def-controller/network_manager.go deleted file mode 100644 index 63763afd50..0000000000 --- a/go-controller/pkg/network-attach-def-controller/network_manager.go +++ /dev/null @@ -1,230 +0,0 @@ -package networkAttachDefController - -import ( - "context" - "errors" - "fmt" - "sync" - "time" - - "k8s.io/client-go/util/workqueue" - "k8s.io/klog/v2" - - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" -) - -// networkManager is a level driven controller that handles creating, starting, -// stopping and cleaning up network controllers -type networkManager interface { - // EnsureNetwork will add the network controller for the provided network - // configuration. If a controller already exists for the same network with a - // different configuration, the existing controller is stopped and cleaned - // up before creating the new one. If a controller already exists for the - // same network configuration, it synchronizes the list of NADs to the - // controller. - EnsureNetwork(util.NetInfo) - - // DeleteNetwork removes the network controller for the provided network - DeleteNetwork(string) - - // Start the controller - Start() error - - // Stop the controller - Stop() - - getNetwork(string) util.NetInfo -} - -func newNetworkManager(name string, ncm NetworkControllerManager) networkManager { - nc := &networkManagerImpl{ - name: fmt.Sprintf("[%s network manager]", name), - ncm: ncm, - networks: map[string]util.NetInfo{}, - networkControllers: map[string]*networkControllerState{}, - } - // this controller does not feed from an informer, networks are manually - // added to the queue for processing - config := &controller.ReconcilerConfig{ - RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), - Reconcile: nc.sync, - Threadiness: 1, - } - nc.controller = controller.NewReconciler( - nc.name, - config, - ) - return nc -} - -type networkControllerState struct { - controller NetworkController - stoppedAndDeleting bool -} - -type networkManagerImpl struct { - sync.RWMutex - name string - controller controller.Reconciler - ncm NetworkControllerManager - networks map[string]util.NetInfo - networkControllers map[string]*networkControllerState -} - -// Start will cleanup stale networks that have not been ensured via -// EnsuredNetwork before this call -func (nm *networkManagerImpl) Start() error { - return controller.StartWithInitialSync(nm.syncAll, nm.controller) -} - -func (nm *networkManagerImpl) Stop() { - controller.Stop(nm.controller) - for _, networkControllerState := range nm.getAllNetworkStates() { - networkControllerState.controller.Stop() - } -} - -func (nm *networkManagerImpl) EnsureNetwork(network util.NetInfo) { - nm.setNetwork(network.GetNetworkName(), network) - nm.controller.Reconcile(network.GetNetworkName()) -} - -func (nm *networkManagerImpl) DeleteNetwork(network string) { - nm.setNetwork(network, nil) - nm.controller.Reconcile(network) -} - -func (nm *networkManagerImpl) setNetwork(network string, netInfo util.NetInfo) { - nm.Lock() - defer nm.Unlock() - if netInfo == nil { - delete(nm.networks, network) - return - } - nm.networks[network] = netInfo -} - -func (nm *networkManagerImpl) getNetwork(network string) util.NetInfo { - nm.RLock() - defer nm.RUnlock() - return nm.networks[network] -} - -func (nm *networkManagerImpl) getAllNetworks() []util.BasicNetInfo { - nm.RLock() - defer nm.RUnlock() - networks := make([]util.BasicNetInfo, 0, len(nm.networks)) - for _, network := range nm.networks { - networks = append(networks, network) - } - return networks -} - -func (nm *networkManagerImpl) setNetworkState(network string, state *networkControllerState) { - nm.Lock() - defer nm.Unlock() - if state == nil { - delete(nm.networkControllers, network) - return - } - nm.networkControllers[network] = state -} - -func (nm *networkManagerImpl) getNetworkState(network string) *networkControllerState { - nm.RLock() - defer nm.RUnlock() - return nm.networkControllers[network] -} - -func (nm *networkManagerImpl) getAllNetworkStates() []*networkControllerState { - nm.RLock() - defer nm.RUnlock() - networkStates := make([]*networkControllerState, 0, len(nm.networks)) - for _, state := range nm.networkControllers { - networkStates = append(networkStates, state) - } - return networkStates -} - -func (nm *networkManagerImpl) sync(network string) error { - startTime := time.Now() - klog.V(5).Infof("%s: sync network %s", nm.name, network) - defer func() { - klog.V(4).Infof("%s: finished syncing network %s, took %v", nm.name, network, time.Since(startTime)) - }() - - want := nm.getNetwork(network) - have := nm.getNetworkState(network) - - // we will dispose of the old network if deletion is in progress or if - // configuration changed - dispose := have != nil && (have.stoppedAndDeleting || !have.controller.Equals(want)) - - if dispose { - if !have.stoppedAndDeleting { - have.controller.Stop() - } - have.stoppedAndDeleting = true - err := have.controller.Cleanup() - if err != nil { - return fmt.Errorf("%s: failed to cleanup network %s: %w", nm.name, network, err) - } - nm.setNetworkState(network, nil) - } - - // no network needed so nothing to do - if want == nil { - return nil - } - - // we didn't dispose of current controller, so this might just be an update of the network NADs - if have != nil && !dispose { - have.controller.SetNADs(want.GetNADs()...) - return nil - } - - // setup & start the new network controller - nc, err := nm.ncm.NewNetworkController(util.CopyNetInfo(want)) - if err != nil { - return fmt.Errorf("%s: failed to create network %s: %w", nm.name, network, err) - } - - err = nc.Start(context.Background()) - if err != nil { - return fmt.Errorf("%s: failed to start network %s: %w", nm.name, network, err) - } - nm.setNetworkState(network, &networkControllerState{controller: nc}) - - return nil -} - -func (nm *networkManagerImpl) syncAll() error { - // as we sync upon start, consider networks that have not been ensured as - // stale and clean them up - validNetworks := nm.getAllNetworks() - if err := nm.ncm.CleanupDeletedNetworks(validNetworks...); err != nil { - return err - } - - // sync all known networks. There is no informer for networks. Keys are added by NAD controller. - // Certain downstream controllers that handle configuration for multiple networks depend on being - // aware of all the existing networks on initialization. To achieve that, we need to start existing - // networks synchronously. Otherwise, these controllers might incorrectly assess valid configuration - // as stale. - start := time.Now() - klog.Infof("%s: syncing all networks", nm.name) - for _, network := range validNetworks { - if err := nm.sync(network.GetNetworkName()); errors.Is(err, ErrNetworkControllerTopologyNotManaged) { - klog.V(5).Infof( - "ignoring network %q since %q does not manage it", - network.GetNetworkName(), - nm.name, - ) - } else if err != nil { - return fmt.Errorf("failed to sync network %s: %w", network.GetNetworkName(), err) - } - } - klog.Infof("%s: finished syncing all networks. Time taken: %s", nm.name, time.Since(start)) - return nil -} diff --git a/go-controller/pkg/networkmanager/api.go b/go-controller/pkg/networkmanager/api.go new file mode 100644 index 0000000000..58853d1ed3 --- /dev/null +++ b/go-controller/pkg/networkmanager/api.go @@ -0,0 +1,167 @@ +package networkmanager + +import ( + "context" + "errors" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "k8s.io/client-go/tools/record" +) + +var ErrNetworkControllerTopologyNotManaged = errors.New("no cluster network controller to manage topology") + +// Interface is the main package entrypoint and provides network related +// information to the rest of the project. +type Interface interface { + GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) + GetNetwork(networkName string) util.NetInfo + // DoWithLock takes care of locking and unlocking while iterating over all role primary user defined networks. + DoWithLock(f func(network util.NetInfo) error) error + GetActiveNetworkNamespaces(networkName string) ([]string, error) +} + +// Controller handles the runtime of the package +type Controller interface { + Interface() Interface + Start() error + Stop() +} + +// Default returns a default implementation that assumes the default network is +// the only ever existing network. Used when multi-network capabilities are not +// enabled or testing. +func Default() Controller { + return def +} + +// NewForCluster builds a controller for cluster manager +func NewForCluster( + cm ControllerManager, + wf watchFactory, + recorder record.EventRecorder, +) (Controller, error) { + return new( + "clustermanager-nad-controller", + "", + "", + cm, + wf, + recorder, + ) +} + +// NewForZone builds a controller for zone manager +func NewForZone( + zone string, + cm ControllerManager, + wf watchFactory, +) (Controller, error) { + return new( + "zone-nad-controller", + zone, + "", + cm, + wf, + nil, + ) +} + +// NewForNode builds a controller for node manager +func NewForNode( + node string, + cm ControllerManager, + wf watchFactory, +) (Controller, error) { + return new( + "node-nad-controller", + "", + node, + cm, + wf, + nil, + ) +} + +// New builds a new Controller. It's aware of networks configured in the system, +// gathers relevant information about them for the project and handles the +// lifecycle of their corresponding network controllers. +func new( + name string, + zone string, + node string, + cm ControllerManager, + wf watchFactory, + recorder record.EventRecorder, +) (Controller, error) { + return newController(name, zone, node, cm, wf, recorder) +} + +// ControllerManager manages controllers. Needs to be provided in order to build +// new network controllers and to to be informed of potential stale networks in +// case it has clean-up of it's own to do. +type ControllerManager interface { + NewNetworkController(netInfo util.NetInfo) (NetworkController, error) + GetDefaultNetworkController() ReconcilableNetworkController + CleanupStaleNetworks(validNetworks ...util.NetInfo) error +} + +// ReconcilableNetworkController is a network controller that can reconcile +// certain network configuration changes. +type ReconcilableNetworkController interface { + util.NetInfo + + // Reconcile informs the controller of network configuration changes. + // Implementations should not return any error at or after updating this + // network information on their as there is nothing network manager can do + // about it. In this case implementations should either carry their on + // retries or log the error and give up. + Reconcile(util.NetInfo) error +} + +// BaseNetworkController is a ReconcilableNetworkController that can be started and +// stopped. +type BaseNetworkController interface { + ReconcilableNetworkController + Start(ctx context.Context) error + Stop() +} + +// NetworkController is a BaseNetworkController that can also clean up after +// itself. +type NetworkController interface { + BaseNetworkController + Cleanup() error +} + +// defaultNetworkManager assumes the default network is +// the only ever existing network. Used when multi-network capabilities are not +// enabled or testing. +type defaultNetworkManager struct{} + +func (nm defaultNetworkManager) Interface() Interface { + return &nm +} + +func (nm defaultNetworkManager) Start() error { + return nil +} + +func (nm defaultNetworkManager) Stop() {} + +func (nm defaultNetworkManager) GetActiveNetworkForNamespace(string) (util.NetInfo, error) { + return &util.DefaultNetInfo{}, nil +} + +func (nm defaultNetworkManager) GetNetwork(networkName string) util.NetInfo { + return &util.DefaultNetInfo{} +} + +func (nm defaultNetworkManager) DoWithLock(f func(network util.NetInfo) error) error { + return f(&util.DefaultNetInfo{}) +} + +func (nm defaultNetworkManager) GetActiveNetworkNamespaces(networkName string) ([]string, error) { + return []string{"default"}, nil +} + +var def Controller = &defaultNetworkManager{} diff --git a/go-controller/pkg/network-attach-def-controller/network_attach_def_controller.go b/go-controller/pkg/networkmanager/nad_controller.go similarity index 57% rename from go-controller/pkg/network-attach-def-controller/network_attach_def_controller.go rename to go-controller/pkg/networkmanager/nad_controller.go index aabdbfe4bd..63d7ec6e1c 100644 --- a/go-controller/pkg/network-attach-def-controller/network_attach_def_controller.go +++ b/go-controller/pkg/networkmanager/nad_controller.go @@ -1,8 +1,6 @@ -package networkAttachDefController +package networkmanager import ( - "context" - "errors" "fmt" "reflect" "sync" @@ -24,67 +22,43 @@ import ( nadinformers "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1" nadlisters "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + rainformers "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/informers/externalversions/routeadvertisements/v1" userdefinednetworkinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/userdefinednetwork/v1" userdefinednetworklister "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/listers/userdefinednetwork/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" utiludn "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/udn" ) -var ErrNetworkControllerTopologyNotManaged = errors.New("no cluster network controller to manage topology") - -type BaseNetworkController interface { - Start(ctx context.Context) error - Stop() -} - -type NetworkController interface { - BaseNetworkController - util.NetInfo - // Cleanup cleans up the NetworkController-owned resources, it could be called to clean up network controllers that are deleted when - // ovn-k8s is down; so it's receiver could be a dummy network controller, it just needs to know its network name. - Cleanup() error -} - -// NetworkControllerManager manages all network controllers -type NetworkControllerManager interface { - NewNetworkController(netInfo util.NetInfo) (NetworkController, error) - CleanupDeletedNetworks(validNetworks ...util.BasicNetInfo) error -} - type watchFactory interface { NADInformer() nadinformers.NetworkAttachmentDefinitionInformer UserDefinedNetworkInformer() userdefinednetworkinformer.UserDefinedNetworkInformer ClusterUserDefinedNetworkInformer() userdefinednetworkinformer.ClusterUserDefinedNetworkInformer NamespaceInformer() coreinformers.NamespaceInformer + RouteAdvertisementsInformer() rainformers.RouteAdvertisementsInformer + NodeCoreInformer() coreinformers.NodeInformer } -type NADController interface { - Start() error - Stop() - GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) - GetNetwork(networkName string) (util.NetInfo, error) - // DoWithLock takes care of locking and unlocking while iterating over all role primary user defined networks. - DoWithLock(f func(network util.NetInfo) error) error - GetActiveNetworkNamespaces(networkName string) ([]string, error) -} - -// NetAttachDefinitionController handles namespaced scoped NAD events and +// nadController handles namespaced scoped NAD events and // manages cluster scoped networks defined in those NADs. NADs are mostly // referenced from pods to give them access to the network. Different NADs can // define the same network as long as those definitions are actually equal. // Unexpected situations are handled on best effort basis but improper NAD // administration can lead to undefined behavior if referenced by running pods. -type NetAttachDefinitionController struct { +type nadController struct { sync.RWMutex - name string - netAttachDefLister nadlisters.NetworkAttachmentDefinitionLister - udnLister userdefinednetworklister.UserDefinedNetworkLister - cudnLister userdefinednetworklister.ClusterUserDefinedNetworkLister - namespaceLister corev1listers.NamespaceLister - controller controller.Controller - recorder record.EventRecorder - // networkManager is used to manage the network controllers - networkManager networkManager + + name string + nadLister nadlisters.NetworkAttachmentDefinitionLister + udnLister userdefinednetworklister.UserDefinedNetworkLister + cudnLister userdefinednetworklister.ClusterUserDefinedNetworkLister + namespaceLister corev1listers.NamespaceLister + controller controller.Controller + recorder record.EventRecorder + + // networkController reconciles network specific controllers + networkController *networkController // nads to network mapping nads map[string]string @@ -93,84 +67,86 @@ type NetAttachDefinitionController struct { primaryNADs map[string]string } -func NewNetAttachDefinitionController( +func newController( name string, - ncm NetworkControllerManager, + zone string, + node string, + cm ControllerManager, wf watchFactory, recorder record.EventRecorder, -) (*NetAttachDefinitionController, error) { - nadController := &NetAttachDefinitionController{ - name: fmt.Sprintf("[%s NAD controller]", name), - recorder: recorder, - nads: map[string]string{}, - primaryNADs: map[string]string{}, - networkManager: newNetworkManager(name, ncm), +) (*nadController, error) { + c := &nadController{ + name: fmt.Sprintf("[%s NAD controller]", name), + recorder: recorder, + nadLister: wf.NADInformer().Lister(), + networkController: newNetworkController(name, zone, node, cm, wf), + nads: map[string]string{}, + primaryNADs: map[string]string{}, } config := &controller.ControllerConfig[nettypes.NetworkAttachmentDefinition]{ RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), - Reconcile: nadController.sync, + Informer: wf.NADInformer().Informer(), + Lister: c.nadLister.List, + Reconcile: c.sync, ObjNeedsUpdate: nadNeedsUpdate, Threadiness: 1, } - nadInformer := wf.NADInformer() - if nadInformer != nil { - nadController.netAttachDefLister = nadInformer.Lister() - config.Informer = nadInformer.Informer() - config.Lister = nadController.netAttachDefLister.List - } if util.IsNetworkSegmentationSupportEnabled() { if udnInformer := wf.UserDefinedNetworkInformer(); udnInformer != nil { - nadController.udnLister = udnInformer.Lister() + c.udnLister = udnInformer.Lister() } if cudnInformer := wf.ClusterUserDefinedNetworkInformer(); cudnInformer != nil { - nadController.cudnLister = cudnInformer.Lister() + c.cudnLister = cudnInformer.Lister() } if nsInformer := wf.NamespaceInformer(); nsInformer != nil { - nadController.namespaceLister = nsInformer.Lister() + c.namespaceLister = nsInformer.Lister() } } - - nadController.controller = controller.NewController( - nadController.name, + c.controller = controller.NewController( + c.name, config, ) - return nadController, nil + return c, nil +} + +func (c *nadController) Interface() Interface { + return c } -func (nadController *NetAttachDefinitionController) Start() error { +func (c *nadController) Start() error { // initial sync here will ensure networks in network manager // network manager will use this initial set of ensured networks to consider // any other network stale on its own sync err := controller.StartWithInitialSync( - nadController.syncAll, - nadController.controller, + c.syncAll, + c.controller, ) if err != nil { return err } - err = nadController.networkManager.Start() + err = c.networkController.Start() if err != nil { return err } - klog.Infof("%s: started", nadController.name) + klog.Infof("%s: started", c.name) return nil } -func (nadController *NetAttachDefinitionController) Stop() { - klog.Infof("%s: shutting down", nadController.name) - controller.Stop(nadController.controller) - nadController.networkManager.Stop() +func (c *nadController) Stop() { + klog.Infof("%s: shutting down", c.name) + controller.Stop(c.controller) + c.networkController.Stop() } -func (nadController *NetAttachDefinitionController) syncAll() (err error) { - existingNADs, err := nadController.netAttachDefLister.List(labels.Everything()) +func (c *nadController) syncAll() (err error) { + existingNADs, err := c.nadLister.List(labels.Everything()) if err != nil { - return fmt.Errorf("%s: failed to list NADs: %w", nadController.name, err) + return fmt.Errorf("%s: failed to list NADs: %w", c.name, err) } // create all networks with their updated list of NADs and only then start @@ -180,68 +156,69 @@ func (nadController *NetAttachDefinitionController) syncAll() (err error) { for _, nad := range existingNADs { key, err := cache.MetaNamespaceKeyFunc(nad) if err != nil { - klog.Errorf("%s: failed to sync %v: %v", nadController.name, nad, err) + klog.Errorf("%s: failed to sync %v: %v", c.name, nad, err) continue } - err = nadController.syncNAD(key, nad) + err = c.syncNAD(key, nad) if err != nil { - return fmt.Errorf("%s: failed to sync %s: %v", nadController.name, key, err) + return fmt.Errorf("%s: failed to sync %s: %v", c.name, key, err) } } return nil } -func (nadController *NetAttachDefinitionController) sync(key string) error { +func (c *nadController) sync(key string) error { startTime := time.Now() - klog.V(5).Infof("%s: sync NAD %s", nadController.name, key) + klog.V(5).Infof("%s: sync NAD %s", c.name, key) defer func() { - klog.V(4).Infof("%s: finished syncing NAD %s, took %v", nadController.name, key, time.Since(startTime)) + klog.V(4).Infof("%s: finished syncing NAD %s, took %v", c.name, key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { - klog.Errorf("%s: failed splitting key %s: %v", nadController.name, key, err) + klog.Errorf("%s: failed splitting key %s: %v", c.name, key, err) return nil } - nad, err := nadController.netAttachDefLister.NetworkAttachmentDefinitions(namespace).Get(name) + nad, err := c.nadLister.NetworkAttachmentDefinitions(namespace).Get(name) if err != nil && !apierrors.IsNotFound(err) { return err } - return nadController.syncNAD(key, nad) + return c.syncNAD(key, nad) } -func (nadController *NetAttachDefinitionController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefinition) error { +func (c *nadController) syncNAD(key string, nad *nettypes.NetworkAttachmentDefinition) error { var nadNetworkName string - var nadNetwork, oldNetwork, ensureNetwork util.NetInfo + var nadNetwork util.NetInfo + var oldNetwork, ensureNetwork util.MutableNetInfo var err error namespace, _, err := cache.SplitMetaNamespaceKey(key) if err != nil { - return fmt.Errorf("%s: failed splitting key %s: %v", nadController.name, key, err) + return fmt.Errorf("%s: failed splitting key %s: %v", c.name, key, err) } if nad != nil { nadNetwork, err = util.ParseNADInfo(nad) if err != nil { - if nadController.recorder != nil { - nadController.recorder.Eventf(&corev1.ObjectReference{Kind: nad.Kind, Namespace: nad.Namespace, Name: nad.Name}, corev1.EventTypeWarning, + if c.recorder != nil { + c.recorder.Eventf(&corev1.ObjectReference{Kind: nad.Kind, Namespace: nad.Namespace, Name: nad.Name}, corev1.EventTypeWarning, "InvalidConfig", "Failed to parse network config: %v", err.Error()) } - klog.Errorf("%s: failed parsing NAD %s: %v", nadController.name, key, err) + klog.Errorf("%s: failed parsing NAD %s: %v", c.name, key, err) return nil } nadNetworkName = nadNetwork.GetNetworkName() } - nadController.Lock() - defer nadController.Unlock() + c.Lock() + defer c.Unlock() // We can only have one primary NAD per namespace - primaryNAD := nadController.primaryNADs[namespace] + primaryNAD := c.primaryNADs[namespace] if nadNetwork != nil && nadNetwork.IsPrimaryNetwork() && primaryNAD != "" && primaryNAD != key { - return fmt.Errorf("%s: NAD %s is primary for the namespace, NAD %s can't be primary", nadController.name, primaryNAD, key) + return fmt.Errorf("%s: NAD %s is primary for the namespace, NAD %s can't be primary", c.name, primaryNAD, key) } // As multiple NADs may define networks with the same name, these networks @@ -252,16 +229,17 @@ func (nadController *NetAttachDefinitionController) syncNAD(key string, nad *net // - Return an error AND clean up NAD from the old network // the NAD refers to a different network than before - if nadNetworkName != nadController.nads[key] { - oldNetwork = nadController.networkManager.getNetwork(nadController.nads[key]) + if nadNetworkName != c.nads[key] { + oldNetwork = c.networkController.getNetwork(c.nads[key]) } - currentNetwork := nadController.networkManager.getNetwork(nadNetworkName) + + currentNetwork := c.networkController.getNetwork(nadNetworkName) switch { case currentNetwork == nil: // the NAD refers to a new network, ensure it - ensureNetwork = nadNetwork - case currentNetwork.Equals(nadNetwork): + ensureNetwork = util.NewMutableNetInfo(nadNetwork) + case util.AreNetworksCompatible(currentNetwork, nadNetwork): // the NAD refers to an existing compatible network, ensure that // existing network holds a reference to this NAD ensureNetwork = currentNetwork @@ -270,7 +248,7 @@ func (nadController *NetAttachDefinitionController) syncNAD(key string, nad *net // network, remove the reference from the old network and ensure that // existing network holds a reference to this NAD oldNetwork = currentNetwork - ensureNetwork = nadNetwork + ensureNetwork = util.NewMutableNetInfo(nadNetwork) // the NAD refers to an existing incompatible network referred by other // NADs, return error case oldNetwork == nil: @@ -279,7 +257,7 @@ func (nadController *NetAttachDefinitionController) syncNAD(key string, nad *net oldNetwork = currentNetwork fallthrough default: - err = fmt.Errorf("%s: NAD %s CNI config does not match that of network %s", nadController.name, key, nadNetworkName) + err = fmt.Errorf("%s: NAD %s CNI config does not match that of network %s", c.name, key, nadNetworkName) } // remove the NAD reference from the old network and delete the network if @@ -288,41 +266,36 @@ func (nadController *NetAttachDefinitionController) syncNAD(key string, nad *net oldNetworkName := oldNetwork.GetNetworkName() oldNetwork.DeleteNADs(key) if len(oldNetwork.GetNADs()) == 0 { - nadController.networkManager.DeleteNetwork(oldNetworkName) + c.networkController.DeleteNetwork(oldNetworkName) } else { - nadController.networkManager.EnsureNetwork(oldNetwork) + c.networkController.EnsureNetwork(oldNetwork) } } // this was a nad delete if ensureNetwork == nil { - delete(nadController.nads, key) - if nadController.primaryNADs[namespace] == key { - delete(nadController.primaryNADs, namespace) + delete(c.nads, key) + if c.primaryNADs[namespace] == key { + delete(c.primaryNADs, namespace) } return err } - if ensureNetwork.IsDefault() { - klog.V(5).Infof("%s: NAD add for default network (key %s), skip it", nadController.name, key) - return nil - } - // ensure the network is associated with the NAD ensureNetwork.AddNADs(key) - nadController.nads[key] = ensureNetwork.GetNetworkName() + c.nads[key] = ensureNetwork.GetNetworkName() // track primary NAD switch { case ensureNetwork.IsPrimaryNetwork(): - nadController.primaryNADs[namespace] = key + c.primaryNADs[namespace] = key default: - if nadController.primaryNADs[namespace] == key { - delete(nadController.primaryNADs, namespace) + if c.primaryNADs[namespace] == key { + delete(c.primaryNADs, namespace) } } // reconcile the network - nadController.networkManager.EnsureNetwork(ensureNetwork) + c.networkController.EnsureNetwork(ensureNetwork) return nil } @@ -337,33 +310,35 @@ func nadNeedsUpdate(oldNAD, newNAD *nettypes.NetworkAttachmentDefinition) bool { return false } - return !reflect.DeepEqual(oldNAD.Spec, newNAD.Spec) + // also reconcile the network in case its route advertisements changed + return !reflect.DeepEqual(oldNAD.Spec, newNAD.Spec) || + oldNAD.Annotations[types.OvnRouteAdvertisementsKey] != newNAD.Annotations[types.OvnRouteAdvertisementsKey] } -func (nadController *NetAttachDefinitionController) GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) { +func (c *nadController) GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) { if !util.IsNetworkSegmentationSupportEnabled() { return &util.DefaultNetInfo{}, nil } - nadController.RLock() - defer nadController.RUnlock() - primaryNAD := nadController.primaryNADs[namespace] + c.RLock() + defer c.RUnlock() + primaryNAD := c.primaryNADs[namespace] if primaryNAD != "" { // we have a primary NAD, get the network - netName := nadController.nads[primaryNAD] + netName := c.nads[primaryNAD] if netName == "" { // this should never happen where we have a nad keyed in the primaryNADs // map, but it doesn't exist in the nads map panic("NAD Controller broken consistency between primary NADs and cached NADs") } - network := nadController.networkManager.getNetwork(netName) - n := util.CopyNetInfo(network) + network := c.networkController.getNetwork(netName) + n := util.NewMutableNetInfo(network) // update the returned netInfo copy to only have the primary NAD for this namespace n.SetNADs(primaryNAD) return n, nil } // no primary network found, make sure we just haven't processed it yet and no UDN / CUDN exists - udns, err := nadController.udnLister.UserDefinedNetworks(namespace).List(labels.Everything()) + udns, err := c.udnLister.UserDefinedNetworks(namespace).List(labels.Everything()) if err != nil { return nil, fmt.Errorf("error getting user defined networks: %w", err) } @@ -372,7 +347,7 @@ func (nadController *NetAttachDefinitionController) GetActiveNetworkForNamespace return nil, util.NewUnprocessedActiveNetworkError(namespace, udn.Name) } } - cudns, err := nadController.cudnLister.List(labels.Everything()) + cudns, err := c.cudnLister.List(labels.Everything()) if err != nil { return nil, fmt.Errorf("failed to list CUDNs: %w", err) } @@ -385,7 +360,7 @@ func (nadController *NetAttachDefinitionController) GetActiveNetworkForNamespace if err != nil { return nil, fmt.Errorf("failed to convert CUDN %q namespaceSelector: %w", cudn.Name, err) } - selectedNamespaces, err := nadController.namespaceLister.List(cudnNamespaceSelector) + selectedNamespaces, err := c.namespaceLister.List(cudnNamespaceSelector) if err != nil { return nil, fmt.Errorf("failed to list namespaces using selector %q: %w", cudnNamespaceSelector, err) } @@ -399,26 +374,15 @@ func (nadController *NetAttachDefinitionController) GetActiveNetworkForNamespace return &util.DefaultNetInfo{}, nil } -func (nadController *NetAttachDefinitionController) GetNetwork(networkName string) (util.NetInfo, error) { - if !util.IsNetworkSegmentationSupportEnabled() { - return &util.DefaultNetInfo{}, nil +func (c *nadController) GetNetwork(name string) util.NetInfo { + network := c.networkController.getNetwork(name) + if network == nil && name == types.DefaultNetworkName { + return &util.DefaultNetInfo{} } - nadController.RLock() - defer nadController.RUnlock() - if networkName == "" { - return nil, fmt.Errorf("network must not be empty") - } - if networkName == "default" { - return &util.DefaultNetInfo{}, nil - } - network := nadController.networkManager.getNetwork(networkName) - if network == nil { - return nil, fmt.Errorf("failed to find network %q", networkName) - } - return util.CopyNetInfo(network), nil + return network } -func (nadController *NetAttachDefinitionController) GetActiveNetworkNamespaces(networkName string) ([]string, error) { +func (nadController *nadController) GetActiveNetworkNamespaces(networkName string) ([]string, error) { if !util.IsNetworkSegmentationSupportEnabled() { return []string{"default"}, nil } @@ -437,7 +401,7 @@ func (nadController *NetAttachDefinitionController) GetActiveNetworkNamespaces(n // DoWithLock iterates over all role primary user defined networks and executes the given fn with each network as input. // An error will not block execution and instead all errors will be aggregated and returned when all networks are processed. -func (nadController *NetAttachDefinitionController) DoWithLock(f func(network util.NetInfo) error) error { +func (nadController *nadController) DoWithLock(f func(network util.NetInfo) error) error { if !util.IsNetworkSegmentationSupportEnabled() { defaultNetwork := &util.DefaultNetInfo{} return f(defaultNetwork) @@ -456,16 +420,13 @@ func (nadController *NetAttachDefinitionController) DoWithLock(f func(network ut // map, but it doesn't exist in the nads map panic("NAD Controller broken consistency between primary NADs and cached NADs") } - network := nadController.networkManager.getNetwork(netName) - n := util.CopyNetInfo(network) + network := nadController.networkController.getNetwork(netName) + n := util.NewMutableNetInfo(network) // update the returned netInfo copy to only have the primary NAD for this namespace n.SetNADs(primaryNAD) if err := f(n); err != nil { errs = append(errs, err) } } - if len(errs) > 0 { - return errors.Join(errs...) - } - return nil + return errors.Join(errs...) } diff --git a/go-controller/pkg/network-attach-def-controller/network_attach_def_controller_test.go b/go-controller/pkg/networkmanager/nad_controller_test.go similarity index 73% rename from go-controller/pkg/network-attach-def-controller/network_attach_def_controller_test.go rename to go-controller/pkg/networkmanager/nad_controller_test.go index aff03de77a..de404ddb61 100644 --- a/go-controller/pkg/network-attach-def-controller/network_attach_def_controller_test.go +++ b/go-controller/pkg/networkmanager/nad_controller_test.go @@ -1,10 +1,9 @@ -package networkAttachDefController +package networkmanager import ( "context" "encoding/json" "fmt" - "reflect" "sync" "testing" @@ -25,73 +24,84 @@ import ( ) type testNetworkController struct { - util.NetInfo - tncm *testNetworkControllerManager + util.ReconcilableNetInfo + tcm *testControllerManager } func (tnc *testNetworkController) Start(context.Context) error { - tnc.tncm.Lock() - defer tnc.tncm.Unlock() + tnc.tcm.Lock() + defer tnc.tcm.Unlock() fmt.Printf("starting network: %s\n", testNetworkKey(tnc)) - tnc.tncm.started = append(tnc.tncm.started, testNetworkKey(tnc)) + tnc.tcm.started = append(tnc.tcm.started, testNetworkKey(tnc)) return nil } func (tnc *testNetworkController) Stop() { - tnc.tncm.Lock() - defer tnc.tncm.Unlock() - tnc.tncm.stopped = append(tnc.tncm.stopped, testNetworkKey(tnc)) + tnc.tcm.Lock() + defer tnc.tcm.Unlock() + tnc.tcm.stopped = append(tnc.tcm.stopped, testNetworkKey(tnc)) } func (tnc *testNetworkController) Cleanup() error { - tnc.tncm.Lock() - defer tnc.tncm.Unlock() - tnc.tncm.cleaned = append(tnc.tncm.cleaned, testNetworkKey(tnc)) + tnc.tcm.Lock() + defer tnc.tcm.Unlock() + tnc.tcm.cleaned = append(tnc.tcm.cleaned, testNetworkKey(tnc)) return nil } +func (tnc *testNetworkController) Reconcile(netInfo util.NetInfo) error { + return util.ReconcileNetInfo(tnc.ReconcilableNetInfo, netInfo) +} + // GomegaString is used to avoid printing embedded mutexes which can cause a // race func (tnc *testNetworkController) GomegaString() string { - return format.Object(tnc.NetInfo.GetNetworkName(), 1) + return format.Object(tnc.GetNetworkName(), 1) } func testNetworkKey(nInfo util.NetInfo) string { return nInfo.GetNetworkName() + " " + nInfo.TopologyType() } -type testNetworkControllerManager struct { +type testControllerManager struct { sync.Mutex - controllers map[string]NetworkController - started []string - stopped []string - cleaned []string + + defaultNetwork *testNetworkController + controllers map[string]NetworkController + + started []string + stopped []string + cleaned []string raiseErrorWhenCreatingController error - valid []util.BasicNetInfo + valid []util.NetInfo } -func (tncm *testNetworkControllerManager) NewNetworkController(netInfo util.NetInfo) (NetworkController, error) { - tncm.Lock() - defer tncm.Unlock() - if tncm.raiseErrorWhenCreatingController != nil { - return nil, tncm.raiseErrorWhenCreatingController +func (tcm *testControllerManager) NewNetworkController(netInfo util.NetInfo) (NetworkController, error) { + tcm.Lock() + defer tcm.Unlock() + if tcm.raiseErrorWhenCreatingController != nil { + return nil, tcm.raiseErrorWhenCreatingController } t := &testNetworkController{ - NetInfo: netInfo, - tncm: tncm, + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), + tcm: tcm, } - tncm.controllers[testNetworkKey(netInfo)] = t + tcm.controllers[testNetworkKey(netInfo)] = t return t, nil } -func (tncm *testNetworkControllerManager) CleanupDeletedNetworks(validNetworks ...util.BasicNetInfo) error { - tncm.valid = validNetworks +func (tcm *testControllerManager) CleanupStaleNetworks(validNetworks ...util.NetInfo) error { + tcm.valid = validNetworks return nil } -func TestNetAttachDefinitionController(t *testing.T) { +func (tcm *testControllerManager) GetDefaultNetworkController() ReconcilableNetworkController { + return tcm.defaultNetwork +} + +func TestNADController(t *testing.T) { networkAPrimary := &ovncnitypes.NetConf{ Topology: types.Layer2Topology, NetConf: cnitypes.NetConf{ @@ -131,9 +141,8 @@ func TestNetAttachDefinitionController(t *testing.T) { } networkDefault := &ovncnitypes.NetConf{ - Topology: types.Layer3Topology, NetConf: cnitypes.NetConf{ - Name: "default", + Name: types.DefaultNetworkName, Type: "ovn-k8s-cni-overlay", }, MTU: 1400, @@ -154,14 +163,19 @@ func TestNetAttachDefinitionController(t *testing.T) { expected []expected }{ { - name: "NAD on default network should be skipped", + name: "NAD on default network is tracked with default controller", args: []args{ { nad: "test/nad_1", network: networkDefault, }, }, - expected: []expected{}, + expected: []expected{ + { + network: networkDefault, + nads: []string{"test/nad_1"}, + }, + }, }, { name: "NAD added", @@ -432,17 +446,21 @@ func TestNetAttachDefinitionController(t *testing.T) { g.Expect(err).ToNot(gomega.HaveOccurred()) config.OVNKubernetesFeature.EnableNetworkSegmentation = true config.OVNKubernetesFeature.EnableMultiNetwork = true - tncm := &testNetworkControllerManager{ + tcm := &testControllerManager{ controllers: map[string]NetworkController{}, + defaultNetwork: &testNetworkController{ + ReconcilableNetInfo: &util.DefaultNetInfo{}, + }, } - nadController := &NetAttachDefinitionController{ - nads: map[string]string{}, - networkManager: newNetworkManager("", tncm), - primaryNADs: map[string]string{}, + nadController := &nadController{ + nads: map[string]string{}, + primaryNADs: map[string]string{}, + networkController: newNetworkController("", "", "", tcm, nil), } + netController := nadController.networkController - g.Expect(nadController.networkManager.Start()).To(gomega.Succeed()) - defer nadController.networkManager.Stop() + g.Expect(nadController.networkController.Start()).To(gomega.Succeed()) + defer nadController.networkController.Stop() for _, args := range tt.args { namespace, name, err := cache.SplitMetaNamespaceKey(args.nad) @@ -464,6 +482,9 @@ func TestNetAttachDefinitionController(t *testing.T) { } meetsExpectations := func(g gomega.Gomega) { + // test that the manager has all the desired networks + g.Expect(netController.getAllNetworks()).To(gomega.HaveLen(len(tt.expected))) + var expectRunning []string for _, expected := range tt.expected { netInfo, err := util.NewNetInfo(expected.network) @@ -472,33 +493,47 @@ func TestNetAttachDefinitionController(t *testing.T) { name := netInfo.GetNetworkName() testNetworkKey := testNetworkKey(netInfo) func() { - tncm.Lock() - defer tncm.Unlock() - // test that the controller have the expected config and NADs - g.Expect(tncm.controllers).To(gomega.HaveKey(testNetworkKey)) - g.Expect(tncm.controllers[testNetworkKey].Equals(netInfo)).To(gomega.BeTrue(), + netController.Lock() + defer netController.Unlock() + tcm.Lock() + defer tcm.Unlock() + + // test that the desired networks have the expected + // config and NADs, including the default network which + // could have had NAD/Advertisement changes as well + g.Expect(netController.networks).To(gomega.HaveKey(name)) + g.Expect(util.AreNetworksCompatible(netController.networks[name], netInfo)).To(gomega.BeTrue(), fmt.Sprintf("matching network config for network %s", name)) - g.Expect(tncm.controllers[testNetworkKey].GetNADs()).To(gomega.ConsistOf(expected.nads), + g.Expect(netController.networks[name].GetNADs()).To(gomega.ConsistOf(expected.nads), fmt.Sprintf("matching NADs for network %s", name)) + + // test that the actual controllers have the expected config and NADs + if !netInfo.IsDefault() { + g.Expect(tcm.controllers).To(gomega.HaveKey(testNetworkKey)) + g.Expect(util.AreNetworksCompatible(tcm.controllers[testNetworkKey], netInfo)).To(gomega.BeTrue(), + fmt.Sprintf("matching network config for network %s", name)) + g.Expect(tcm.controllers[testNetworkKey].GetNADs()).To(gomega.ConsistOf(expected.nads), + fmt.Sprintf("matching NADs for network %s", name)) + expectRunning = append(expectRunning, testNetworkKey) + } }() - expectRunning = append(expectRunning, testNetworkKey) if netInfo.IsPrimaryNetwork() && !netInfo.IsDefault() { - netInfo.SetNADs(expected.nads...) key := expected.nads[0] namespace, _, err := cache.SplitMetaNamespaceKey(key) g.Expect(err).ToNot(gomega.HaveOccurred()) netInfoFound, err := nadController.GetActiveNetworkForNamespace(namespace) g.Expect(err).ToNot(gomega.HaveOccurred()) - g.Expect(reflect.DeepEqual(netInfoFound, netInfo)).To(gomega.BeTrue()) + g.Expect(util.AreNetworksCompatible(netInfoFound, netInfo)).To(gomega.BeTrue()) + g.Expect(netInfoFound.GetNADs()).To(gomega.ConsistOf(expected.nads)) } } - tncm.Lock() - defer tncm.Unlock() - expectStopped := sets.New(tncm.started...).Difference(sets.New(expectRunning...)).UnsortedList() + tcm.Lock() + defer tcm.Unlock() + expectStopped := sets.New(tcm.started...).Difference(sets.New(expectRunning...)).UnsortedList() // test that the controllers are started, stopped and cleaned up as expected - g.Expect(tncm.started).To(gomega.ContainElements(expectRunning), "started network controllers") - g.Expect(tncm.stopped).To(gomega.ConsistOf(expectStopped), "stopped network controllers") - g.Expect(tncm.cleaned).To(gomega.ConsistOf(expectStopped), "cleaned up network controllers") + g.Expect(tcm.started).To(gomega.ContainElements(expectRunning), "started network controllers") + g.Expect(tcm.stopped).To(gomega.ConsistOf(expectStopped), "stopped network controllers") + g.Expect(tcm.cleaned).To(gomega.ConsistOf(expectStopped), "cleaned up network controllers") } g.Eventually(meetsExpectations).Should(gomega.Succeed()) @@ -566,23 +601,22 @@ func TestSyncAll(t *testing.T) { wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClient) g.Expect(err).ToNot(gomega.HaveOccurred()) - tncm := &testNetworkControllerManager{ + tcm := &testControllerManager{ controllers: map[string]NetworkController{}, } if tt.syncAllError != nil { - tncm.raiseErrorWhenCreatingController = tt.syncAllError + tcm.raiseErrorWhenCreatingController = tt.syncAllError } - nadController, err := NewNetAttachDefinitionController( - "SUT", - tncm, + controller, err := NewForCluster( + tcm, wf, nil, ) g.Expect(err).ToNot(gomega.HaveOccurred()) expectedNetworks := map[string]util.NetInfo{} - expectedPrimaryNetworks := map[string]util.BasicNetInfo{} + expectedPrimaryNetworks := map[string]util.NetInfo{} for _, testNAD := range tt.testNADs { namespace, name, err := cache.SplitMetaNamespaceKey(testNAD.name) g.Expect(err).ToNot(gomega.HaveOccurred()) @@ -610,26 +644,27 @@ func TestSyncAll(t *testing.T) { g.Expect(err).ToNot(gomega.HaveOccurred()) defer wf.Shutdown() - err = nadController.Start() + err = controller.Start() g.Expect(err).ToNot(gomega.HaveOccurred()) // sync has already happened, stop - nadController.Stop() + controller.Stop() - actualNetworks := map[string]util.BasicNetInfo{} - for _, network := range tncm.valid { + actualNetworks := map[string]util.NetInfo{} + for _, network := range tcm.valid { actualNetworks[network.GetNetworkName()] = network } g.Expect(actualNetworks).To(gomega.HaveLen(len(expectedNetworks))) for name, network := range expectedNetworks { g.Expect(actualNetworks).To(gomega.HaveKey(name)) - g.Expect(actualNetworks[name].Equals(network)).To(gomega.BeTrue()) + g.Expect(util.AreNetworksCompatible(actualNetworks[name], network)).To(gomega.BeTrue()) } - actualPrimaryNetwork, err := nadController.GetActiveNetworkForNamespace("test") + actualPrimaryNetwork, err := controller.Interface().GetActiveNetworkForNamespace("test") g.Expect(err).ToNot(gomega.HaveOccurred()) g.Expect(expectedPrimaryNetworks).To(gomega.HaveKey(actualPrimaryNetwork.GetNetworkName())) - g.Expect(expectedPrimaryNetworks[actualPrimaryNetwork.GetNetworkName()].Equals(actualPrimaryNetwork)).To(gomega.BeTrue()) + expectedPrimaryNetwork := expectedPrimaryNetworks[actualPrimaryNetwork.GetNetworkName()] + g.Expect(util.AreNetworksCompatible(expectedPrimaryNetwork, actualPrimaryNetwork)).To(gomega.BeTrue()) }) } } diff --git a/go-controller/pkg/networkmanager/network_controller.go b/go-controller/pkg/networkmanager/network_controller.go new file mode 100644 index 0000000000..e697a0ea36 --- /dev/null +++ b/go-controller/pkg/networkmanager/network_controller.go @@ -0,0 +1,523 @@ +package networkmanager + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "sync" + "time" + + nadlisters "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" + ratypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1" + ralisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1/apis/listers/routeadvertisements/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +func newNetworkController(name, zone, node string, cm ControllerManager, wf watchFactory) *networkController { + nc := &networkController{ + name: fmt.Sprintf("[%s network controller]", name), + node: node, + zone: zone, + cm: cm, + networks: map[string]util.MutableNetInfo{}, + networkControllers: map[string]*networkControllerState{}, + } + + // this controller does not feed from an informer, networks are manually + // added to the queue for processing + networkConfig := &controller.ReconcilerConfig{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Reconcile: nc.syncNetwork, + Threadiness: 1, + } + nc.networkReconciler = controller.NewReconciler( + nc.name, + networkConfig, + ) + + // we don't care about route advertisements in cluster manager + if nc.hasRouteAdvertisements() { + nc.nadLister = wf.NADInformer().Lister() + nc.raLister = wf.RouteAdvertisementsInformer().Lister() + nc.nodeLister = wf.NodeCoreInformer().Lister() + + // ra controller + raConfig := &controller.ControllerConfig[ratypes.RouteAdvertisements]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Informer: wf.RouteAdvertisementsInformer().Informer(), + Lister: nc.raLister.List, + Reconcile: func(string) error { return nc.syncRunningNetworks() }, + ObjNeedsUpdate: raNeedsUpdate, + Threadiness: 1, + } + nc.raController = controller.NewController( + nc.name, + raConfig, + ) + + // node controller + nodeConfig := &controller.ControllerConfig[corev1.Node]{ + RateLimiter: workqueue.DefaultTypedControllerRateLimiter[string](), + Informer: wf.NodeCoreInformer().Informer(), + Lister: nc.nodeLister.List, + Reconcile: func(string) error { return nc.syncRunningNetworks() }, + ObjNeedsUpdate: nodeNeedsUpdate, + Threadiness: 1, + } + nc.nodeController = controller.NewController( + nc.name, + nodeConfig, + ) + } + + return nc +} + +type networkControllerState struct { + controller NetworkController + stoppedAndDeleting bool +} + +type networkController struct { + sync.RWMutex + + name string + zone string + node string + + nadLister nadlisters.NetworkAttachmentDefinitionLister + raLister ralisters.RouteAdvertisementsLister + nodeLister corelisters.NodeLister + + networkReconciler controller.Reconciler + raController controller.Controller + nodeController controller.Controller + + cm ControllerManager + networks map[string]util.MutableNetInfo + networkControllers map[string]*networkControllerState +} + +// Start will cleanup stale networks that have not been ensured via +// EnsuredNetwork before this call +func (c *networkController) Start() error { + controllers := []controller.Reconciler{c.networkReconciler} + if c.raController != nil { + controllers = append(controllers, c.raController) + } + if c.nodeController != nil { + controllers = append(controllers, c.nodeController) + } + return controller.StartWithInitialSync( + c.syncAll, + controllers..., + ) +} + +func (c *networkController) Stop() { + controllers := []controller.Reconciler{c.networkReconciler} + if c.raController != nil { + controllers = append(controllers, c.raController) + } + if c.nodeController != nil { + controllers = append(controllers, c.nodeController) + } + controller.Stop(controllers...) + for _, networkControllerState := range c.getAllNetworkStates() { + if networkControllerState.controller.GetNetworkName() == types.DefaultNetworkName { + // we don't own the lifecycle of the default network, so don't stop + // it + continue + } + networkControllerState.controller.Stop() + } +} + +func (c *networkController) EnsureNetwork(network util.MutableNetInfo) { + c.setNetwork(network.GetNetworkName(), network) + c.networkReconciler.Reconcile(network.GetNetworkName()) +} + +func (c *networkController) DeleteNetwork(network string) { + switch network { + case types.DefaultNetworkName: + // for the default network however ensure it runs with the default + // config + c.setNetwork(network, &util.DefaultNetInfo{}) + default: + c.setNetwork(network, nil) + } + c.networkReconciler.Reconcile(network) +} + +func (c *networkController) setNetwork(network string, netInfo util.MutableNetInfo) { + c.Lock() + defer c.Unlock() + if netInfo == nil { + delete(c.networks, network) + return + } + c.networks[network] = netInfo +} + +func (c *networkController) getNetwork(network string) util.MutableNetInfo { + c.RLock() + defer c.RUnlock() + return c.networks[network] +} + +func (c *networkController) getAllNetworks() []util.NetInfo { + c.RLock() + defer c.RUnlock() + networks := make([]util.NetInfo, 0, len(c.networks)) + for _, network := range c.networks { + networks = append(networks, network) + } + return networks +} + +func (c *networkController) setNetworkState(network string, state *networkControllerState) { + c.Lock() + defer c.Unlock() + if state == nil { + delete(c.networkControllers, network) + return + } + c.networkControllers[network] = state +} + +func (c *networkController) getNetworkState(network string) *networkControllerState { + c.RLock() + defer c.RUnlock() + state := c.networkControllers[network] + if state == nil { + return &networkControllerState{} + } + return state +} + +func (c *networkController) getReconcilableNetworkState(network string) (ReconcilableNetworkController, bool) { + if network == types.DefaultNetworkName { + return c.cm.GetDefaultNetworkController(), false + } + state := c.getNetworkState(network) + return state.controller, state.stoppedAndDeleting +} + +func (c *networkController) getAllNetworkStates() []*networkControllerState { + c.RLock() + defer c.RUnlock() + networkStates := make([]*networkControllerState, 0, len(c.networks)) + for _, state := range c.networkControllers { + networkStates = append(networkStates, state) + } + return networkStates +} + +func (c *networkController) syncAll() error { + // as we sync upon start, consider networks that have not been ensured as + // stale and clean them up + validNetworks := c.getAllNetworks() + if err := c.cm.CleanupStaleNetworks(validNetworks...); err != nil { + return err + } + + // sync all known networks. There is no informer for networks. Keys are added by NAD controller. + // Certain downstream controllers that handle configuration for multiple networks depend on being + // aware of all the existing networks on initialization. To achieve that, we need to start existing + // networks synchronously. Otherwise, these controllers might incorrectly assess valid configuration + // as stale. + start := time.Now() + klog.Infof("%s: syncing all networks", c.name) + for _, network := range validNetworks { + err := c.syncNetwork(network.GetNetworkName()) + if errors.Is(err, ErrNetworkControllerTopologyNotManaged) { + klog.V(5).Infof("Ignoring network %q since %q does not manage it", network.GetNetworkName(), c.name) + continue + } + if err != nil { + return fmt.Errorf("failed to sync network %s: %w", network.GetNetworkName(), err) + } + } + klog.Infof("%s: finished syncing all networks. Time taken: %s", c.name, time.Since(start)) + return nil +} + +func (c *networkController) syncRunningNetworks() error { + c.networkReconciler.Reconcile(types.DefaultNetworkName) + for _, network := range c.getAllNetworkStates() { + c.networkReconciler.Reconcile(network.controller.GetNetworkName()) + } + + return nil +} + +// syncNetwork must be called with nm mutex locked +func (c *networkController) syncNetwork(network string) error { + startTime := time.Now() + klog.V(5).Infof("%s: sync network %s", c.name, network) + defer func() { + klog.V(4).Infof("%s: finished syncing network %s, took %v", c.name, network, time.Since(startTime)) + }() + + have, stoppedAndDeleting := c.getReconcilableNetworkState(network) + want := c.getNetwork(network) + + compatible := util.AreNetworksCompatible(have, want) + + // we will dispose of the old network if deletion is in progress or if + // non-reconcilable configuration changed + dispose := stoppedAndDeleting || !compatible + if dispose { + err := c.deleteNetwork(network) + if err != nil { + return err + } + have = nil + } + + // fetch other relevant network information + err := c.gatherNetwork(want) + if err != nil { + return fmt.Errorf("failed to fetch other network information for network %s: %w", network, err) + } + + ensureNetwork := !compatible || util.DoesNetworkNeedReconciliation(have, want) + if !ensureNetwork { + // no network changes + return nil + } + + // ensure the network controller + err = c.ensureNetwork(want) + if err != nil { + return fmt.Errorf("%s: failed to ensure network %s: %w", c.name, network, err) + } + + return nil +} + +func (c *networkController) ensureNetwork(network util.MutableNetInfo) error { + if network == nil { + return nil + } + + networkName := network.GetNetworkName() + reconcilable, _ := c.getReconcilableNetworkState(networkName) + + // this might just be an update of reconcilable network configuration + if reconcilable != nil { + err := reconcilable.Reconcile(network) + if err != nil { + return fmt.Errorf("failed to reconcile network %s: %w", networkName, err) + } + return nil + } + + // otherwise setup & start the new network controller + nc, err := c.cm.NewNetworkController(network) + if err != nil { + return fmt.Errorf("failed to create network %s: %w", networkName, err) + } + + err = nc.Start(context.Background()) + if err != nil { + return fmt.Errorf("failed to start network %s: %w", networkName, err) + } + c.setNetworkState(network.GetNetworkName(), &networkControllerState{controller: nc}) + + return nil +} + +func (c *networkController) deleteNetwork(network string) error { + have := c.getNetworkState(network) + if have.controller == nil { + return nil + } + + if !have.stoppedAndDeleting { + have.controller.Stop() + } + have.stoppedAndDeleting = true + + err := have.controller.Cleanup() + if err != nil { + return fmt.Errorf("%s: failed to cleanup network %s: %w", c.name, network, err) + } + + c.setNetworkState(network, nil) + return nil +} + +func (c *networkController) gatherNetwork(network util.MutableNetInfo) error { + if network == nil { + return nil + } + return c.setAdvertisements(network) +} + +func (c *networkController) setAdvertisements(network util.MutableNetInfo) error { + if !network.IsDefault() && !network.IsPrimaryNetwork() { + return nil + } + if !c.hasRouteAdvertisements() { + return nil + } + + raNames := sets.New[string]() + for _, nadNamespacedName := range network.GetNADs() { + namespace, name, err := cache.SplitMetaNamespaceKey(nadNamespacedName) + if err != nil { + return err + } + + nad, err := c.nadLister.NetworkAttachmentDefinitions(namespace).Get(name) + if err != nil { + return err + } + + var nadRANames []string + if nad.Annotations[types.OvnRouteAdvertisementsKey] != "" { + err = json.Unmarshal([]byte(nad.Annotations[types.OvnRouteAdvertisementsKey]), &nadRANames) + if err != nil { + return err + } + } + + raNames.Insert(nadRANames...) + } + + podAdvertisements := map[string][]string{} + eipAdvertisements := map[string][]string{} + for raName := range raNames { + ra, err := c.raLister.Get(raName) + if err != nil { + return err + } + + advertisements := sets.New(ra.Spec.Advertisements...) + if !advertisements.Has(ratypes.PodNetwork) { + continue + } + + accepted := meta.FindStatusCondition(ra.Status.Conditions, "Accepted") + if accepted == nil { + // if there is no status we can safely ignore + continue + } + if accepted.Status != metav1.ConditionTrue || accepted.ObservedGeneration != ra.Generation { + // if the RA is not accepted, we commit to no change, best to + // preserve the old config while we can't validate new config + return fmt.Errorf("failed to reconcile network %q: RouteAdvertisements %q not in accepted status", network.GetNetworkName(), ra.Name) + } + + nodeSelector, err := metav1.LabelSelectorAsSelector(&ra.Spec.NodeSelector) + if err != nil { + return err + } + + nodes, err := c.nodeLister.List(nodeSelector) + if err != nil { + return err + } + + vrf := ra.Spec.TargetVRF + if vrf == "" { + vrf = types.DefaultNetworkName + } + + for _, node := range nodes { + if !c.isNodeManaged(node) { + continue + } + if advertisements.Has(ratypes.PodNetwork) { + podAdvertisements[node.Name] = append(podAdvertisements[node.Name], vrf) + } + if advertisements.Has(ratypes.EgressIP) { + eipAdvertisements[node.Name] = append(eipAdvertisements[node.Name], vrf) + } + } + } + network.SetPodNetworkAdvertisedVRFs(podAdvertisements) + network.SetEgressIPAdvertisedVRFs(eipAdvertisements) + return nil +} + +func (c *networkController) hasRouteAdvertisements() bool { + return util.IsRouteAdvertisementsEnabled() +} + +func (c *networkController) isNodeManaged(node *corev1.Node) bool { + switch { + case c.node == "" && c.zone == "": + // cluster manager manages all nodes + return true + case util.GetNodeZone(node) == c.zone: + // ovnkube-controller manages nodes of its zone + return true + case node.Name == c.node: + // ovnkube-node only manages a specific node + return true + } + return false +} + +func raNeedsUpdate(oldRA, newRA *ratypes.RouteAdvertisements) bool { + if oldRA == nil || newRA == nil { + // handle RA add/delete through the NAD annotation update + return false + } + + // don't process resync or objects that are marked for deletion + if oldRA.ResourceVersion == newRA.ResourceVersion || + !newRA.GetDeletionTimestamp().IsZero() { + return false + } + + // the RA needs to be accepted on its current generation, otherwise we + // commit to no change + newCondition := meta.FindStatusCondition(newRA.Status.Conditions, "Accepted") + if newCondition == nil { + return false + } + if newCondition.Status != metav1.ConditionTrue { + return false + } + if newCondition.ObservedGeneration != newRA.Generation { + return false + } + // there had to be a change on observed generation or status, otherwise we + // already processed this RA in its current form + oldCondition := meta.FindStatusCondition(oldRA.Status.Conditions, "Accepted") + if oldCondition == nil { + return true + } + return oldCondition.ObservedGeneration != newCondition.ObservedGeneration || oldCondition.Status != newCondition.Status +} + +func nodeNeedsUpdate(oldNode, newNode *corev1.Node) bool { + if oldNode == nil || newNode == nil { + return true + } + + // don't process resync or objects that are marked for deletion + if oldNode.ResourceVersion == newNode.ResourceVersion || + !newNode.GetDeletionTimestamp().IsZero() { + return false + } + + return !reflect.DeepEqual(oldNode.Labels, newNode.Labels) || oldNode.Annotations[util.OvnNodeZoneName] != newNode.Annotations[util.OvnNodeZoneName] +} diff --git a/go-controller/pkg/networkmanager/network_controller_test.go b/go-controller/pkg/networkmanager/network_controller_test.go new file mode 100644 index 0000000000..a477c2192c --- /dev/null +++ b/go-controller/pkg/networkmanager/network_controller_test.go @@ -0,0 +1,240 @@ +package networkmanager + +import ( + "context" + "testing" + + "github.com/onsi/gomega" + + cnitypes "github.com/containernetworking/cni/pkg/types" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" + + ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + ratypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/routeadvertisements/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +func TestSetAdvertisements(t *testing.T) { + testZoneName := "testZone" + testNodeName := "testNode" + testNodeOnZoneName := "testNodeOnZone" + testNADName := "test/NAD" + testRAName := "testRA" + testVRFName := "testVRF" + + defaultNetwork := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: types.DefaultNetworkName, + Type: "ovn-k8s-cni-overlay", + }, + MTU: 1400, + } + primaryNetwork := &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: "primary", + Type: "ovn-k8s-cni-overlay", + }, + Topology: "layer3", + Role: "primary", + MTU: 1400, + } + + podNetworkRA := ratypes.RouteAdvertisements{ + ObjectMeta: v1.ObjectMeta{ + Name: testRAName, + }, + Spec: ratypes.RouteAdvertisementsSpec{ + TargetVRF: testVRFName, + Advertisements: []ratypes.AdvertisementType{ + ratypes.PodNetwork, + }, + }, + Status: ratypes.RouteAdvertisementsStatus{ + Conditions: []v1.Condition{ + { + Type: "Accepted", + Status: v1.ConditionTrue, + }, + }, + }, + } + nonPodNetworkRA := ratypes.RouteAdvertisements{ + ObjectMeta: v1.ObjectMeta{ + Name: testRAName, + }, + Spec: ratypes.RouteAdvertisementsSpec{ + TargetVRF: testVRFName, + }, + Status: ratypes.RouteAdvertisementsStatus{ + Conditions: []v1.Condition{ + { + Type: "Accepted", + Status: v1.ConditionTrue, + }, + }, + }, + } + podNetworkRANotAccepted := podNetworkRA + podNetworkRANotAccepted.Status = ratypes.RouteAdvertisementsStatus{} + podNetworkRARejected := *podNetworkRA.DeepCopy() + podNetworkRARejected.Status.Conditions[0].Status = v1.ConditionFalse + podNetworkRAOutdated := podNetworkRA + podNetworkRAOutdated.Generation = 1 + + testNode := corev1.Node{ + ObjectMeta: v1.ObjectMeta{ + Name: testNodeName, + }, + } + testNodeOnZone := corev1.Node{ + ObjectMeta: v1.ObjectMeta{ + Name: testNodeOnZoneName, + Annotations: map[string]string{ + util.OvnNodeZoneName: testZoneName, + }, + }, + } + otherNode := corev1.Node{ + ObjectMeta: v1.ObjectMeta{ + Name: "otherNode", + }, + } + + tests := []struct { + name string + network *ovncnitypes.NetConf + ra *ratypes.RouteAdvertisements + node corev1.Node + expectNoNetwork bool + expected map[string][]string + }{ + { + name: "reconciles VRF advertisements for selected node of default node network controller", + network: defaultNetwork, + ra: &podNetworkRA, + node: testNode, + expected: map[string][]string{ + testNodeName: {testVRFName}, + }, + }, + { + name: "reconciles VRF advertisements for selected node in same zone as default OVN network controller", + network: primaryNetwork, + ra: &podNetworkRA, + node: testNodeOnZone, + expected: map[string][]string{ + testNodeOnZoneName: {testVRFName}, + }, + }, + { + name: "ignores advertisements that are not for the pod network", + network: defaultNetwork, + ra: &nonPodNetworkRA, + node: testNode, + }, + { + name: "ignores advertisements that are not for applicable node", + network: defaultNetwork, + ra: &podNetworkRA, + node: otherNode, + }, + { + name: "ignores advertisements that are not accepted", + network: defaultNetwork, + ra: &podNetworkRANotAccepted, + node: testNode, + }, + { + name: "fails for advertisements that are rejected", + network: primaryNetwork, + ra: &podNetworkRARejected, + node: testNode, + expectNoNetwork: true, + }, + { + name: "fails for advertisements that are old", + network: primaryNetwork, + ra: &podNetworkRAOutdated, + node: testNode, + expectNoNetwork: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableRouteAdvertisements = true + fakeClient := util.GetOVNClientset().GetOVNKubeControllerClientset() + wf, err := factory.NewOVNKubeControllerWatchFactory(fakeClient) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + tcm := &testControllerManager{ + controllers: map[string]NetworkController{}, + defaultNetwork: &testNetworkController{ + ReconcilableNetInfo: &util.DefaultNetInfo{}, + }, + } + nm := newNetworkController("", testZoneName, testNodeName, tcm, wf) + + namespace, name, err := cache.SplitMetaNamespaceKey(testNADName) + g.Expect(err).ToNot(gomega.HaveOccurred()) + nad, err := buildNAD(name, namespace, tt.network) + g.Expect(err).ToNot(gomega.HaveOccurred()) + nad.Annotations = map[string]string{ + types.OvnRouteAdvertisementsKey: "[\"" + tt.ra.Name + "\"]", + } + + _, err = fakeClient.KubeClient.CoreV1().Nodes().Create(context.Background(), &tt.node, v1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = fakeClient.RouteAdvertisementsClient.K8sV1().RouteAdvertisements().Create(context.Background(), tt.ra, v1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = fakeClient.NetworkAttchDefClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespace).Create(context.Background(), nad, v1.CreateOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + err = wf.Start() + g.Expect(err).ToNot(gomega.HaveOccurred()) + defer wf.Shutdown() + g.Expect(nm.Start()).To(gomega.Succeed()) + defer nm.Stop() + + netInfo, err := util.NewNetInfo(tt.network) + g.Expect(err).ToNot(gomega.HaveOccurred()) + mutableNetInfo := util.NewMutableNetInfo(netInfo) + mutableNetInfo.AddNADs(testNADName) + + nm.EnsureNetwork(mutableNetInfo) + + meetsExpectations := func(g gomega.Gomega) { + tcm.Lock() + defer tcm.Unlock() + var reconcilable ReconcilableNetworkController + switch tt.network.Name { + case types.DefaultNetworkName: + reconcilable = tcm.GetDefaultNetworkController() + default: + reconcilable = tcm.controllers[testNetworkKey(netInfo)] + } + + if tt.expectNoNetwork { + g.Expect(reconcilable).To(gomega.BeNil()) + return + } + g.Expect(reconcilable).ToNot(gomega.BeNil()) + + if tt.expected == nil { + tt.expected = map[string][]string{} + } + g.Expect(reconcilable.GetPodNetworkAdvertisedVRFs()).To(gomega.Equal(tt.expected)) + } + + g.Eventually(meetsExpectations).Should(gomega.Succeed()) + g.Consistently(meetsExpectations).Should(gomega.Succeed()) + }) + } +} diff --git a/go-controller/pkg/node/base_node_network_controller_dpu.go b/go-controller/pkg/node/base_node_network_controller_dpu.go index bb50446d07..ef97fd7a24 100644 --- a/go-controller/pkg/node/base_node_network_controller_dpu.go +++ b/go-controller/pkg/node/base_node_network_controller_dpu.go @@ -113,7 +113,7 @@ func (bnnc *BaseNodeNetworkController) watchPodsDPU() (*factory.Handler, error) // For default network, NAD name is DefaultNetworkName. nadToDPUCDMap := map[string]*util.DPUConnectionDetails{} if bnnc.IsSecondary() { - on, networkMap, err := util.GetPodNADToNetworkMapping(pod, bnnc.NetInfo) + on, networkMap, err := util.GetPodNADToNetworkMapping(pod, bnnc.GetNetInfo()) if err != nil || !on { if err != nil { // configuration error, no need to retry, do not return error diff --git a/go-controller/pkg/node/base_node_network_controller_dpu_test.go b/go-controller/pkg/node/base_node_network_controller_dpu_test.go index 2f86122feb..444883cca9 100644 --- a/go-controller/pkg/node/base_node_network_controller_dpu_test.go +++ b/go-controller/pkg/node/base_node_network_controller_dpu_test.go @@ -117,7 +117,7 @@ var _ = Describe("Node DPU tests", func() { apbExternalRouteClient := adminpolicybasedrouteclient.NewSimpleClientset() factoryMock = factorymocks.NodeWatchFactory{} cnnci := newCommonNodeNetworkControllerInfo(nil, &kubeMock, apbExternalRouteClient, &factoryMock, nil, "", routeManager) - dnnc = newDefaultNodeNetworkController(cnnci, nil, nil, routeManager, nil) + dnnc = newDefaultNodeNetworkController(cnnci, nil, nil, routeManager) podInformer = coreinformermocks.PodInformer{} podNamespaceLister = v1mocks.PodNamespaceLister{} diff --git a/go-controller/pkg/node/default_node_network_controller.go b/go-controller/pkg/node/default_node_network_controller.go index b5d2548800..ba20c9a45d 100644 --- a/go-controller/pkg/node/default_node_network_controller.go +++ b/go-controller/pkg/node/default_node_network_controller.go @@ -33,7 +33,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/informer" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/controllers/egressip" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/controllers/egressservice" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/linkmanager" @@ -65,7 +65,7 @@ type BaseNodeNetworkController struct { CommonNodeNetworkControllerInfo // network information - util.NetInfo + util.ReconcilableNetInfo // podNADToDPUCDMap tracks the NAD/DPU_ConnectionDetails mapping for all NADs that each pod requests. // Key is pod.UUID; value is nadToDPUCDMap (of map[string]*util.DPUConnectionDetails). Key of nadToDPUCDMap @@ -116,7 +116,7 @@ type DefaultNodeNetworkController struct { apbExternalRouteNodeController *apbroute.ExternalGatewayNodeController - nadController *nad.NetAttachDefinitionController + networkManager networkmanager.Interface cniServer *cni.Server @@ -126,19 +126,19 @@ type DefaultNodeNetworkController struct { } type preStartSetup struct { - mgmtPorts []managementPortEntry + mgmtPorts []*managementPortEntry mgmtPortConfig *managementPortConfig nodeAddress net.IP sbZone string } func newDefaultNodeNetworkController(cnnci *CommonNodeNetworkControllerInfo, stopChan chan struct{}, - wg *sync.WaitGroup, routeManager *routemanager.Controller, nadController *nad.NetAttachDefinitionController) *DefaultNodeNetworkController { + wg *sync.WaitGroup, routeManager *routemanager.Controller) *DefaultNodeNetworkController { c := &DefaultNodeNetworkController{ BaseNodeNetworkController: BaseNodeNetworkController{ CommonNodeNetworkControllerInfo: *cnnci, - NetInfo: &util.DefaultNetInfo{}, + ReconcilableNetInfo: &util.DefaultNetInfo{}, stopChan: stopChan, wg: wg, }, @@ -146,18 +146,18 @@ func newDefaultNodeNetworkController(cnnci *CommonNodeNetworkControllerInfo, sto } if util.IsNetworkSegmentationSupportEnabled() && !config.OVNKubernetesFeature.DisableUDNHostIsolation { c.udnHostIsolationManager = NewUDNHostIsolationManager(config.IPv4Mode, config.IPv6Mode, - cnnci.watchFactory.PodCoreInformer(), nadController) + cnnci.watchFactory.PodCoreInformer()) } c.linkManager = linkmanager.NewController(cnnci.name, config.IPv4Mode, config.IPv6Mode, c.updateGatewayMAC) return c } // NewDefaultNodeNetworkController creates a new network controller for node management of the default network -func NewDefaultNodeNetworkController(cnnci *CommonNodeNetworkControllerInfo, nadController *nad.NetAttachDefinitionController) (*DefaultNodeNetworkController, error) { +func NewDefaultNodeNetworkController(cnnci *CommonNodeNetworkControllerInfo, networkManager networkmanager.Interface) (*DefaultNodeNetworkController, error) { var err error stopChan := make(chan struct{}) wg := &sync.WaitGroup{} - nc := newDefaultNodeNetworkController(cnnci, stopChan, wg, cnnci.routeManager, nadController) + nc := newDefaultNodeNetworkController(cnnci, stopChan, wg, cnnci.routeManager) if len(config.Kubernetes.HealthzBindAddress) != 0 { klog.Infof("Enable node proxy healthz server on %s", config.Kubernetes.HealthzBindAddress) @@ -177,7 +177,7 @@ func NewDefaultNodeNetworkController(cnnci *CommonNodeNetworkControllerInfo, nad return nil, err } - nc.nadController = nadController + nc.networkManager = networkManager nc.initRetryFrameworkForNode() @@ -189,6 +189,46 @@ func (nc *DefaultNodeNetworkController) initRetryFrameworkForNode() { nc.retryEndpointSlices = nc.newRetryFrameworkNode(factory.EndpointSliceForStaleConntrackRemovalType) } +func (oc *DefaultNodeNetworkController) shouldReconcileNetworkChange(old, new util.NetInfo) bool { + wasPodNetworkAdvertisedAtNode := util.IsPodNetworkAdvertisedAtNode(old, oc.name) + isPodNetworkAdvertisedAtNode := util.IsPodNetworkAdvertisedAtNode(new, oc.name) + return wasPodNetworkAdvertisedAtNode != isPodNetworkAdvertisedAtNode +} + +func (oc *DefaultNodeNetworkController) Reconcile(netInfo util.NetInfo) error { + // inspect changes first + reconcilePodNetwork := oc.shouldReconcileNetworkChange(oc.ReconcilableNetInfo, netInfo) + + // reconcile subcontrollers + if reconcilePodNetwork { + isPodNetworkAdvertisedAtNode := util.IsPodNetworkAdvertisedAtNode(netInfo, oc.name) + if oc.Gateway != nil { + oc.Gateway.SetPodNetworkAdvertised(isPodNetworkAdvertisedAtNode) + err := oc.Gateway.Reconcile() + if err != nil { + return fmt.Errorf("failed to reconcile gateway: %v", err) + } + } + for _, mgmtPort := range oc.gatewaySetup.mgmtPorts { + mgmtPort.SetPodNetworkAdvertised(isPodNetworkAdvertisedAtNode) + mgmtPort.Reconcile() + } + } + + // Update network information. We can do this now because gateway and + // management port reconciliation done above does not rely on NetInfo + err := util.ReconcileNetInfo(oc.ReconcilableNetInfo, netInfo) + if err != nil { + return fmt.Errorf("failed to reconcile network %s: %v", oc.GetNetworkName(), err) + } + + return nil +} + +func (oc *DefaultNodeNetworkController) isPodNetworkAdvertisedAtNode() bool { + return util.IsPodNetworkAdvertisedAtNode(oc, oc.name) +} + func clearOVSFlowTargets() error { _, _, err := util.RunOVSVsctl( "--", @@ -302,24 +342,68 @@ func setOVSFlowTargets(node *kapi.Node) error { return nil } +// validateEncapIP returns false if there is an error or if the given IP is not known local IP address. +func validateEncapIP(encapIP string) (bool, error) { + links, err := netlink.LinkList() + if err != nil { + return false, fmt.Errorf("failed to get all the links on the node: %v", err) + } + for _, link := range links { + addrs, err := util.GetFilteredInterfaceAddrs(link, config.IPv4Mode, config.IPv6Mode) + if err != nil { + return false, err + } + for _, addr := range addrs { + if addr.IP.String() == encapIP { + return true, nil + } + } + } + return false, nil +} + func setupOVNNode(node *kapi.Node) error { var err error + nodePrimaryIP, err := util.GetNodePrimaryIP(node) + if err != nil { + return fmt.Errorf("failed to obtain local primary IP from node %q: %v", node.Name, err) + } + encapIP := config.Default.EncapIP if encapIP == "" { - encapIP, err = util.GetNodePrimaryIP(node) - if err != nil { - return fmt.Errorf("failed to obtain local IP from node %q: %v", node.Name, err) - } - config.Default.EncapIP = encapIP + config.Default.EffectiveEncapIP = nodePrimaryIP } else { // OVN allows `external_ids:ovn-encap-ip` to be a list of IPs separated by comma. + config.Default.EffectiveEncapIP = encapIP ovnEncapIps := strings.Split(encapIP, ",") for _, ovnEncapIp := range ovnEncapIps { if ip := net.ParseIP(strings.TrimSpace(ovnEncapIp)); ip == nil { return fmt.Errorf("invalid IP address %q in provided encap-ip setting %q", ovnEncapIp, encapIP) } } + // if there are more than one encap IPs, it must be configured explicitly. otherwise: + if len(ovnEncapIps) == 1 { + encapIP = ovnEncapIps[0] + if encapIP == nodePrimaryIP { + // the current encap IP is node primary IP, unset config.Default.EncapIP to indicate it is + // implicitly configured through the old external_ids:ovn-encap-ip value and needs to be updated + // if node primary IP changes. + config.Default.EncapIP = "" + } else { + // the encap IP may be incorrectly set or; + // previous implicitly set with the old primary node IP through the old external_ids:ovn-encap-ip value, + // that has changed when ovnkube-node is down. + // validate it to see if it is still a valid local IP address. + valid, err := validateEncapIP(encapIP) + if err != nil { + return fmt.Errorf("invalid encap IP %s: %v", encapIP, err) + } + if !valid { + return fmt.Errorf("invalid encap IP %s: does not exist", encapIP) + } + } + } } setExternalIdsCmd := []string{ @@ -327,7 +411,7 @@ func setupOVNNode(node *kapi.Node) error { "Open_vSwitch", ".", fmt.Sprintf("external_ids:ovn-encap-type=%s", config.Default.EncapType), - fmt.Sprintf("external_ids:ovn-encap-ip=%s", encapIP), + fmt.Sprintf("external_ids:ovn-encap-ip=%s", config.Default.EffectiveEncapIP), fmt.Sprintf("external_ids:ovn-remote-probe-interval=%d", config.Default.InactivityProbe), fmt.Sprintf("external_ids:ovn-openflow-probe-interval=%d", @@ -449,11 +533,6 @@ func isOVNControllerReady() (bool, error) { return true, nil } -type managementPortEntry struct { - port ManagementPort - config *managementPortConfig -} - // getEnvNameFromResourceName gets the device plugin env variable from the device plugin resource name. func getEnvNameFromResourceName(resource string) string { res1 := strings.ReplaceAll(resource, ".", "_") @@ -618,7 +697,7 @@ func getMgmtPortAndRepName(node *kapi.Node) (string, string, error) { } func createNodeManagementPorts(node *kapi.Node, nodeLister listers.NodeLister, nodeAnnotator kube.Annotator, kubeInterface kube.Interface, waiter *startupWaiter, - subnets []*net.IPNet, routeManager *routemanager.Controller) ([]managementPortEntry, *managementPortConfig, error) { + subnets []*net.IPNet, routeManager *routemanager.Controller, isRoutingAdvertised bool) ([]*managementPortEntry, *managementPortConfig, error) { netdevName, rep, err := getMgmtPortAndRepName(node) if err != nil { return nil, nil, err @@ -633,13 +712,14 @@ func createNodeManagementPorts(node *kapi.Node, nodeLister listers.NodeLister, n ports := NewManagementPorts(node.Name, subnets, netdevName, rep) var mgmtPortConfig *managementPortConfig - mgmtPorts := make([]managementPortEntry, 0) + mgmtPorts := make([]*managementPortEntry, 0) for _, port := range ports { - config, err := port.Create(routeManager, node, nodeLister, kubeInterface, waiter) + config, err := port.Create(isRoutingAdvertised, routeManager, node, nodeLister, kubeInterface, waiter) if err != nil { return nil, nil, err } - mgmtPorts = append(mgmtPorts, managementPortEntry{port: port, config: config}) + mgmtPorts = append(mgmtPorts, NewManagementPortEntry(port, config, routeManager)) + // Save this management port config for later usage. // Since only one OVS internal port / Representor config may exist it is fine just to overwrite it if _, ok := port.(*managementPortNetdev); !ok { @@ -839,7 +919,7 @@ func (nc *DefaultNodeNetworkController) PreStart(ctx context.Context) error { if !ok { return fmt.Errorf("cannot get kubeclient for starting CNI server") } - cniServer, err = cni.NewCNIServer(nc.watchFactory, kclient.KClient, nc.nadController) + cniServer, err = cni.NewCNIServer(nc.watchFactory, kclient.KClient, nc.networkManager) if err != nil { return err } @@ -850,8 +930,15 @@ func (nc *DefaultNodeNetworkController) PreStart(ctx context.Context) error { waiter := newStartupWaiter() // Setup management ports - mgmtPorts, mgmtPortConfig, err := createNodeManagementPorts(node, nc.watchFactory.NodeCoreInformer().Lister(), nodeAnnotator, - nc.Kube, waiter, subnets, nc.routeManager) + mgmtPorts, mgmtPortConfig, err := createNodeManagementPorts( + node, + nc.watchFactory.NodeCoreInformer().Lister(), + nodeAnnotator, + nc.Kube, + waiter, + subnets, + nc.routeManager, + nc.isPodNetworkAdvertisedAtNode()) if err != nil { return err } @@ -1142,7 +1229,7 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error { // start management ports health check for _, mgmtPort := range nc.gatewaySetup.mgmtPorts { - mgmtPort.port.CheckManagementPortHealth(nc.routeManager, mgmtPort.config, nc.stopChan) + mgmtPort.Start(nc.stopChan) if config.OVNKubernetesFeature.EnableEgressIP { // Start the health checking server used by egressip, if EgressIPNodeHealthCheckPort is specified if err := nc.startEgressIPHealthCheckingServer(mgmtPort); err != nil { @@ -1214,7 +1301,7 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error { if config.OVNKubernetesFeature.EnableEgressIP && !util.PlatformTypeIsEgressIPCloudProvider() { c, err := egressip.NewController(nc.Kube, nc.watchFactory.EgressIPInformer(), nc.watchFactory.NodeInformer(), - nc.watchFactory.NamespaceInformer(), nc.watchFactory.PodCoreInformer(), nc.nadController.GetActiveNetworkForNamespace, + nc.watchFactory.NamespaceInformer(), nc.watchFactory.PodCoreInformer(), nc.networkManager.GetActiveNetworkForNamespace, nc.routeManager, config.IPv4Mode, config.IPv6Mode, nc.name, nc.linkManager) if err != nil { return fmt.Errorf("failed to create egress IP controller: %v", err) @@ -1245,7 +1332,7 @@ func (nc *DefaultNodeNetworkController) Stop() { nc.wg.Wait() } -func (nc *DefaultNodeNetworkController) startEgressIPHealthCheckingServer(mgmtPortEntry managementPortEntry) error { +func (nc *DefaultNodeNetworkController) startEgressIPHealthCheckingServer(mgmtPortEntry *managementPortEntry) error { healthCheckPort := config.OVNKubernetesFeature.EgressIPNodeHealthCheckPort if healthCheckPort == 0 { klog.Infof("Egress IP health check server skipped: no port specified") @@ -1392,11 +1479,11 @@ func (nc *DefaultNodeNetworkController) WatchNamespaces() error { // enough, it will return an error func (nc *DefaultNodeNetworkController) validateVTEPInterfaceMTU() error { // OVN allows `external_ids:ovn-encap-ip` to be a list of IPs separated by comma - ovnEncapIps := strings.Split(config.Default.EncapIP, ",") + ovnEncapIps := strings.Split(config.Default.EffectiveEncapIP, ",") for _, ip := range ovnEncapIps { ovnEncapIP := net.ParseIP(strings.TrimSpace(ip)) if ovnEncapIP == nil { - return fmt.Errorf("invalid IP address %q in provided encap-ip setting %q", ovnEncapIP, config.Default.EncapIP) + return fmt.Errorf("invalid IP address %q in provided encap-ip setting %q", ovnEncapIP, config.Default.EffectiveEncapIP) } interfaceName, mtu, err := util.GetIFNameAndMTUForAddress(ovnEncapIP) if err != nil { diff --git a/go-controller/pkg/node/default_node_network_controller_test.go b/go-controller/pkg/node/default_node_network_controller_test.go index c1b83f7c86..39246fbac6 100644 --- a/go-controller/pkg/node/default_node_network_controller_test.go +++ b/go-controller/pkg/node/default_node_network_controller_test.go @@ -69,12 +69,12 @@ var _ = Describe("Node", func() { name: nodeName, Kube: kubeMock, }, - NetInfo: &util.DefaultNetInfo{}, + ReconcilableNetInfo: &util.DefaultNetInfo{}, }, } config.Default.MTU = configDefaultMTU - config.Default.EncapIP = "10.1.0.40" + config.Default.EffectiveEncapIP = "10.1.0.40" }) @@ -219,7 +219,7 @@ var _ = Describe("Node", func() { BeforeEach(func() { config.IPv4Mode = true config.IPv6Mode = false - config.Default.EncapIP = "10.1.0.40,10.2.0.50" + config.Default.EffectiveEncapIP = "10.1.0.40,10.2.0.50" netlinkOpsMock.On("LinkByIndex", 5).Return(netlinkLinkMock, nil) }) diff --git a/go-controller/pkg/node/gateway.go b/go-controller/pkg/node/gateway.go index 2b375d7690..bc4ab0e41d 100644 --- a/go-controller/pkg/node/gateway.go +++ b/go-controller/pkg/node/gateway.go @@ -32,6 +32,7 @@ type Gateway interface { Start() GetGatewayBridgeIface() string SetDefaultGatewayBridgeMAC(addr net.HardwareAddr) + SetPodNetworkAdvertised(bool) Reconcile() error } @@ -55,6 +56,9 @@ type gateway struct { watchFactory *factory.WatchFactory // used for retry stopChan <-chan struct{} wg *sync.WaitGroup + + isPodNetworkAdvertisedLock sync.Mutex + isPodNetworkAdvertised bool } func (g *gateway) AddService(svc *kapi.Service) error { @@ -464,6 +468,12 @@ func (g *gateway) SetDefaultGatewayBridgeMAC(macAddr net.HardwareAddr) { klog.Infof("Default gateway bridge MAC address updated to %s", macAddr) } +func (g *gateway) SetPodNetworkAdvertised(isPodNetworkAdvertised bool) { + g.isPodNetworkAdvertisedLock.Lock() + defer g.isPodNetworkAdvertisedLock.Unlock() + g.isPodNetworkAdvertised = isPodNetworkAdvertised +} + // Reconcile handles triggering updates to different components of a gateway, like OFM, Services func (g *gateway) Reconcile() error { klog.Info("Reconciling gateway with updates") @@ -475,7 +485,11 @@ func (g *gateway) Reconcile() error { if err != nil { return fmt.Errorf("failed to get subnets for node: %s for OpenFlow cache update; err: %w", node.Name, err) } - if err := g.openflowManager.updateBridgeFlowCache(subnets, g.nodeIPManager.ListAddresses()); err != nil { + if err := g.openflowManager.updateBridgeFlowCache(subnets, g.nodeIPManager.ListAddresses(), g.isPodNetworkAdvertised); err != nil { + return err + } + err = g.updateSNATRules() + if err != nil { return err } // Services create OpenFlow flows as well, need to update them all @@ -508,6 +522,25 @@ func (g *gateway) addAllServices() []error { return errs } +func (g *gateway) updateSNATRules() error { + g.isPodNetworkAdvertisedLock.Lock() + defer g.isPodNetworkAdvertisedLock.Unlock() + var ipnets []*net.IPNet + if g.nodeIPManager.mgmtPortConfig.ipv4 != nil { + ipnets = append(ipnets, g.nodeIPManager.mgmtPortConfig.ipv4.ifAddr) + } + if g.nodeIPManager.mgmtPortConfig.ipv6 != nil { + ipnets = append(ipnets, g.nodeIPManager.mgmtPortConfig.ipv6.ifAddr) + } + subnets := util.IPsToNetworkIPs(ipnets...) + + if g.isPodNetworkAdvertised || config.Gateway.Mode != config.GatewayModeLocal { + return delLocalGatewayPodSubnetNATRules(subnets...) + } + + return addLocalGatewayPodSubnetNATRules(subnets...) +} + type bridgeConfiguration struct { sync.Mutex nodeName string diff --git a/go-controller/pkg/node/gateway_init.go b/go-controller/pkg/node/gateway_init.go index 80cb08ecfa..6530d352c0 100644 --- a/go-controller/pkg/node/gateway_init.go +++ b/go-controller/pkg/node/gateway_init.go @@ -359,8 +359,22 @@ func (nc *DefaultNodeNetworkController) initGatewayPreStart(subnets []*net.IPNet switch config.Gateway.Mode { case config.GatewayModeLocal, config.GatewayModeShared: klog.Info("Preparing Gateway") - gw, err = newGateway(nc.name, subnets, gatewayNextHops, gatewayIntf, egressGWInterface, ifAddrs, nodeAnnotator, - managementPortConfig, nc.Kube, nc.watchFactory, nc.routeManager, nc.linkManager, nc.nadController, config.Gateway.Mode) + gw, err = newGateway( + nc.name, + subnets, + gatewayNextHops, + gatewayIntf, + egressGWInterface, + ifAddrs, + nodeAnnotator, + managementPortConfig, + nc.Kube, + nc.watchFactory, + nc.routeManager, + nc.linkManager, + nc.networkManager, + config.Gateway.Mode, + ) case config.GatewayModeDisabled: var chassisID string klog.Info("Gateway Mode is disabled") @@ -436,6 +450,7 @@ func (nc *DefaultNodeNetworkController) initGatewayMainStart(gw *gateway, waiter if portClaimWatcher != nil { gw.portClaimWatcher = portClaimWatcher } + gw.isPodNetworkAdvertised = nc.isPodNetworkAdvertisedAtNode() initGwFunc := func() error { return gw.Init(nc.stopChan, nc.wg) @@ -530,7 +545,7 @@ func (nc *DefaultNodeNetworkController) initGatewayDPUHost(kubeNodeIP net.IP) er if err := initSharedGatewayIPTables(); err != nil { return err } - gw.nodePortWatcherIptables = newNodePortWatcherIptables(nc.nadController) + gw.nodePortWatcherIptables = newNodePortWatcherIptables(nc.networkManager) gw.loadBalancerHealthChecker = newLoadBalancerHealthChecker(nc.name, nc.watchFactory) portClaimWatcher, err := newPortClaimWatcher(nc.recorder) if err != nil { diff --git a/go-controller/pkg/node/gateway_init_linux_test.go b/go-controller/pkg/node/gateway_init_linux_test.go index 8dc76d86aa..9a5efb64d0 100644 --- a/go-controller/pkg/node/gateway_init_linux_test.go +++ b/go-controller/pkg/node/gateway_init_linux_test.go @@ -32,12 +32,11 @@ import ( udnfakeclient "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned/fake" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" nodenft "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" linkMock "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/vishvananda/netlink" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" utilMock "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/mocks" @@ -55,6 +54,7 @@ const baseNFTRulesFmt = ` add table inet ovn-kubernetes add chain inet ovn-kubernetes mgmtport-snat { type nat hook postrouting priority 100 ; comment "OVN SNAT to Management Port" ; } add rule inet ovn-kubernetes mgmtport-snat oifname != %q return +add rule inet ovn-kubernetes mgmtport-snat meta nfproto ipv4 ip saddr 10.1.1.0 counter return add rule inet ovn-kubernetes mgmtport-snat meta l4proto . th dport @mgmtport-no-snat-nodeports counter return add rule inet ovn-kubernetes mgmtport-snat ip daddr . meta l4proto . th dport @mgmtport-no-snat-services-v4 counter return add rule inet ovn-kubernetes mgmtport-snat counter snat ip to 10.1.1.0 @@ -253,7 +253,6 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, ifName: nodeName, link: nil, routerMAC: nil, - nft: nft, ipv4: &fakeMgmtPortIPFamilyConfig, ipv6: nil, } @@ -289,16 +288,6 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, err = nodeAnnotator.Run() Expect(err).NotTo(HaveOccurred()) rm := routemanager.NewController() - var nadController *networkAttachDefController.NetAttachDefinitionController - if util.IsNetworkSegmentationSupportEnabled() { - testNCM := &nad.FakeNetworkControllerManager{} - nadController, err = networkAttachDefController.NewNetAttachDefinitionController("test", testNCM, wf, nil) - Expect(err).NotTo(HaveOccurred()) - err = nadController.Start() - Expect(err).NotTo(HaveOccurred()) - defer nadController.Stop() - } - Expect(err).NotTo(HaveOccurred()) wg.Add(1) go testNS.Do(func(netNS ns.NetNS) error { defer GinkgoRecover() @@ -345,8 +334,22 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, gatewayNextHops, gatewayIntf, err := getGatewayNextHops() Expect(err).NotTo(HaveOccurred()) ifAddrs := ovntest.MustParseIPNets(eth0CIDR) - sharedGw, err := newGateway(nodeName, ovntest.MustParseIPNets(nodeSubnet), gatewayNextHops, gatewayIntf, "", ifAddrs, nodeAnnotator, - &fakeMgmtPortConfig, k, wf, rm, nil, nadController, config.GatewayModeShared) + sharedGw, err := newGateway( + nodeName, + ovntest.MustParseIPNets(nodeSubnet), + gatewayNextHops, + gatewayIntf, + "", + ifAddrs, + nodeAnnotator, + &fakeMgmtPortConfig, + k, + wf, + rm, + nil, + networkmanager.Default().Interface(), + config.GatewayModeShared, + ) Expect(err).NotTo(HaveOccurred()) err = sharedGw.initFunc() Expect(err).NotTo(HaveOccurred()) @@ -695,12 +698,11 @@ func shareGatewayInterfaceDPUTest(app *cli.App, testNS ns.NetNS, gwIP: nodeNet.IP, } - nft := nodenft.SetFakeNFTablesHelper() + _ = nodenft.SetFakeNFTablesHelper() fakeMgmtPortConfig := managementPortConfig{ ifName: nodeName, link: nil, routerMAC: nil, - nft: nft, ipv4: &fakeMgmtPortIPFamilyConfig, ipv6: nil, } @@ -733,15 +735,6 @@ func shareGatewayInterfaceDPUTest(app *cli.App, testNS ns.NetNS, runtime.LockOSThread() defer runtime.UnlockOSThread() rm := routemanager.NewController() - var nadController *networkAttachDefController.NetAttachDefinitionController - if util.IsNetworkSegmentationSupportEnabled() { - testNCM := &nad.FakeNetworkControllerManager{} - nadController, err = networkAttachDefController.NewNetAttachDefinitionController("test", testNCM, wf, nil) - Expect(err).NotTo(HaveOccurred()) - err = nadController.Start() - Expect(err).NotTo(HaveOccurred()) - defer nadController.Stop() - } wg.Add(1) go testNS.Do(func(netNS ns.NetNS) error { defer GinkgoRecover() @@ -757,8 +750,22 @@ func shareGatewayInterfaceDPUTest(app *cli.App, testNS ns.NetNS, gatewayNextHops, gatewayIntf, err := getGatewayNextHops() Expect(err).NotTo(HaveOccurred()) - sharedGw, err := newGateway(nodeName, ovntest.MustParseIPNets(nodeSubnet), gatewayNextHops, - gatewayIntf, "", ifAddrs, nodeAnnotator, &fakeMgmtPortConfig, k, wf, rm, nil, nadController, config.GatewayModeShared) + sharedGw, err := newGateway( + nodeName, + ovntest.MustParseIPNets(nodeSubnet), + gatewayNextHops, + gatewayIntf, + "", + ifAddrs, + nodeAnnotator, + &fakeMgmtPortConfig, + k, + wf, + rm, + nil, + networkmanager.Default().Interface(), + config.GatewayModeShared, + ) Expect(err).NotTo(HaveOccurred()) err = sharedGw.initFunc() Expect(err).NotTo(HaveOccurred()) @@ -819,7 +826,7 @@ func shareGatewayInterfaceDPUTest(app *cli.App, testNS ns.NetNS, Expect(err).NotTo(HaveOccurred()) } -func shareGatewayInterfaceDPUHostTest(app *cli.App, testNS ns.NetNS, uplinkName, hostIP, gwIP string) { +func shareGatewayInterfaceDPUHostTest(app *cli.App, testNS ns.NetNS, uplinkName, hostIP string) { const ( clusterCIDR string = "10.1.0.0/16" svcCIDR string = "172.16.1.0/24" @@ -866,7 +873,7 @@ func shareGatewayInterfaceDPUHostTest(app *cli.App, testNS ns.NetNS, uplinkName, ipnet.IP = ip routeManager := routemanager.NewController() cnnci := NewCommonNodeNetworkControllerInfo(kubeFakeClient, fakeClient.AdminPolicyRouteClient, wf, nil, nodeName, routeManager) - nc := newDefaultNodeNetworkController(cnnci, stop, wg, routeManager, nil) + nc := newDefaultNodeNetworkController(cnnci, stop, wg, routeManager) // must run route manager manually which is usually started with nc.Start() wg.Add(1) go testNS.Do(func(netNS ns.NetNS) error { @@ -1149,7 +1156,6 @@ OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0` ifName: types.K8sMgmtIntfName, link: nil, routerMAC: nil, - nft: nft, ipv4: &fakeMgmtPortIPFamilyConfig, ipv6: nil, } @@ -1197,15 +1203,6 @@ OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0` ip, ipNet, _ := net.ParseCIDR(eth0CIDR) ipNet.IP = ip rm := routemanager.NewController() - var nadController *networkAttachDefController.NetAttachDefinitionController - if util.IsNetworkSegmentationSupportEnabled() { - testNCM := &nad.FakeNetworkControllerManager{} - nadController, err = networkAttachDefController.NewNetAttachDefinitionController("test", testNCM, wf, nil) - Expect(err).NotTo(HaveOccurred()) - err = nadController.Start() - Expect(err).NotTo(HaveOccurred()) - defer nadController.Stop() - } go testNS.Do(func(netNS ns.NetNS) error { defer GinkgoRecover() rm.Run(stop, 10*time.Second) @@ -1218,8 +1215,22 @@ OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0` gatewayNextHops, gatewayIntf, err := getGatewayNextHops() Expect(err).NotTo(HaveOccurred()) ifAddrs := ovntest.MustParseIPNets(eth0CIDR) - localGw, err := newGateway(nodeName, ovntest.MustParseIPNets(nodeSubnet), gatewayNextHops, gatewayIntf, "", ifAddrs, - nodeAnnotator, &fakeMgmtPortConfig, k, wf, rm, nil, nadController, config.GatewayModeLocal) + localGw, err := newGateway( + nodeName, + ovntest.MustParseIPNets(nodeSubnet), + gatewayNextHops, + gatewayIntf, + "", + ifAddrs, + nodeAnnotator, + &fakeMgmtPortConfig, + k, + wf, + rm, + nil, + networkmanager.Default().Interface(), + config.GatewayModeLocal, + ) Expect(err).NotTo(HaveOccurred()) err = localGw.initFunc() Expect(err).NotTo(HaveOccurred()) @@ -1598,7 +1609,7 @@ var _ = Describe("Gateway Operations DPU", func() { }) ovntest.OnSupportedPlatformsIt("sets up a shared interface gateway DPU host", func() { - shareGatewayInterfaceDPUHostTest(app, testNS, uplinkName, hostIP, gwIP) + shareGatewayInterfaceDPUHostTest(app, testNS, uplinkName, hostIP) }) }) }) diff --git a/go-controller/pkg/node/gateway_iptables.go b/go-controller/pkg/node/gateway_iptables.go index 1fc96ca8a6..0f9728e971 100644 --- a/go-controller/pkg/node/gateway_iptables.go +++ b/go-controller/pkg/node/gateway_iptables.go @@ -413,23 +413,9 @@ func getLocalGatewayFilterRules(ifname string, cidr *net.IPNet) []nodeipt.Rule { } } -func getLocalGatewayNATRules(cidr *net.IPNet) []nodeipt.Rule { - // Allow packets to/from the gateway interface in case defaults deny +func getLocalGatewayPodSubnetNATRules(cidr *net.IPNet) []nodeipt.Rule { protocol := getIPTablesProtocol(cidr.IP.String()) - masqueradeIP := config.Gateway.MasqueradeIPs.V4OVNMasqueradeIP - if protocol == iptables.ProtocolIPv6 { - masqueradeIP = config.Gateway.MasqueradeIPs.V6OVNMasqueradeIP - } - rules := []nodeipt.Rule{ - { - Table: "nat", - Chain: "POSTROUTING", - Args: []string{ - "-s", masqueradeIP.String(), - "-j", "MASQUERADE", - }, - Protocol: protocol, - }, + return []nodeipt.Rule{ { Table: "nat", Chain: "POSTROUTING", @@ -440,12 +426,6 @@ func getLocalGatewayNATRules(cidr *net.IPNet) []nodeipt.Rule { Protocol: protocol, }, } - // FIXME(tssurya): If the feature is disabled we should be removing - // these rules - if util.IsNetworkSegmentationSupportEnabled() { - rules = append(rules, getUDNMasqueradeRules(protocol)...) - } - return rules } // getUDNMasqueradeRules is only called for local-gateway-mode @@ -514,6 +494,37 @@ func getUDNMasqueradeRules(protocol iptables.Protocol) []nodeipt.Rule { return rules } +func getLocalGatewayNATRules(cidr *net.IPNet) []nodeipt.Rule { + // Allow packets to/from the gateway interface in case defaults deny + protocol := getIPTablesProtocol(cidr.IP.String()) + masqueradeIP := config.Gateway.MasqueradeIPs.V4OVNMasqueradeIP + if protocol == iptables.ProtocolIPv6 { + masqueradeIP = config.Gateway.MasqueradeIPs.V6OVNMasqueradeIP + } + rules := append( + []nodeipt.Rule{ + { + Table: "nat", + Chain: "POSTROUTING", + Args: []string{ + "-s", masqueradeIP.String(), + "-j", "MASQUERADE", + }, + Protocol: protocol, + }, + }, + getLocalGatewayPodSubnetNATRules(cidr)..., + ) + + // FIXME(tssurya): If the feature is disabled we should be removing + // these rules + if util.IsNetworkSegmentationSupportEnabled() { + rules = append(rules, getUDNMasqueradeRules(protocol)...) + } + + return rules +} + // initLocalGatewayNATRules sets up iptables rules for interfaces func initLocalGatewayNATRules(ifname string, cidr *net.IPNet) error { // Insert the filter table rules because they need to be evaluated BEFORE the DROP rules @@ -528,6 +539,22 @@ func initLocalGatewayNATRules(ifname string, cidr *net.IPNet) error { return appendIptRules(getLocalGatewayNATRules(cidr)) } +func addLocalGatewayPodSubnetNATRules(cidrs ...*net.IPNet) error { + var rules []nodeipt.Rule + for _, cidr := range cidrs { + rules = append(rules, getLocalGatewayPodSubnetNATRules(cidr)...) + } + return appendIptRules(rules) +} + +func delLocalGatewayPodSubnetNATRules(cidrs ...*net.IPNet) error { + var rules []nodeipt.Rule + for _, cidr := range cidrs { + rules = append(rules, getLocalGatewayPodSubnetNATRules(cidr)...) + } + return deleteIptRules(rules) +} + func addChaintoTable(ipt util.IPTablesHelper, tableName, chain string) { if err := ipt.NewChain(tableName, chain); err != nil { klog.V(5).Infof("Chain: \"%s\" in table: \"%s\" already exists, skipping creation: %v", chain, tableName, err) diff --git a/go-controller/pkg/node/gateway_localnet_linux_test.go b/go-controller/pkg/node/gateway_localnet_linux_test.go index 4ec1cd42d0..b410dd6051 100644 --- a/go-controller/pkg/node/gateway_localnet_linux_test.go +++ b/go-controller/pkg/node/gateway_localnet_linux_test.go @@ -14,6 +14,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" nodenft "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" @@ -73,6 +74,7 @@ func initFakeNodePortWatcher(iptV4, iptV6 util.IPTablesHelper) *nodePortWatcher }, }, }, + networkManager: networkmanager.Default().Interface(), } return &fNPW } @@ -299,7 +301,6 @@ var _ = Describe("Node Operations", func() { ifName: fakeNodeName, link: nil, routerMAC: nil, - nft: nft, ipv4: &fakeMgmtPortIPFamilyConfig, ipv6: nil, } diff --git a/go-controller/pkg/node/gateway_shared_intf.go b/go-controller/pkg/node/gateway_shared_intf.go index 289e1ae523..4d8517f3af 100644 --- a/go-controller/pkg/node/gateway_shared_intf.go +++ b/go-controller/pkg/node/gateway_shared_intf.go @@ -16,7 +16,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/controllers/egressservice" nodeipt "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iptables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/linkmanager" @@ -180,12 +180,12 @@ func configureUDNServicesNFTables() error { // nodePortWatcherIptables manages iptables rules for shared gateway // to ensure that services using NodePorts are accessible. type nodePortWatcherIptables struct { - nadController *nad.NetAttachDefinitionController + networkManager networkmanager.Interface } -func newNodePortWatcherIptables(nadController *nad.NetAttachDefinitionController) *nodePortWatcherIptables { +func newNodePortWatcherIptables(networkManager networkmanager.Interface) *nodePortWatcherIptables { return &nodePortWatcherIptables{ - nadController: nadController, + networkManager: networkManager, } } @@ -203,7 +203,7 @@ type nodePortWatcher struct { serviceInfoLock sync.Mutex ofm *openflowManager nodeIPManager *addressManager - nadController *nad.NetAttachDefinitionController + networkManager networkmanager.Interface watchFactory factory.NodeWatchFactory } @@ -818,7 +818,7 @@ func (npw *nodePortWatcher) AddService(service *kapi.Service) error { klog.V(5).Infof("Adding service %s in namespace %s", service.Name, service.Namespace) - netInfo, err := npw.nadController.GetActiveNetworkForNamespace(service.Namespace) + netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(service.Namespace) if err != nil { return fmt.Errorf("error getting active network for service %s in namespace %s: %w", service.Name, service.Namespace, err) } @@ -845,7 +845,7 @@ func (npw *nodePortWatcher) AddService(service *kapi.Service) error { service.Name, service.Namespace) if err := addServiceRules(service, netInfo, sets.List(localEndpoints), hasLocalHostNetworkEp, npw); err != nil { npw.getAndDeleteServiceInfo(name) - return fmt.Errorf("AddService failed for nodePortWatcher: %w, trying delete: %w", err, delServiceRules(service, sets.List(localEndpoints), npw)) + return fmt.Errorf("AddService failed for nodePortWatcher: %w, trying delete: %v", err, delServiceRules(service, sets.List(localEndpoints), npw)) } } else { // Need to update flows here in case an attribute of the gateway has changed, such as MAC address @@ -890,7 +890,7 @@ func (npw *nodePortWatcher) UpdateService(old, new *kapi.Service) error { if util.ServiceTypeHasClusterIP(new) && util.IsClusterIPSet(new) { klog.V(5).Infof("Adding new service rules for: %v", new) - netInfo, err := npw.nadController.GetActiveNetworkForNamespace(new.Namespace) + netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(new.Namespace) if err != nil { return fmt.Errorf("error getting active network for service %s in namespace %s: %w", new.Name, new.Namespace, err) } @@ -997,7 +997,7 @@ func (npw *nodePortWatcher) SyncServices(services []interface{}) error { continue } - netInfo, err := npw.nadController.GetActiveNetworkForNamespace(service.Namespace) + netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(service.Namespace) if err != nil { errors = append(errors, err) continue @@ -1074,7 +1074,7 @@ func (npw *nodePortWatcher) AddEndpointSlice(epSlice *discovery.EndpointSlice) e var errors []error var svc *kapi.Service - netInfo, err := npw.nadController.GetActiveNetworkForNamespace(epSlice.Namespace) + netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(epSlice.Namespace) if err != nil { return fmt.Errorf("error getting active network for endpointslice %s in namespace %s: %w", epSlice.Name, epSlice.Namespace, err) } @@ -1118,10 +1118,14 @@ func (npw *nodePortWatcher) AddEndpointSlice(epSlice *discovery.EndpointSlice) e // Here we make sure the correct rules are programmed whenever an AddEndpointSlice event is // received, only alter flows if we need to, i.e if cache wasn't set or if it was and // hasLocalHostNetworkEp or localEndpoints state (for LB svc where NPs=0) changed, to prevent flow churn - out, exists := npw.getAndSetServiceInfo(*svcNamespacedName, svc, hasLocalHostNetworkEp, localEndpoints) + out, exists := npw.getServiceInfo(*svcNamespacedName) if !exists { klog.V(5).Infof("Endpointslice %s ADD event in namespace %s is creating rules", epSlice.Name, epSlice.Namespace) - return addServiceRules(svc, netInfo, sets.List(localEndpoints), hasLocalHostNetworkEp, npw) + if err = addServiceRules(svc, netInfo, sets.List(localEndpoints), hasLocalHostNetworkEp, npw); err != nil { + return err + } + npw.addOrSetServiceInfo(*svcNamespacedName, svc, hasLocalHostNetworkEp, localEndpoints) + return nil } if out.hasLocalHostNetworkEp != hasLocalHostNetworkEp || @@ -1132,6 +1136,8 @@ func (npw *nodePortWatcher) AddEndpointSlice(epSlice *discovery.EndpointSlice) e } if err = addServiceRules(svc, netInfo, sets.List(localEndpoints), hasLocalHostNetworkEp, npw); err != nil { errors = append(errors, err) + } else { + npw.updateServiceInfo(*svcNamespacedName, svc, &hasLocalHostNetworkEp, localEndpoints) } return utilerrors.Join(errors...) } @@ -1144,7 +1150,7 @@ func (npw *nodePortWatcher) DeleteEndpointSlice(epSlice *discovery.EndpointSlice var errors []error var hasLocalHostNetworkEp = false - netInfo, err := npw.nadController.GetActiveNetworkForNamespace(epSlice.Namespace) + netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(epSlice.Namespace) if err != nil { return fmt.Errorf("error getting active network for endpointslice %s in namespace %s: %w", epSlice.Name, epSlice.Namespace, err) } @@ -1177,7 +1183,7 @@ func (npw *nodePortWatcher) DeleteEndpointSlice(epSlice *discovery.EndpointSlice } localEndpoints := npw.GetLocalEligibleEndpointAddresses(epSlices, svc) if svcConfig, exists := npw.updateServiceInfo(*namespacedName, nil, &hasLocalHostNetworkEp, localEndpoints); exists { - netInfo, err := npw.nadController.GetActiveNetworkForNamespace(namespacedName.Namespace) + netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(namespacedName.Namespace) if err != nil { return fmt.Errorf("error getting active network for service %s in namespace %s: %w", svc.Name, svc.Namespace, err) } @@ -1210,7 +1216,7 @@ func (npw *nodePortWatcher) UpdateEndpointSlice(oldEpSlice, newEpSlice *discover var err error var errors []error - netInfo, err := npw.nadController.GetActiveNetworkForNamespace(newEpSlice.Namespace) + netInfo, err := npw.networkManager.GetActiveNetworkForNamespace(newEpSlice.Namespace) if err != nil { return fmt.Errorf("error getting active network for endpointslice %s in namespace %s: %w", newEpSlice.Name, newEpSlice.Namespace, err) } @@ -1298,7 +1304,7 @@ func (npwipt *nodePortWatcherIptables) AddService(service *kapi.Service) error { return nil } - netInfo, err := npwipt.nadController.GetActiveNetworkForNamespace(service.Namespace) + netInfo, err := npwipt.networkManager.GetActiveNetworkForNamespace(service.Namespace) if err != nil { return fmt.Errorf("error getting active network for service %s in namespace %s: %w", service.Name, service.Namespace, err) } @@ -1326,7 +1332,7 @@ func (npwipt *nodePortWatcherIptables) UpdateService(old, new *kapi.Service) err } if util.ServiceTypeHasClusterIP(new) && util.IsClusterIPSet(new) { - netInfo, err := npwipt.nadController.GetActiveNetworkForNamespace(new.Namespace) + netInfo, err := npwipt.networkManager.GetActiveNetworkForNamespace(new.Namespace) if err != nil { return fmt.Errorf("error getting active network for service %s in namespace %s: %w", new.Name, new.Namespace, err) } @@ -1738,7 +1744,7 @@ func flowsForDefaultBridge(bridge *bridgeConfiguration, extraIPs []net.IP) ([]st return dftFlows, nil } -func commonFlows(subnets []*net.IPNet, bridge *bridgeConfiguration) ([]string, error) { +func commonFlows(subnets []*net.IPNet, bridge *bridgeConfiguration, isPodNetworkAdvertised bool) ([]string, error) { // CAUTION: when adding new flows where the in_port is ofPortPatch and the out_port is ofPortPhys, ensure // that dl_src is included in match criteria! ofPortPhys := bridge.ofPortPhys @@ -1959,49 +1965,57 @@ func commonFlows(subnets []*net.IPNet, bridge *bridgeConfiguration) ([]string, e if config.OVNKubernetesFeature.EnableEgressIP { for _, clusterEntry := range config.Default.ClusterSubnets { cidr := clusterEntry.CIDR - ipPrefix := "ip" - if utilnet.IsIPv6CIDR(cidr) { - ipPrefix = "ipv6" - } + ipv := getIPv(cidr) // table 0, drop packets coming from pods headed externally that were not SNATed. dftFlows = append(dftFlows, fmt.Sprintf("cookie=%s, priority=104, in_port=%s, %s, %s_src=%s, actions=drop", - defaultOpenFlowCookie, defaultNetConfig.ofPortPatch, ipPrefix, ipPrefix, cidr)) + defaultOpenFlowCookie, defaultNetConfig.ofPortPatch, ipv, ipv, cidr)) } for _, subnet := range subnets { - ipPrefix := "ip" - if utilnet.IsIPv6CIDR(subnet) { - ipPrefix = "ipv6" - } + ipv := getIPv(subnet) if ofPortPhys != "" { // table 0, commit connections from local pods. // ICNIv2 requires that local pod traffic can leave the node without SNAT. dftFlows = append(dftFlows, fmt.Sprintf("cookie=%s, priority=109, in_port=%s, dl_src=%s, %s, %s_src=%s"+ "actions=ct(commit, zone=%d, exec(set_field:%s->ct_mark)), output:%s", - defaultOpenFlowCookie, defaultNetConfig.ofPortPatch, bridgeMacAddress, ipPrefix, ipPrefix, subnet, + defaultOpenFlowCookie, defaultNetConfig.ofPortPatch, bridgeMacAddress, ipv, ipv, subnet, config.Default.ConntrackZone, ctMarkOVN, ofPortPhys)) } } } if ofPortPhys != "" { - actions := fmt.Sprintf("output:%s", defaultNetConfig.ofPortPatch) - - if config.Gateway.DisableSNATMultipleGWs { + if config.Gateway.DisableSNATMultipleGWs || isPodNetworkAdvertised { // table 1, traffic to pod subnet go directly to OVN + output := defaultNetConfig.ofPortPatch + if isPodNetworkAdvertised && config.Gateway.Mode == config.GatewayModeLocal { + // except if advertised through BGP, go to kernel + // TODO: MEG enabled pods should still go through the patch port + // but holding this until + // https://issues.redhat.com/browse/FDP-646 is fixed, for now we + // are assuming MEG & BGP are not used together + output = ovsLocalPort + } for _, clusterEntry := range config.Default.ClusterSubnets { cidr := clusterEntry.CIDR - var ipPrefix string - if utilnet.IsIPv6CIDR(cidr) { - ipPrefix = "ipv6" - } else { - ipPrefix = "ip" - } + ipv := getIPv(cidr) dftFlows = append(dftFlows, fmt.Sprintf("cookie=%s, priority=15, table=1, %s, %s_dst=%s, "+ - "actions=%s", - defaultOpenFlowCookie, ipPrefix, ipPrefix, cidr, actions)) + "actions=output:%s", + defaultOpenFlowCookie, ipv, ipv, cidr, output)) + } + if output == defaultNetConfig.ofPortPatch { + // except node management traffic + for _, subnet := range subnets { + mgmtIP := util.GetNodeManagementIfAddr(subnet) + ipv := getIPv(mgmtIP) + dftFlows = append(dftFlows, + fmt.Sprintf("cookie=%s, priority=16, table=1, %s, %s_dst=%s, "+ + "actions=output:%s", + defaultOpenFlowCookie, ipv, ipv, mgmtIP.IP, ovsLocalPort), + ) + } } } @@ -2043,6 +2057,16 @@ func commonFlows(subnets []*net.IPNet, bridge *bridgeConfiguration) ([]string, e dftFlows = append(dftFlows, fmt.Sprintf("cookie=%s, priority=10, table=11, reg0=0x1, "+ "actions=output:%s", defaultOpenFlowCookie, ofPortHost)) + + // Send UDN destined traffic to right patch port + for _, netConfig := range bridge.patchedNetConfigs() { + if netConfig.masqCTMark != ctMarkOVN { + dftFlows = append(dftFlows, + fmt.Sprintf("cookie=%s, priority=5, table=11, ct_mark=%s, "+ + "actions=output:%s", defaultOpenFlowCookie, netConfig.masqCTMark, netConfig.ofPortPatch)) + } + } + dftFlows = append(dftFlows, fmt.Sprintf("cookie=%s, priority=1, table=11, "+ "actions=output:%s", defaultOpenFlowCookie, defaultNetConfig.ofPortPatch)) @@ -2152,10 +2176,21 @@ func initSvcViaMgmPortRoutingRules(hostSubnets []*net.IPNet) error { return nil } -func newGateway(nodeName string, subnets []*net.IPNet, gwNextHops []net.IP, gwIntf, egressGWIntf string, - gwIPs []*net.IPNet, nodeAnnotator kube.Annotator, cfg *managementPortConfig, kube kube.Interface, - watchFactory factory.NodeWatchFactory, routeManager *routemanager.Controller, linkManager *linkmanager.Controller, - nadController *nad.NetAttachDefinitionController, gatewayMode config.GatewayMode) (*gateway, error) { +func newGateway( + nodeName string, + subnets []*net.IPNet, + gwNextHops []net.IP, + gwIntf, egressGWIntf string, + gwIPs []*net.IPNet, + nodeAnnotator kube.Annotator, + cfg *managementPortConfig, + kube kube.Interface, + watchFactory factory.NodeWatchFactory, + routeManager *routemanager.Controller, + linkManager *linkmanager.Controller, + networkManager networkmanager.Interface, + gatewayMode config.GatewayMode, +) (*gateway, error) { klog.Info("Creating new gateway") gw := &gateway{} @@ -2265,7 +2300,7 @@ func newGateway(nodeName string, subnets []*net.IPNet, gwNextHops []net.IP, gwIn // resync flows on IP change gw.nodeIPManager.OnChanged = func() { klog.V(5).Info("Node addresses changed, re-syncing bridge flows") - if err := gw.openflowManager.updateBridgeFlowCache(subnets, gw.nodeIPManager.ListAddresses()); err != nil { + if err := gw.openflowManager.updateBridgeFlowCache(subnets, gw.nodeIPManager.ListAddresses(), gw.isPodNetworkAdvertised); err != nil { // very unlikely - somehow node has lost its IP address klog.Errorf("Failed to re-generate gateway flows after address change: %v", err) } @@ -2289,7 +2324,7 @@ func newGateway(nodeName string, subnets []*net.IPNet, gwNextHops []net.IP, gwIn } } klog.Info("Creating Gateway Node Port Watcher") - gw.nodePortWatcher, err = newNodePortWatcher(gwBridge, gw.openflowManager, gw.nodeIPManager, watchFactory, nadController) + gw.nodePortWatcher, err = newNodePortWatcher(gwBridge, gw.openflowManager, gw.nodeIPManager, watchFactory, networkManager) if err != nil { return err } @@ -2309,9 +2344,13 @@ func newGateway(nodeName string, subnets []*net.IPNet, gwNextHops []net.IP, gwIn return gw, nil } -func newNodePortWatcher(gwBridge *bridgeConfiguration, ofm *openflowManager, - nodeIPManager *addressManager, watchFactory factory.NodeWatchFactory, - nadController *nad.NetAttachDefinitionController) (*nodePortWatcher, error) { +func newNodePortWatcher( + gwBridge *bridgeConfiguration, + ofm *openflowManager, + nodeIPManager *addressManager, + watchFactory factory.NodeWatchFactory, + networkManager networkmanager.Interface, +) (*nodePortWatcher, error) { // Get ofport of physical interface ofportPhys, stderr, err := util.GetOVSOfPort("--if-exists", "get", @@ -2369,16 +2408,16 @@ func newNodePortWatcher(gwBridge *bridgeConfiguration, ofm *openflowManager, gatewayIPv4, gatewayIPv6 := getGatewayFamilyAddrs(gwBridge.ips) npw := &nodePortWatcher{ - dpuMode: dpuMode, - gatewayIPv4: gatewayIPv4, - gatewayIPv6: gatewayIPv6, - ofportPhys: ofportPhys, - gwBridge: gwBridge.bridgeName, - serviceInfo: make(map[ktypes.NamespacedName]*serviceConfig), - nodeIPManager: nodeIPManager, - ofm: ofm, - watchFactory: watchFactory, - nadController: nadController, + dpuMode: dpuMode, + gatewayIPv4: gatewayIPv4, + gatewayIPv6: gatewayIPv6, + ofportPhys: ofportPhys, + gwBridge: gwBridge.bridgeName, + serviceInfo: make(map[ktypes.NamespacedName]*serviceConfig), + nodeIPManager: nodeIPManager, + ofm: ofm, + watchFactory: watchFactory, + networkManager: networkManager, } return npw, nil } @@ -2796,3 +2835,11 @@ func deleteMasqueradeResources(link netlink.Link, staleMasqueradeIPs *config.Mas return utilerrors.Join(aggregatedErrors...) } + +func getIPv(ipnet *net.IPNet) string { + prefix := "ip" + if utilnet.IsIPv6CIDR(ipnet) { + prefix = "ipv6" + } + return prefix +} diff --git a/go-controller/pkg/node/gateway_udn_test.go b/go-controller/pkg/node/gateway_udn_test.go index 6ccfa01409..ea4615cb0e 100644 --- a/go-controller/pkg/node/gateway_udn_test.go +++ b/go-controller/pkg/node/gateway_udn_test.go @@ -12,7 +12,6 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" . "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" "github.com/vishvananda/netlink" @@ -29,7 +28,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" factoryMocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory/mocks" kubemocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube/mocks" - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iprulemanager" nodenft "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager" @@ -37,7 +36,6 @@ import ( ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" coreinformermocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/client-go/informers/core/v1" v1mocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/client-go/listers/core/v1" - fakenad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -259,7 +257,7 @@ func checkDefaultSvcIsolationOVSFlows(flows []string, defaultConfig *bridgeUDNCo Expect(nTable2Flows).To(Equal(1)) } -func checkUDNSvcIsolationOVSFlows(flows []string, netConfig *bridgeUDNConfiguration, netName, ofPortHost, bridgeMAC string, svcCIDR *net.IPNet, expectedNFlows int) { +func checkUDNSvcIsolationOVSFlows(flows []string, netConfig *bridgeUDNConfiguration, netName, bridgeMAC string, svcCIDR *net.IPNet, expectedNFlows int) { By(fmt.Sprintf("Checking UDN %s service isolation flows for %s; expected %d flows", netName, svcCIDR.String(), expectedNFlows)) @@ -533,8 +531,8 @@ var _ = Describe("UserDefinedNetworkGateway", func() { }, }, Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{ - {corev1.NodeInternalIP, strings.Split(v4NodeIP, "/")[0]}, - {corev1.NodeInternalIP, strings.Split(v6NodeIP, "/")[0]}}, + {Type: corev1.NodeInternalIP, Address: strings.Split(v4NodeIP, "/")[0]}, + {Type: corev1.NodeInternalIP, Address: strings.Split(v6NodeIP, "/")[0]}}, }, } nad := ovntest.GenerateNAD(netName, "rednad", "greenamespace", @@ -574,7 +572,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { Expect(err).NotTo(HaveOccurred()) _, _ = util.SetFakeIPTablesHelpers() - nft := nodenft.SetFakeNFTablesHelper() + _ = nodenft.SetFakeNFTablesHelper() // Make a fake MgmtPortConfig with only the fields we care about fakeMgmtPortV4IPFamilyConfig := managementPortIPFamilyConfig{ @@ -590,7 +588,6 @@ var _ = Describe("UserDefinedNetworkGateway", func() { ifName: nodeName, link: nil, routerMAC: nil, - nft: nft, ipv4: &fakeMgmtPortV4IPFamilyConfig, ipv6: &fakeMgmtPortV6IPFamilyConfig, } @@ -625,13 +622,24 @@ var _ = Describe("UserDefinedNetworkGateway", func() { defer GinkgoRecover() gatewayNextHops, gatewayIntf, err := getGatewayNextHops() Expect(err).NotTo(HaveOccurred()) - testNCM := &fakenad.FakeNetworkControllerManager{} - nadController, err := networkAttachDefController.NewNetAttachDefinitionController("test", testNCM, wf, nil) - Expect(err).NotTo(HaveOccurred()) // make preparations for creating openflow manager in DNCC which can be used for SNCC - localGw, err := newGateway(nodeName, ovntest.MustParseIPNets(v4NodeSubnet, v6NodeSubnet), gatewayNextHops, - gatewayIntf, "", ifAddrs, nodeAnnotatorMock, &fakeMgmtPortConfig, &kubeMock, wf, rm, nil, nadController, config.GatewayModeLocal) + localGw, err := newGateway( + nodeName, + ovntest.MustParseIPNets(v4NodeSubnet, v6NodeSubnet), + gatewayNextHops, + gatewayIntf, + "", + ifAddrs, + nodeAnnotatorMock, + &fakeMgmtPortConfig, + &kubeMock, + wf, + rm, + nil, + networkmanager.Default().Interface(), + config.GatewayModeLocal, + ) Expect(err).NotTo(HaveOccurred()) stop := make(chan struct{}) wg := &sync.WaitGroup{} @@ -691,7 +699,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { checkDefaultSvcIsolationOVSFlows(flowMap["DEFAULT"], defaultUdnConfig, ofPortHost, bridgeMAC, svcCIDR) // Expect exactly one flow per UDN for table 2 for service isolation. - checkUDNSvcIsolationOVSFlows(flowMap["DEFAULT"], bridgeUdnConfig, "bluenet", ofPortHost, bridgeMAC, svcCIDR, 1) + checkUDNSvcIsolationOVSFlows(flowMap["DEFAULT"], bridgeUdnConfig, "bluenet", bridgeMAC, svcCIDR, 1) } // The second call to checkPorts() will return no ofPort for the UDN - simulating a deletion that already was @@ -721,7 +729,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { checkDefaultSvcIsolationOVSFlows(flowMap["DEFAULT"], defaultUdnConfig, ofPortHost, bridgeMAC, svcCIDR) // Expect no more flows per UDN for table 2 for service isolation. - checkUDNSvcIsolationOVSFlows(flowMap["DEFAULT"], bridgeUdnConfig, "bluenet", ofPortHost, bridgeMAC, svcCIDR, 0) + checkUDNSvcIsolationOVSFlows(flowMap["DEFAULT"], bridgeUdnConfig, "bluenet", bridgeMAC, svcCIDR, 0) } return nil }) @@ -744,8 +752,8 @@ var _ = Describe("UserDefinedNetworkGateway", func() { }, }, Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{ - {corev1.NodeInternalIP, strings.Split(v4NodeIP, "/")[0]}, - {corev1.NodeInternalIP, strings.Split(v6NodeIP, "/")[0]}}, + {Type: corev1.NodeInternalIP, Address: strings.Split(v4NodeIP, "/")[0]}, + {Type: corev1.NodeInternalIP, Address: strings.Split(v6NodeIP, "/")[0]}}, }, } nad := ovntest.GenerateNAD(netName, "rednad", "greenamespace", @@ -786,7 +794,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { err = wf.Start() _, _ = util.SetFakeIPTablesHelpers() - nft := nodenft.SetFakeNFTablesHelper() + _ = nodenft.SetFakeNFTablesHelper() Expect(err).NotTo(HaveOccurred()) @@ -804,7 +812,6 @@ var _ = Describe("UserDefinedNetworkGateway", func() { ifName: nodeName, link: nil, routerMAC: nil, - nft: nft, ipv4: &fakeMgmtPortV4IPFamilyConfig, ipv6: &fakeMgmtPortV6IPFamilyConfig, } @@ -839,12 +846,23 @@ var _ = Describe("UserDefinedNetworkGateway", func() { defer GinkgoRecover() gatewayNextHops, gatewayIntf, err := getGatewayNextHops() Expect(err).NotTo(HaveOccurred()) - testNCM := &fakenad.FakeNetworkControllerManager{} - nadController, err := networkAttachDefController.NewNetAttachDefinitionController("test", testNCM, wf, nil) - Expect(err).NotTo(HaveOccurred()) // make preparations for creating openflow manager in DNCC which can be used for SNCC - localGw, err := newGateway(nodeName, ovntest.MustParseIPNets(v4NodeSubnet, v6NodeSubnet), gatewayNextHops, - gatewayIntf, "", ifAddrs, nodeAnnotatorMock, &fakeMgmtPortConfig, &kubeMock, wf, rm, nil, nadController, config.GatewayModeLocal) + localGw, err := newGateway( + nodeName, + ovntest.MustParseIPNets(v4NodeSubnet, v6NodeSubnet), + gatewayNextHops, + gatewayIntf, + "", + ifAddrs, + nodeAnnotatorMock, + &fakeMgmtPortConfig, + &kubeMock, + wf, + rm, + nil, + networkmanager.Default().Interface(), + config.GatewayModeLocal, + ) Expect(err).NotTo(HaveOccurred()) stop := make(chan struct{}) wg := &sync.WaitGroup{} @@ -902,7 +920,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { checkDefaultSvcIsolationOVSFlows(flowMap["DEFAULT"], defaultUdnConfig, ofPortHost, bridgeMAC, svcCIDR) // Expect exactly one flow per UDN for tables 0 and 2 for service isolation. - checkUDNSvcIsolationOVSFlows(flowMap["DEFAULT"], bridgeUdnConfig, "bluenet", ofPortHost, bridgeMAC, svcCIDR, 1) + checkUDNSvcIsolationOVSFlows(flowMap["DEFAULT"], bridgeUdnConfig, "bluenet", bridgeMAC, svcCIDR, 1) } // The second call to checkPorts() will return no ofPort for the UDN - simulating a deletion that already was @@ -932,7 +950,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { checkDefaultSvcIsolationOVSFlows(flowMap["DEFAULT"], defaultUdnConfig, ofPortHost, bridgeMAC, svcCIDR) // Expect no more flows per UDN for tables 0 and 2 for service isolation. - checkUDNSvcIsolationOVSFlows(flowMap["DEFAULT"], bridgeUdnConfig, "bluenet", ofPortHost, bridgeMAC, svcCIDR, 0) + checkUDNSvcIsolationOVSFlows(flowMap["DEFAULT"], bridgeUdnConfig, "bluenet", bridgeMAC, svcCIDR, 0) } return nil }) @@ -1187,7 +1205,7 @@ func TestConstructUDNVRFIPRules(t *testing.T) { config.Gateway.V4MasqueradeSubnet = "169.254.0.0/16" for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - g := gomega.NewWithT(t) + g := NewWithT(t) config.IPv4Mode = test.v4mode config.IPv6Mode = test.v6mode cidr := "" @@ -1207,15 +1225,15 @@ func TestConstructUDNVRFIPRules(t *testing.T) { udnGateway, err := NewUserDefinedNetworkGateway(netInfo, 3, nil, nil, nil, nil, nil, &gateway{}) g.Expect(err).NotTo(HaveOccurred()) rules, err := udnGateway.constructUDNVRFIPRules(test.vrftableID) - g.Expect(err).To(gomega.BeNil()) + g.Expect(err).To(BeNil()) for i, rule := range rules { - g.Expect(rule.Priority).To(gomega.Equal(test.expectedRules[i].priority)) - g.Expect(rule.Table).To(gomega.Equal(test.expectedRules[i].table)) - g.Expect(rule.Family).To(gomega.Equal(test.expectedRules[i].family)) + g.Expect(rule.Priority).To(Equal(test.expectedRules[i].priority)) + g.Expect(rule.Table).To(Equal(test.expectedRules[i].table)) + g.Expect(rule.Family).To(Equal(test.expectedRules[i].family)) if rule.Dst != nil { - g.Expect(*rule.Dst).To(gomega.Equal(test.expectedRules[i].dst)) + g.Expect(*rule.Dst).To(Equal(test.expectedRules[i].dst)) } else { - g.Expect(rule.Mark).To(gomega.Equal(test.expectedRules[i].mark)) + g.Expect(rule.Mark).To(Equal(test.expectedRules[i].mark)) } } }) diff --git a/go-controller/pkg/node/management-port-dpu.go b/go-controller/pkg/node/management-port-dpu.go index 13850af74c..1f243fa481 100644 --- a/go-controller/pkg/node/management-port-dpu.go +++ b/go-controller/pkg/node/management-port-dpu.go @@ -6,7 +6,6 @@ import ( "time" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/wait" listers "k8s.io/client-go/listers/core/v1" "k8s.io/klog/v2" @@ -32,7 +31,7 @@ func newManagementPortRepresentor(nodeName string, hostSubnets []*net.IPNet, rep } } -func (mp *managementPortRepresentor) Create(_ *routemanager.Controller, node *v1.Node, +func (mp *managementPortRepresentor) Create(isPodNetworkAdvertised bool, _ *routemanager.Controller, node *v1.Node, nodeLister listers.NodeLister, kubeInterface kube.Interface, waiter *startupWaiter) (*managementPortConfig, error) { k8sMgmtIntfName := types.K8sMgmtIntfName if config.OvnKubeNode.Mode == types.NodeModeFull { @@ -109,56 +108,51 @@ func (mp *managementPortRepresentor) Create(_ *routemanager.Controller, node *v1 } mpcfg := &managementPortConfig{ - ifName: k8sMgmtIntfName, - link: link, + ifName: k8sMgmtIntfName, + link: link, + reconcilePeriod: 5 * time.Second, } + mpcfg.isPodNetworkAdvertised.Store(isPodNetworkAdvertised) waiter.AddWait(managementPortReady, nil) return mpcfg, nil } -func (mp *managementPortRepresentor) checkRepresentorPortHealth(cfg *managementPortConfig) { +func (mp *managementPortRepresentor) checkRepresentorPortHealth(cfg *managementPortConfig) error { // After host reboot, management port link name changes back to default name. link, err := util.GetNetLinkOps().LinkByName(cfg.ifName) if err != nil { - klog.Errorf("Failed to get link device %s, error: %v", cfg.ifName, err) + klog.Warningf("Failed to get link device %s: %v", cfg.ifName, err) // Get management port representor by name link, err := util.GetNetLinkOps().LinkByName(mp.repName) if err != nil { - klog.Errorf("Failed to get link device %s, error: %v", mp.repName, err) - return + return fmt.Errorf("failed to get link device %s: %w", mp.repName, err) } if err = util.GetNetLinkOps().LinkSetDown(link); err != nil { - klog.Errorf("Failed to set link down for device %s. %v", mp.repName, err) - return + return fmt.Errorf("failed to set link down for device %s: %w", mp.repName, err) } if err = util.GetNetLinkOps().LinkSetName(link, cfg.ifName); err != nil { - klog.Errorf("Rename link from %s to %s failed: %v", mp.repName, cfg.ifName, err) - return + return fmt.Errorf("failed to rename link from %s to %s: %w", mp.repName, cfg.ifName, err) } if link.Attrs().MTU != config.Default.MTU { if err = util.GetNetLinkOps().LinkSetMTU(link, config.Default.MTU); err != nil { - klog.Errorf("Failed to set link MTU for device %s. %v", cfg.ifName, err) + return fmt.Errorf("failed to set link MTU for device %s: %w", cfg.ifName, err) } } if err = util.GetNetLinkOps().LinkSetUp(link); err != nil { - klog.Errorf("Failed to set link up for device %s. %v", cfg.ifName, err) + return fmt.Errorf("failed to set link up for device %s: %w", cfg.ifName, err) } cfg.link = link } else if (link.Attrs().Flags & net.FlagUp) != net.FlagUp { if err = util.GetNetLinkOps().LinkSetUp(link); err != nil { - klog.Errorf("Failed to set link up for device %s. %v", cfg.ifName, err) + return fmt.Errorf("failed to set link up for device %s: %w", cfg.ifName, err) } } + return nil } -func (mp *managementPortRepresentor) CheckManagementPortHealth(_ *routemanager.Controller, cfg *managementPortConfig, stopChan chan struct{}) { - go wait.Until( - func() { - mp.checkRepresentorPortHealth(cfg) - }, - 5*time.Second, - stopChan) +func (mp *managementPortRepresentor) CheckManagementPortHealth(_ *routemanager.Controller, cfg *managementPortConfig) error { + return mp.checkRepresentorPortHealth(cfg) } // Port representors should not have any IP address assignable to them, thus always return false. @@ -179,7 +173,7 @@ func newManagementPortNetdev(hostSubnets []*net.IPNet, netdevName string) Manage } } -func (mp *managementPortNetdev) Create(routeManager *routemanager.Controller, node *v1.Node, +func (mp *managementPortNetdev) Create(isRoutingAdvertised bool, routeManager *routemanager.Controller, node *v1.Node, nodeLister listers.NodeLister, kubeInterface kube.Interface, waiter *startupWaiter) (*managementPortConfig, error) { klog.Infof("Lookup netdevice link and existing management port using '%v'", mp.netdevName) link, err := util.GetNetLinkOps().LinkByName(mp.netdevName) @@ -245,20 +239,15 @@ func (mp *managementPortNetdev) Create(routeManager *routemanager.Controller, no } // Setup Iptable and routes - cfg, err := createPlatformManagementPort(routeManager, types.K8sMgmtIntfName, mp.hostSubnets) + cfg, err := createPlatformManagementPort(routeManager, types.K8sMgmtIntfName, mp.hostSubnets, isRoutingAdvertised) if err != nil { return nil, err } return cfg, nil } -func (mp *managementPortNetdev) CheckManagementPortHealth(routeManager *routemanager.Controller, cfg *managementPortConfig, stopChan chan struct{}) { - go wait.Until( - func() { - checkManagementPortHealth(routeManager, cfg) - }, - 30*time.Second, - stopChan) +func (mp *managementPortNetdev) CheckManagementPortHealth(routeManager *routemanager.Controller, cfg *managementPortConfig) error { + return checkManagementPortHealth(routeManager, cfg) } // Management port Netdev should have IP addresses assignable to them. diff --git a/go-controller/pkg/node/management-port.go b/go-controller/pkg/node/management-port.go index ada7342be3..f3918669a3 100644 --- a/go-controller/pkg/node/management-port.go +++ b/go-controller/pkg/node/management-port.go @@ -9,7 +9,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" listers "k8s.io/client-go/listers/core/v1" - + "k8s.io/client-go/util/retry" "k8s.io/klog/v2" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" @@ -23,10 +23,10 @@ import ( type ManagementPort interface { // Create Management port, use annotator to update node annotation with management port details // and waiter to set up condition to wait on for management port creation - Create(routeManager *routemanager.Controller, node *v1.Node, nodeLister listers.NodeLister, kubeInterface kube.Interface, waiter *startupWaiter) (*managementPortConfig, error) + Create(isRoutingAdvertised bool, routeManager *routemanager.Controller, node *v1.Node, nodeLister listers.NodeLister, kubeInterface kube.Interface, waiter *startupWaiter) (*managementPortConfig, error) // CheckManagementPortHealth checks periodically for management port health until stopChan is posted // or closed and reports any warnings/errors to log - CheckManagementPortHealth(routeManager *routemanager.Controller, cfg *managementPortConfig, stopChan chan struct{}) + CheckManagementPortHealth(routeManager *routemanager.Controller, cfg *managementPortConfig) error // Currently, the management port(s) that doesn't have an assignable IP address are the following cases: // - Full mode with HW backed device (e.g. Virtual Function Representor). // - DPU mode with Virtual Function Representor. @@ -78,7 +78,7 @@ func newManagementPort(nodeName string, hostSubnets []*net.IPNet) ManagementPort } } -func (mp *managementPort) Create(routeManager *routemanager.Controller, node *v1.Node, +func (mp *managementPort) Create(isRoutingAdvertised bool, routeManager *routemanager.Controller, node *v1.Node, nodeLister listers.NodeLister, kubeInterface kube.Interface, waiter *startupWaiter) (*managementPortConfig, error) { for _, mgmtPortName := range []string{types.K8sMgmtIntfName, types.K8sMgmtIntfName + "_0"} { if err := syncMgmtPortInterface(mp.hostSubnets, mgmtPortName, true); err != nil { @@ -114,7 +114,7 @@ func (mp *managementPort) Create(routeManager *routemanager.Controller, node *v1 return nil, err } - cfg, err := createPlatformManagementPort(routeManager, types.K8sMgmtIntfName, mp.hostSubnets) + cfg, err := createPlatformManagementPort(routeManager, types.K8sMgmtIntfName, mp.hostSubnets, isRoutingAdvertised) if err != nil { return nil, err } @@ -123,13 +123,8 @@ func (mp *managementPort) Create(routeManager *routemanager.Controller, node *v1 return cfg, nil } -func (mp *managementPort) CheckManagementPortHealth(routeManager *routemanager.Controller, cfg *managementPortConfig, stopChan chan struct{}) { - go wait.Until( - func() { - checkManagementPortHealth(routeManager, cfg) - }, - 30*time.Second, - stopChan) +func (mp *managementPort) CheckManagementPortHealth(routeManager *routemanager.Controller, cfg *managementPortConfig) error { + return checkManagementPortHealth(routeManager, cfg) } // OVS Internal Port Netdev should have IP addresses assignable to them. @@ -162,3 +157,72 @@ func managementPortReady() (bool, error) { klog.Infof("Management port %s is ready", k8sMgmtIntfName) return true, nil } + +type managementPortEntry struct { + port ManagementPort + config *managementPortConfig + routeManager *routemanager.Controller + reconcile chan struct{} +} + +func NewManagementPortEntry(port ManagementPort, cfg *managementPortConfig, routeManager *routemanager.Controller) *managementPortEntry { + return &managementPortEntry{ + port: port, + config: cfg, + routeManager: routeManager, + reconcile: make(chan struct{}, 1), + } +} + +func (p *managementPortEntry) Start(stopChan <-chan struct{}) { + go func() { + timer := time.NewTicker(p.config.reconcilePeriod) + defer timer.Stop() + for { + select { + case <-stopChan: + return + case <-timer.C: + p.Reconcile() + case <-p.reconcile: + err := retry.OnError( + wait.Backoff{ + Duration: 10 * time.Millisecond, + Steps: 4, + Factor: 5.0, + Cap: p.config.reconcilePeriod, + }, + func(error) bool { + select { + case <-stopChan: + return false + default: + return true + } + }, + p.doReconcile, + ) + if err != nil { + klog.Errorf("Failed to reconcile management port %s: %v", p.config.ifName, err) + } + } + timer.Reset(p.config.reconcilePeriod) + } + }() +} + +func (p *managementPortEntry) Reconcile() { + // trigger a new reconciliation if none was pending + select { + case p.reconcile <- struct{}{}: + default: + } +} + +func (p *managementPortEntry) doReconcile() error { + return p.port.CheckManagementPortHealth(p.routeManager, p.config) +} + +func (p *managementPortEntry) SetPodNetworkAdvertised(isPodNetworkAdvertised bool) { + p.config.isPodNetworkAdvertised.Store(isPodNetworkAdvertised) +} diff --git a/go-controller/pkg/node/management-port_dpu_test.go b/go-controller/pkg/node/management-port_dpu_test.go index ef88ddd2f2..7ae8fe4030 100644 --- a/go-controller/pkg/node/management-port_dpu_test.go +++ b/go-controller/pkg/node/management-port_dpu_test.go @@ -14,6 +14,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" kubeMocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube/mocks" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" mocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/vishvananda/netlink" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -64,6 +65,7 @@ var _ = Describe("Mananagement port DPU tests", func() { Expect(err).NotTo(HaveOccurred()) waiter = newStartupWaiter() util.SetNetLinkOpMockInst(netlinkOpsMock) + nftables.SetFakeNFTablesHelper() }) AfterEach(func() { @@ -82,7 +84,7 @@ var _ = Describe("Mananagement port DPU tests", func() { netlinkOpsMock.On("LinkByName", "non-existent-netdev").Return(nil, fmt.Errorf("netlink mock error")) netlinkOpsMock.On("IsLinkNotFoundError", mock.Anything).Return(false) - _, err := mgmtPortDpu.Create(nil, nil, nil, nil, waiter) + _, err := mgmtPortDpu.Create(false, nil, nil, nil, nil, waiter) Expect(execMock.CalledMatchesExpected()).To(BeTrue(), execMock.ErrorDesc) Expect(err).To(HaveOccurred()) }) @@ -100,7 +102,7 @@ var _ = Describe("Mananagement port DPU tests", func() { nil, fmt.Errorf("failed to get interface")) netlinkOpsMock.On("IsLinkNotFoundError", mock.Anything).Return(true) - _, err := mgmtPortDpu.Create(nil, nil, nil, nil, waiter) + _, err := mgmtPortDpu.Create(false, nil, nil, nil, nil, waiter) Expect(err).To(HaveOccurred()) Expect(execMock.CalledMatchesExpected()).To(BeTrue(), execMock.ErrorDesc) }) @@ -124,7 +126,7 @@ var _ = Describe("Mananagement port DPU tests", func() { }) mockOVSListInterfaceMgmtPortNotExistCmd(execMock, types.K8sMgmtIntfName+"_0") - _, err := mgmtPortDpu.Create(nil, nil, nil, nil, waiter) + _, err := mgmtPortDpu.Create(false, nil, nil, nil, nil, waiter) Expect(execMock.CalledMatchesExpected()).To(BeTrue(), execMock.ErrorDesc) Expect(err).To(HaveOccurred()) }) @@ -184,7 +186,7 @@ var _ = Describe("Mananagement port DPU tests", func() { Expect(err).NotTo(HaveOccurred()) Expect(watchFactory.Start()).To(Succeed()) - mpcfg, err := mgmtPortDpu.Create(nil, node, watchFactory.NodeCoreInformer().Lister(), kubeInterface, waiter) + mpcfg, err := mgmtPortDpu.Create(false, nil, node, watchFactory.NodeCoreInformer().Lister(), kubeInterface, waiter) Expect(execMock.CalledMatchesExpected()).To(BeTrue(), execMock.ErrorDesc) Expect(err).ToNot(HaveOccurred()) Expect(mpcfg.ifName).To(Equal(types.K8sMgmtIntfName + "_0")) @@ -240,7 +242,7 @@ var _ = Describe("Mananagement port DPU tests", func() { Expect(err).NotTo(HaveOccurred()) Expect(watchFactory.Start()).To(Succeed()) - mpcfg, err := mgmtPortDpu.Create(nil, node, watchFactory.NodeCoreInformer().Lister(), kubeInterface, waiter) + mpcfg, err := mgmtPortDpu.Create(false, nil, node, watchFactory.NodeCoreInformer().Lister(), kubeInterface, waiter) Expect(execMock.CalledMatchesExpected()).To(BeTrue(), execMock.ErrorDesc) Expect(err).ToNot(HaveOccurred()) Expect(mpcfg.ifName).To(Equal(types.K8sMgmtIntfName + "_0")) @@ -256,7 +258,7 @@ var _ = Describe("Mananagement port DPU tests", func() { netlinkOpsMock.On("LinkByName", "non-existent-netdev").Return(nil, fmt.Errorf("netlink mock error")) netlinkOpsMock.On("IsLinkNotFoundError", mock.Anything).Return(false) - _, err := mgmtPortDpuHost.Create(nil, nil, nil, nil, waiter) + _, err := mgmtPortDpuHost.Create(false, nil, nil, nil, nil, waiter) Expect(err).To(HaveOccurred()) }) @@ -270,7 +272,7 @@ var _ = Describe("Mananagement port DPU tests", func() { nil, fmt.Errorf("failed to get interface")) netlinkOpsMock.On("IsLinkNotFoundError", mock.Anything).Return(true) - _, err := mgmtPortDpuHost.Create(nil, nil, nil, nil, waiter) + _, err := mgmtPortDpuHost.Create(false, nil, nil, nil, nil, waiter) Expect(err).To(HaveOccurred()) }) @@ -307,7 +309,7 @@ var _ = Describe("Mananagement port DPU tests", func() { netlinkOpsMock.On("LinkByName", mock.Anything).Return(nil, fmt.Errorf( "createPlatformManagementPort error")) - _, err = mgmtPortDpuHost.Create(nil, nil, nil, nil, nil) + _, err = mgmtPortDpuHost.Create(false, nil, nil, nil, nil, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("createPlatformManagementPort error")) }) @@ -333,7 +335,7 @@ var _ = Describe("Mananagement port DPU tests", func() { netlinkOpsMock.On("LinkByName", mock.Anything).Return(nil, fmt.Errorf( "createPlatformManagementPort error")).Once() - _, err = mgmtPortDpuHost.Create(nil, nil, nil, nil, nil) + _, err = mgmtPortDpuHost.Create(false, nil, nil, nil, nil, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring( "createPlatformManagementPort error")) diff --git a/go-controller/pkg/node/management-port_linux.go b/go-controller/pkg/node/management-port_linux.go index a79beb8cad..eecbc7a653 100644 --- a/go-controller/pkg/node/management-port_linux.go +++ b/go-controller/pkg/node/management-port_linux.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "net" + "sync/atomic" "time" "github.com/coreos/go-iptables/iptables" @@ -52,18 +53,17 @@ type managementPortIPFamilyConfig struct { } type managementPortConfig struct { - ifName string - link netlink.Link - routerMAC net.HardwareAddr - nft knftables.Interface + ifName string + link netlink.Link + routerMAC net.HardwareAddr + isPodNetworkAdvertised atomic.Bool + reconcilePeriod time.Duration ipv4 *managementPortIPFamilyConfig ipv6 *managementPortIPFamilyConfig } func newManagementPortIPFamilyConfig(hostSubnet *net.IPNet, isIPv6 bool) (*managementPortIPFamilyConfig, error) { - var err error - cfg := &managementPortIPFamilyConfig{ ifAddr: util.GetNodeManagementIfAddr(hostSubnet), gwIP: util.GetNodeGatewayIfAddr(hostSubnet).IP, @@ -91,23 +91,17 @@ func newManagementPortIPFamilyConfig(hostSubnet *net.IPNet, isIPv6 bool) (*manag cfg.allSubnets = append(cfg.allSubnets, masqueradeSubnet) } - if err != nil { - return nil, err - } - return cfg, nil } -func newManagementPortConfig(interfaceName string, hostSubnets []*net.IPNet) (*managementPortConfig, error) { - nft, err := nodenft.GetNFTablesHelper() - if err != nil { - return nil, err - } - +func newManagementPortConfig(interfaceName string, hostSubnets []*net.IPNet, isPodNetworkAdvertised bool) (*managementPortConfig, error) { mpcfg := &managementPortConfig{ - ifName: interfaceName, - nft: nft, + ifName: interfaceName, + reconcilePeriod: 30 * time.Second, } + mpcfg.isPodNetworkAdvertised.Store(isPodNetworkAdvertised) + + var err error if mpcfg.link, err = util.LinkSetUp(mpcfg.ifName); err != nil { return nil, err } @@ -238,8 +232,13 @@ func setupManagementPortIPFamilyConfig(routeManager *routemanager.Controller, mp return warnings, err } + protocol := iptables.ProtocolIPv4 + if mpcfg.ipv6 != nil && cfg == mpcfg.ipv6 { + protocol = iptables.ProtocolIPv6 + } + // IPv6 forwarding is enabled globally - if cfg == mpcfg.ipv4 { + if protocol == iptables.ProtocolIPv4 { stdout, stderr, err := util.RunSysctl("-w", fmt.Sprintf("net.ipv4.conf.%s.forwarding=1", types.K8sMgmtIntfName)) if err != nil || stdout != fmt.Sprintf("net.ipv4.conf.%s.forwarding = 1", types.K8sMgmtIntfName) { return warnings, fmt.Errorf("could not set the correct forwarding value for interface %s: stdout: %v, stderr: %v, err: %v", @@ -272,7 +271,12 @@ func setupManagementPortNFTables(cfg *managementPortConfig) error { counterIfDebug = "counter" } - tx := cfg.nft.NewTransaction() + nft, err := nodenft.GetNFTablesHelper() + if err != nil { + return err + } + + tx := nft.NewTransaction() tx.Add(&knftables.Chain{ Name: nftablesMgmtPortChain, Comment: knftables.PtrTo("OVN SNAT to Management Port"), @@ -316,7 +320,30 @@ func setupManagementPortNFTables(cfg *managementPortConfig) error { ), }) + isPodNetworkAdvertised := cfg.isPodNetworkAdvertised.Load() + if cfg.ipv4 != nil { + if isPodNetworkAdvertised { + tx.Add(&knftables.Rule{ + Chain: nftablesMgmtPortChain, + Rule: knftables.Concat( + "meta nfproto ipv4", + "fib saddr type != local", + counterIfDebug, + "return", + ), + }) + } + // don't SNAT if the source IP is already the Mgmt port IP + tx.Add(&knftables.Rule{ + Chain: nftablesMgmtPortChain, + Rule: knftables.Concat( + "meta nfproto ipv4", + "ip saddr", cfg.ipv4.ifAddr.IP, + counterIfDebug, + "return", + ), + }) tx.Add(&knftables.Rule{ Chain: nftablesMgmtPortChain, Rule: knftables.Concat( @@ -335,6 +362,27 @@ func setupManagementPortNFTables(cfg *managementPortConfig) error { } if cfg.ipv6 != nil { + if isPodNetworkAdvertised { + tx.Add(&knftables.Rule{ + Chain: nftablesMgmtPortChain, + Rule: knftables.Concat( + "meta nfproto ipv6", + "fib saddr type != local", + counterIfDebug, + "return", + ), + }) + } + // don't SNAT if the source IP is already the Mgmt port IP + tx.Add(&knftables.Rule{ + Chain: nftablesMgmtPortChain, + Rule: knftables.Concat( + "meta nfproto ipv6", + "ip6 saddr", cfg.ipv6.ifAddr.IP, + counterIfDebug, + "return", + ), + }) tx.Add(&knftables.Rule{ Chain: nftablesMgmtPortChain, Rule: knftables.Concat( @@ -352,7 +400,7 @@ func setupManagementPortNFTables(cfg *managementPortConfig) error { }) } - err := cfg.nft.Run(context.TODO(), tx) + err = nft.Run(context.TODO(), tx) if err != nil { return fmt.Errorf("could not update nftables rule for management port: %v", err) } @@ -362,11 +410,11 @@ func setupManagementPortNFTables(cfg *managementPortConfig) error { // createPlatformManagementPort creates a management port attached to the node switch // that lets the node access its pods via their private IP address. This is used // for health checking and other management tasks. -func createPlatformManagementPort(routeManager *routemanager.Controller, interfaceName string, localSubnets []*net.IPNet) (*managementPortConfig, error) { +func createPlatformManagementPort(routeManager *routemanager.Controller, interfaceName string, localSubnets []*net.IPNet, isRoutingAdvertised bool) (*managementPortConfig, error) { var cfg *managementPortConfig var err error - if cfg, err = newManagementPortConfig(interfaceName, localSubnets); err != nil { + if cfg, err = newManagementPortConfig(interfaceName, localSubnets, isRoutingAdvertised); err != nil { return nil, err } @@ -523,15 +571,14 @@ func DelLegacyMgtPortIptRules() { // 1. route entries to cluster CIDR and service CIDR through management port // 2. ARP entry for the node subnet's gateway ip // 3. nftables rules for SNATing packets entering the logical topology -func checkManagementPortHealth(routeManager *routemanager.Controller, cfg *managementPortConfig) { +func checkManagementPortHealth(routeManager *routemanager.Controller, cfg *managementPortConfig) error { warnings, err := setupManagementPortConfig(routeManager, cfg) for _, warning := range warnings { klog.Warningf(warning) } if err != nil { - klog.Errorf(err.Error()) - } - if err = setupManagementPortNFTables(cfg); err != nil { - klog.Errorf(err.Error()) + return err } + + return setupManagementPortNFTables(cfg) } diff --git a/go-controller/pkg/node/management-port_linux_test.go b/go-controller/pkg/node/management-port_linux_test.go index c97a73c991..c6e0c4675a 100644 --- a/go-controller/pkg/node/management-port_linux_test.go +++ b/go-controller/pkg/node/management-port_linux_test.go @@ -10,7 +10,6 @@ import ( "io/ioutil" "net" "os" - "path/filepath" "strings" "sync" "time" @@ -50,14 +49,6 @@ var _ = AfterSuite(func() { Expect(err).NotTo(HaveOccurred()) }) -func createTempFile(name string) (string, error) { - fname := filepath.Join(tmpDir, name) - if err := ioutil.WriteFile(fname, []byte{0x20}, 0o644); err != nil { - return "", err - } - return fname, nil -} - type managementPortTestConfig struct { family int @@ -67,6 +58,8 @@ type managementPortTestConfig struct { expectedManagementPortIP string expectedGatewayIP string + + isRoutingAdvertised bool } func (mptc *managementPortTestConfig) GetNodeSubnetCIDR() *net.IPNet { @@ -92,6 +85,8 @@ func checkMgmtPortTestNFTables(configs []managementPortTestConfig, mgmtPortName var returnRule, snatV4Rule, snatV6Rule string var wantReturnRule, wantSNATV4Rule, wantSNATV6Rule bool + var returnNonLocalV4Rule, returnNonLocalV6Rule, returnMgmtIPV4Rule, returnMgmtIPV6Rule string + var wantReturnNonLocalV4Rule, wantReturnNonLocalV6Rule, wantReturnMgmtIPV4Rule, wantReturnMgmtIPV6Rule bool returnRule = fmt.Sprintf("oifname != %q return", mgmtPortName) wantReturnRule = true @@ -100,9 +95,17 @@ func checkMgmtPortTestNFTables(configs []managementPortTestConfig, mgmtPortName if cfg.family == netlink.FAMILY_V4 { snatV4Rule = "snat ip to " + cfg.expectedManagementPortIP wantSNATV4Rule = true + returnNonLocalV4Rule = "meta nfproto ipv4 fib saddr type != local" + wantReturnNonLocalV4Rule = cfg.isRoutingAdvertised + returnMgmtIPV4Rule = "meta nfproto ipv4 ip saddr " + cfg.expectedManagementPortIP + wantReturnMgmtIPV4Rule = true } else { snatV6Rule = "snat ip6 to " + cfg.expectedManagementPortIP wantSNATV6Rule = true + returnNonLocalV6Rule = "meta nfproto ipv6 fib saddr type != local" + wantReturnNonLocalV6Rule = cfg.isRoutingAdvertised + returnMgmtIPV6Rule = "meta nfproto ipv6 ip6 saddr " + cfg.expectedManagementPortIP + wantReturnMgmtIPV6Rule = true } } @@ -113,12 +116,24 @@ func checkMgmtPortTestNFTables(configs []managementPortTestConfig, mgmtPortName wantSNATV4Rule = false } else if wantSNATV6Rule && strings.Contains(rule.Rule, snatV6Rule) { wantSNATV6Rule = false + } else if wantReturnNonLocalV4Rule && strings.Contains(rule.Rule, returnNonLocalV4Rule) { + wantReturnNonLocalV4Rule = false + } else if wantReturnNonLocalV6Rule && strings.Contains(rule.Rule, returnNonLocalV6Rule) { + wantReturnNonLocalV6Rule = false + } else if wantReturnMgmtIPV4Rule && strings.Contains(rule.Rule, returnMgmtIPV4Rule) { + wantReturnMgmtIPV4Rule = false + } else if wantReturnMgmtIPV6Rule && strings.Contains(rule.Rule, returnMgmtIPV6Rule) { + wantReturnMgmtIPV6Rule = false } } Expect(wantReturnRule).To(BeFalse(), "did not find rule with %q", returnRule) Expect(wantSNATV4Rule).To(BeFalse(), "did not find rule with %q", snatV4Rule) Expect(wantSNATV6Rule).To(BeFalse(), "did not find rule with %q", snatV6Rule) + Expect(wantReturnNonLocalV4Rule).To(BeFalse(), "did not find rule with %q", returnNonLocalV4Rule) + Expect(wantReturnNonLocalV6Rule).To(BeFalse(), "did not find rule with %q", returnNonLocalV6Rule) + Expect(wantReturnMgmtIPV4Rule).To(BeFalse(), "did not find rule with %q", returnMgmtIPV4Rule) + Expect(wantReturnMgmtIPV6Rule).To(BeFalse(), "did not find rule with %q", returnMgmtIPV6Rule) } // checkMgmtTestPortIpsAndRoutes checks IPs and Routes of the management port @@ -237,8 +252,6 @@ func testManagementPort(ctx *cli.Context, fexec *ovntest.FakeExec, testNS ns.Net mgtPortAddrs[i] = cfg.GetMgtPortAddr() } - nodenft.SetFakeNFTablesHelper() - existingNode := v1.Node{ObjectMeta: metav1.ObjectMeta{ Name: nodeName, }} @@ -286,7 +299,14 @@ func testManagementPort(ctx *cli.Context, fexec *ovntest.FakeExec, testNS ns.Net netdevName, rep := "", "" mgmtPorts := NewManagementPorts(nodeName, nodeSubnetCIDRs, netdevName, rep) - _, err = mgmtPorts[0].Create(rm, &existingNode, watchFactory.NodeCoreInformer().Lister(), kubeInterface, waiter) + _, err = mgmtPorts[0].Create( + configs[0].isRoutingAdvertised, + rm, + &existingNode, + watchFactory.NodeCoreInformer().Lister(), + kubeInterface, + waiter, + ) Expect(err).NotTo(HaveOccurred()) checkMgmtTestPortIpsAndRoutes(configs, mgtPort, mgtPortAddrs, expectedLRPMAC) return nil @@ -383,7 +403,7 @@ func testManagementPortDPU(ctx *cli.Context, fexec *ovntest.FakeExec, testNS ns. netdevName, rep := "pf0vf0", "pf0vf0" mgmtPorts := NewManagementPorts(nodeName, nodeSubnetCIDRs, netdevName, rep) - _, err = mgmtPorts[0].Create(rm, &existingNode, watchFactory.NodeCoreInformer().Lister(), kubeInterface, waiter) + _, err = mgmtPorts[0].Create(false, rm, &existingNode, watchFactory.NodeCoreInformer().Lister(), kubeInterface, waiter) Expect(err).NotTo(HaveOccurred()) // make sure interface was renamed and mtu was set l, err := netlink.LinkByName(mgtPort) @@ -436,8 +456,6 @@ func testManagementPortDPUHost(ctx *cli.Context, fexec *ovntest.FakeExec, testNS mgtPortAddrs[i] = cfg.GetMgtPortAddr() } - nodenft.SetFakeNFTablesHelper() - _, err = config.InitConfig(ctx, fexec, nil) Expect(err).NotTo(HaveOccurred()) wg := &sync.WaitGroup{} @@ -460,7 +478,7 @@ func testManagementPortDPUHost(ctx *cli.Context, fexec *ovntest.FakeExec, testNS netdevName, rep := "pf0vf0", "" mgmtPorts := NewManagementPorts(nodeName, nodeSubnetCIDRs, netdevName, rep) - _, err = mgmtPorts[0].Create(rm, nil, nil, nil, nil) + _, err = mgmtPorts[0].Create(configs[0].isRoutingAdvertised, rm, nil, nil, nil, nil) Expect(err).NotTo(HaveOccurred()) checkMgmtTestPortIpsAndRoutes(configs, mgtPort, mgtPortAddrs, expectedLRPMAC) // check mgmt port MAC, mtu and link state @@ -505,6 +523,7 @@ var _ = Describe("Management Port Operations", func() { err := util.SetExec(execMock) Expect(err).NotTo(HaveOccurred()) util.SetNetLinkOpMockInst(netlinkOpsMock) + nodenft.SetFakeNFTablesHelper() }) AfterEach(func() { @@ -842,6 +861,31 @@ var _ = Describe("Management Port Operations", func() { Expect(err).NotTo(HaveOccurred()) }) + ovntest.OnSupportedPlatformsIt("sets up the management port for BGP advertised IPv4 clusters", func() { + app.Action = func(ctx *cli.Context) error { + testManagementPort(ctx, fexec, testNS, + []managementPortTestConfig{ + { + family: netlink.FAMILY_V4, + + clusterCIDR: v4clusterCIDR, + nodeSubnet: v4nodeSubnet, + + expectedManagementPortIP: v4mgtPortIP, + expectedGatewayIP: v4gwIP, + + isRoutingAdvertised: true, + }, + }, v4lrpMAC, true) + return nil + } + err := app.Run([]string{ + app.Name, + "--cluster-subnets=" + v4clusterCIDR, + }) + Expect(err).NotTo(HaveOccurred()) + }) + ovntest.OnSupportedPlatformsIt("sets up the management port for IPv6 clusters", func() { app.Action = func(ctx *cli.Context) error { testManagementPort(ctx, fexec, testNS, @@ -867,6 +911,33 @@ var _ = Describe("Management Port Operations", func() { Expect(err).NotTo(HaveOccurred()) }) + ovntest.OnSupportedPlatformsIt("sets up the management port for BGP advertised IPv6 clusters", func() { + app.Action = func(ctx *cli.Context) error { + testManagementPort(ctx, fexec, testNS, + []managementPortTestConfig{ + { + family: netlink.FAMILY_V6, + + clusterCIDR: v6clusterCIDR, + serviceCIDR: v6serviceCIDR, + nodeSubnet: v6nodeSubnet, + + expectedManagementPortIP: v6mgtPortIP, + expectedGatewayIP: v6gwIP, + + isRoutingAdvertised: true, + }, + }, v6lrpMAC, true) + return nil + } + err := app.Run([]string{ + app.Name, + "--cluster-subnets=" + v6clusterCIDR, + "--k8s-service-cidr=" + v6serviceCIDR, + }) + Expect(err).NotTo(HaveOccurred()) + }) + ovntest.OnSupportedPlatformsIt("sets up the management port for dual-stack clusters", func() { app.Action = func(ctx *cli.Context) error { testManagementPort(ctx, fexec, testNS, @@ -901,14 +972,52 @@ var _ = Describe("Management Port Operations", func() { }) Expect(err).NotTo(HaveOccurred()) }) + + ovntest.OnSupportedPlatformsIt("sets up the management port for BGP advertised dual-stack clusters", func() { + app.Action = func(ctx *cli.Context) error { + testManagementPort(ctx, fexec, testNS, + []managementPortTestConfig{ + { + family: netlink.FAMILY_V4, + + clusterCIDR: v4clusterCIDR, + serviceCIDR: v4serviceCIDR, + nodeSubnet: v4nodeSubnet, + + expectedManagementPortIP: v4mgtPortIP, + expectedGatewayIP: v4gwIP, + + isRoutingAdvertised: true, + }, + { + family: netlink.FAMILY_V6, + + clusterCIDR: v6clusterCIDR, + serviceCIDR: v6serviceCIDR, + nodeSubnet: v6nodeSubnet, + + expectedManagementPortIP: v6mgtPortIP, + expectedGatewayIP: v6gwIP, + + isRoutingAdvertised: true, + }, + }, v4lrpMAC, true) + return nil + } + err := app.Run([]string{ + app.Name, + "--cluster-subnets=" + v4clusterCIDR + "," + v6clusterCIDR, + "--k8s-service-cidr=" + v4serviceCIDR + "," + v6serviceCIDR, + }) + Expect(err).NotTo(HaveOccurred()) + }) }) Context("Management Port, ovnkube node mode dpu", func() { BeforeEach(func() { - var err error // Set up a fake k8sMgmt interface - err = testNS.Do(func(ns.NetNS) error { + err := testNS.Do(func(ns.NetNS) error { defer GinkgoRecover() ovntest.AddLink(mgmtPortNetdev) return nil diff --git a/go-controller/pkg/node/node_ip_handler_linux.go b/go-controller/pkg/node/node_ip_handler_linux.go index 0afd0dbed9..eb5c368eff 100644 --- a/go-controller/pkg/node/node_ip_handler_linux.go +++ b/go-controller/pkg/node/node_ip_handler_linux.go @@ -50,12 +50,12 @@ func newAddressManager(nodeName string, k kube.Interface, config *managementPort // newAddressManagerInternal creates a new address manager; this function is // only expose for testcases to disable netlink subscription to ensure // reproducibility of unit tests. -func newAddressManagerInternal(nodeName string, k kube.Interface, config *managementPortConfig, watchFactory factory.NodeWatchFactory, gwBridge *bridgeConfiguration, useNetlink bool) *addressManager { +func newAddressManagerInternal(nodeName string, k kube.Interface, mgmtConfig *managementPortConfig, watchFactory factory.NodeWatchFactory, gwBridge *bridgeConfiguration, useNetlink bool) *addressManager { mgr := &addressManager{ nodeName: nodeName, watchFactory: watchFactory, cidrs: sets.New[string](), - mgmtPortConfig: config, + mgmtPortConfig: mgmtConfig, gatewayBridge: gwBridge, OnChanged: func() {}, useNetlink: useNetlink, @@ -228,7 +228,7 @@ func (c *addressManager) handleNodePrimaryAddrChange() { klog.Errorf("Address Manager failed to check node primary address change: %v", err) return } - if nodePrimaryAddrChanged { + if nodePrimaryAddrChanged && config.Default.EncapIP == "" { klog.Infof("Node primary address changed to %v. Updating OVN encap IP.", c.nodePrimaryAddr) updateOVNEncapIPAndReconnect(c.nodePrimaryAddr) } @@ -536,6 +536,7 @@ func updateOVNEncapIPAndReconnect(newIP net.IP) { } } + config.Default.EffectiveEncapIP = newIP.String() confCmd := []string{ "set", "Open_vSwitch", diff --git a/go-controller/pkg/node/node_ip_handler_linux_test.go b/go-controller/pkg/node/node_ip_handler_linux_test.go index 5a8f1141f5..087def035b 100644 --- a/go-controller/pkg/node/node_ip_handler_linux_test.go +++ b/go-controller/pkg/node/node_ip_handler_linux_test.go @@ -376,11 +376,12 @@ func configureKubeOVNContext(nodeName string, useNetlink bool) *testCtx { err = tc.watchFactory.Start() Expect(err).NotTo(HaveOccurred()) + _ = nodenft.SetFakeNFTablesHelper() + fakeMgmtPortConfig := &managementPortConfig{ ifName: nodeName, link: nil, routerMAC: nil, - nft: nodenft.SetFakeNFTablesHelper(), ipv4: &managementPortIPFamilyConfig{ allSubnets: nil, ifAddr: tc.mgmtPortIP4, diff --git a/go-controller/pkg/node/openflow_manager.go b/go-controller/pkg/node/openflow_manager.go index 9094ebedc2..91afdd09a1 100644 --- a/go-controller/pkg/node/openflow_manager.go +++ b/go-controller/pkg/node/openflow_manager.go @@ -162,7 +162,8 @@ func newGatewayOpenFlowManager(gwBridge, exGWBridge *bridgeConfiguration, subnet flowChan: make(chan struct{}, 1), } - if err := ofm.updateBridgeFlowCache(subnets, extraIPs); err != nil { + isRoutingAdvertised := false + if err := ofm.updateBridgeFlowCache(subnets, extraIPs, isRoutingAdvertised); err != nil { return nil, err } @@ -206,7 +207,7 @@ func (c *openflowManager) Run(stopChan <-chan struct{}, doneWg *sync.WaitGroup) // updateBridgeFlowCache generates the "static" per-bridge flows // note: this is shared between shared and local gateway modes -func (c *openflowManager) updateBridgeFlowCache(subnets []*net.IPNet, extraIPs []net.IP) error { +func (c *openflowManager) updateBridgeFlowCache(subnets []*net.IPNet, extraIPs []net.IP, isPodNetworkAdvertised bool) error { // protect defaultBridge config from being updated by gw.nodeIPManager c.defaultBridge.Lock() defer c.defaultBridge.Unlock() @@ -218,7 +219,7 @@ func (c *openflowManager) updateBridgeFlowCache(subnets []*net.IPNet, extraIPs [ if err != nil { return err } - dftCommonFlows, err := commonFlows(subnets, c.defaultBridge) + dftCommonFlows, err := commonFlows(subnets, c.defaultBridge, isPodNetworkAdvertised) if err != nil { return err } @@ -232,7 +233,7 @@ func (c *openflowManager) updateBridgeFlowCache(subnets []*net.IPNet, extraIPs [ c.externalGatewayBridge.Lock() defer c.externalGatewayBridge.Unlock() c.updateExBridgeFlowCacheEntry("NORMAL", []string{fmt.Sprintf("table=0,priority=0,actions=%s\n", util.NormalAction)}) - exGWBridgeDftFlows, err := commonFlows(subnets, c.externalGatewayBridge) + exGWBridgeDftFlows, err := commonFlows(subnets, c.externalGatewayBridge, false) if err != nil { return err } diff --git a/go-controller/pkg/node/ovn_test.go b/go-controller/pkg/node/ovn_test.go index e0212f6b4e..92115f6bf0 100644 --- a/go-controller/pkg/node/ovn_test.go +++ b/go-controller/pkg/node/ovn_test.go @@ -85,8 +85,8 @@ func (o *FakeOVNNode) init() { Expect(err).NotTo(HaveOccurred()) cnnci := NewCommonNodeNetworkControllerInfo(o.fakeClient.KubeClient, o.fakeClient.AdminPolicyRouteClient, o.watcher, o.recorder, fakeNodeName, routemanager.NewController()) - o.nc = newDefaultNodeNetworkController(cnnci, o.stopChan, o.wg, routemanager.NewController(), nil) - // watcher is started by nodeNetworkControllerManager, not by nodeNetworkcontroller, so start it here. + o.nc = newDefaultNodeNetworkController(cnnci, o.stopChan, o.wg, routemanager.NewController()) + // watcher is started by nodeControllerManager, not by nodeNetworkController, so start it here. o.watcher.Start() o.nc.PreStart(context.TODO()) o.nc.Start(context.TODO()) diff --git a/go-controller/pkg/node/secondary_node_network_controller.go b/go-controller/pkg/node/secondary_node_network_controller.go index f3bf50f472..229124e799 100644 --- a/go-controller/pkg/node/secondary_node_network_controller.go +++ b/go-controller/pkg/node/secondary_node_network_controller.go @@ -31,12 +31,18 @@ type SecondaryNodeNetworkController struct { // NewSecondaryNodeNetworkController creates a new OVN controller for creating logical network // infrastructure and policy for the given secondary network. It supports layer3, layer2 and // localnet topology types. -func NewSecondaryNodeNetworkController(cnnci *CommonNodeNetworkControllerInfo, netInfo util.NetInfo, - vrfManager *vrfmanager.Controller, ruleManager *iprulemanager.Controller, defaultNetworkGateway Gateway) (*SecondaryNodeNetworkController, error) { +func NewSecondaryNodeNetworkController( + cnnci *CommonNodeNetworkControllerInfo, + netInfo util.NetInfo, + vrfManager *vrfmanager.Controller, + ruleManager *iprulemanager.Controller, + defaultNetworkGateway Gateway, +) (*SecondaryNodeNetworkController, error) { + snnc := &SecondaryNodeNetworkController{ BaseNodeNetworkController: BaseNodeNetworkController{ CommonNodeNetworkControllerInfo: *cnnci, - NetInfo: netInfo, + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), stopChan: make(chan struct{}), wg: &sync.WaitGroup{}, }, @@ -52,7 +58,7 @@ func NewSecondaryNodeNetworkController(cnnci *CommonNodeNetworkControllerInfo, n return nil, fmt.Errorf("error retrieving network id for network %s: %v", netInfo.GetNetworkName(), err) } - snnc.gateway, err = NewUserDefinedNetworkGateway(snnc.NetInfo, networkID, node, + snnc.gateway, err = NewUserDefinedNetworkGateway(snnc.GetNetInfo(), networkID, node, snnc.watchFactory.NodeCoreInformer().Lister(), snnc.Kube, vrfManager, ruleManager, defaultNetworkGateway) if err != nil { return nil, fmt.Errorf("error creating UDN gateway for network %s: %v", netInfo.GetNetworkName(), err) @@ -108,10 +114,19 @@ func (oc *SecondaryNodeNetworkController) getNetworkID() (int, error) { if err != nil { return util.InvalidID, err } - *oc.networkID, err = util.GetNetworkID(nodes, oc.NetInfo) + *oc.networkID, err = util.GetNetworkID(nodes, oc.GetNetInfo()) if err != nil { return util.InvalidID, err } } return *oc.networkID, nil } + +func (oc *SecondaryNodeNetworkController) Reconcile(netInfo util.NetInfo) error { + // reconcile network information, point of no return + err := util.ReconcileNetInfo(oc.ReconcilableNetInfo, netInfo) + if err != nil { + klog.Errorf("Failed to reconcile network %s: %v", oc.GetNetworkName(), err) + } + return nil +} diff --git a/go-controller/pkg/node/secondary_node_network_controller_test.go b/go-controller/pkg/node/secondary_node_network_controller_test.go index 914140e4fe..9151875fd1 100644 --- a/go-controller/pkg/node/secondary_node_network_controller_test.go +++ b/go-controller/pkg/node/secondary_node_network_controller_test.go @@ -70,8 +70,9 @@ var _ = Describe("SecondaryNodeNetworkController", func() { Expect(err).NotTo(HaveOccurred()) Expect(controller.watchFactory.Start()).To(Succeed()) - controller.NetInfo, err = util.ParseNADInfo(nad) + netInfo, err := util.ParseNADInfo(nad) Expect(err).NotTo(HaveOccurred()) + controller.ReconcilableNetInfo = util.NewReconcilableNetInfo(netInfo) networkID, err := controller.getNetworkID() Expect(err).ToNot(HaveOccurred()) @@ -95,8 +96,9 @@ var _ = Describe("SecondaryNodeNetworkController", func() { Expect(err).NotTo(HaveOccurred()) Expect(controller.watchFactory.Start()).To(Succeed()) - controller.NetInfo, err = util.ParseNADInfo(nad) + netInfo, err := util.ParseNADInfo(nad) Expect(err).NotTo(HaveOccurred()) + controller.ReconcilableNetInfo = util.NewReconcilableNetInfo(netInfo) networkID, err := controller.getNetworkID() Expect(err).To(HaveOccurred()) diff --git a/go-controller/pkg/node/udn_isolation.go b/go-controller/pkg/node/udn_isolation.go index a6817bd377..83ee56fbca 100644 --- a/go-controller/pkg/node/udn_isolation.go +++ b/go-controller/pkg/node/udn_isolation.go @@ -24,7 +24,6 @@ import ( "sigs.k8s.io/knftables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" nodenft "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -50,7 +49,6 @@ type UDNHostIsolationManager struct { ipv4, ipv6 bool podController controller.Controller podLister corelisters.PodLister - nadController *nad.NetAttachDefinitionController kubeletCgroupPath string udnPodIPsv4 *nftPodElementsSet @@ -63,11 +61,9 @@ type UDNHostIsolationManager struct { udnOpenPortsICMPv6 *nftPodElementsSet } -func NewUDNHostIsolationManager(ipv4, ipv6 bool, podInformer coreinformers.PodInformer, - nadController *nad.NetAttachDefinitionController) *UDNHostIsolationManager { +func NewUDNHostIsolationManager(ipv4, ipv6 bool, podInformer coreinformers.PodInformer) *UDNHostIsolationManager { m := &UDNHostIsolationManager{ podLister: podInformer.Lister(), - nadController: nadController, ipv4: ipv4, ipv6: ipv6, udnPodIPsv4: newNFTPodElementsSet(nftablesUDNPodIPsv4, false), @@ -91,6 +87,7 @@ func NewUDNHostIsolationManager(ipv4, ipv6 bool, podInformer coreinformers.PodIn // Start must be called on node setup. func (m *UDNHostIsolationManager) Start(ctx context.Context) error { + klog.Infof("Starting UDN host isolation manager") // find kubelet cgroup path. // kind cluster uses "kubelet.slice/kubelet.service", while OCP cluster uses "system.slice/kubelet.service". // as long as ovn-k node is running as a privileged container, we can access the host cgroup directory. @@ -367,7 +364,10 @@ func (m *UDNHostIsolationManager) podInitialSync() error { // ignore openPorts parse error in initial sync pi, _, err := m.getPodInfo(podKey, pod) if err != nil { - return err + // don't fail because of one pod error on initial sync as it may cause crashloop. + // expect pod event to come later with correct/updated annotations. + klog.Warningf("UDNHostIsolationManager failed to get pod info for pod %s/%s on initial sync: %v", pod.Name, pod.Namespace, err) + continue } if pi == nil { // this pod doesn't need to be updated @@ -456,6 +456,10 @@ func (m *UDNHostIsolationManager) getPodInfo(podKey string, pod *v1.Pod) (*podIn if pod == nil { return pi, nil, nil } + if util.PodWantsHostNetwork(pod) { + // host network pods can't be isolated by IP + return nil, nil, nil + } // only add pods with primary UDN primaryUDN, err := m.isPodPrimaryUDN(pod) if err != nil { diff --git a/go-controller/pkg/node/udn_isolation_test.go b/go-controller/pkg/node/udn_isolation_test.go index 35daee2ae9..ffb6f5fd4b 100644 --- a/go-controller/pkg/node/udn_isolation_test.go +++ b/go-controller/pkg/node/udn_isolation_test.go @@ -19,9 +19,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controller" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" nodenft "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -212,11 +210,10 @@ add set inet ovn-kubernetes %s { type %s ; } var _ = Describe("UDN Host isolation", func() { var ( - manager *UDNHostIsolationManager - nadController *networkAttachDefController.NetAttachDefinitionController - wf *factory.WatchFactory - fakeClient *util.OVNNodeClientset - nft *knftables.Fake + manager *UDNHostIsolationManager + wf *factory.WatchFactory + fakeClient *util.OVNNodeClientset + nft *knftables.Fake ) const ( @@ -282,16 +279,10 @@ add rule inet ovn-kubernetes udn-isolation ip6 daddr @udn-pod-default-ips-v6 dro wf, err = factory.NewNodeWatchFactory(fakeClient, "node1") Expect(err).NotTo(HaveOccurred()) - testNCM := &nad.FakeNetworkControllerManager{} - nadController, err = networkAttachDefController.NewNetAttachDefinitionController("test", testNCM, wf, nil) - Expect(err).NotTo(HaveOccurred()) - - manager = NewUDNHostIsolationManager(true, true, wf.PodCoreInformer(), nadController) + manager = NewUDNHostIsolationManager(true, true, wf.PodCoreInformer()) err = wf.Start() Expect(err).NotTo(HaveOccurred()) - err = nadController.Start() - Expect(err).NotTo(HaveOccurred()) // Copy manager.Start() sequence, but using fake nft and without running systemd tracker manager.kubeletCgroupPath = "kubelet.slice/kubelet.service" @@ -312,7 +303,6 @@ add rule inet ovn-kubernetes udn-isolation ip6 daddr @udn-pod-default-ips-v6 dro wf = nil manager = nil - nadController = nil }) AfterEach(func() { @@ -322,9 +312,36 @@ add rule inet ovn-kubernetes udn-isolation ip6 daddr @udn-pod-default-ips-v6 dro if manager != nil { manager.Stop() } - if nadController != nil { - nadController.Stop() + }) + + It("correctly handles host-network and not ready pods on initial sync", func() { + hostNetPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostnet", + UID: ktypes.UID("hostnet"), + Namespace: defaultNamespace, + }, + } + hostNetPod.Spec.HostNetwork = true + notReadyPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "notready", + UID: ktypes.UID("notready"), + Namespace: defaultNamespace, + }, } + + fakeClient = util.GetOVNClientset(hostNetPod, notReadyPod).GetNodeClientset() + var err error + wf, err = factory.NewNodeWatchFactory(fakeClient, "node1") + Expect(err).NotTo(HaveOccurred()) + manager = NewUDNHostIsolationManager(true, true, wf.PodCoreInformer()) + nft = nodenft.SetFakeNFTablesHelper() + manager.nft = nft + + Expect(wf.Start()).To(Succeed()) + Expect(manager.setupUDNIsolationFromHost()).To(Succeed()) + Expect(manager.podInitialSync()).To(Succeed()) }) It("correctly generates initial rules", func() { diff --git a/go-controller/pkg/ovn/admin_network_policy_test.go b/go-controller/pkg/ovn/admin_network_policy_test.go index 1716ebfdcb..bb2a199bab 100644 --- a/go-controller/pkg/ovn/admin_network_policy_test.go +++ b/go-controller/pkg/ovn/admin_network_policy_test.go @@ -7,6 +7,7 @@ import ( "strings" "time" + nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" @@ -15,6 +16,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" anpovn "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/admin_network_policy" + ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -74,11 +76,15 @@ func getANPRulePriority(basePriority, index int32) int32 { } func getDefaultPGForANPSubject(anpName string, portUUIDs []string, acls []*nbdb.ACL, banp bool) *nbdb.PortGroup { + return getUDNPGForANPSubject(anpName, portUUIDs, acls, banp, types.DefaultNetworkName) +} + +func getUDNPGForANPSubject(anpName string, portUUIDs []string, acls []*nbdb.ACL, banp bool, networkName string) *nbdb.PortGroup { lsps := []*nbdb.LogicalSwitchPort{} for _, uuid := range portUUIDs { lsps = append(lsps, &nbdb.LogicalSwitchPort{UUID: uuid}) } - pgDbIDs := anpovn.GetANPPortGroupDbIDs(anpName, banp, DefaultNetworkControllerName) + pgDbIDs := anpovn.GetANPPortGroupDbIDs(anpName, banp, networkName) pg := libovsdbutil.BuildPortGroup( pgDbIDs, @@ -91,7 +97,7 @@ func getDefaultPGForANPSubject(anpName string, portUUIDs []string, acls []*nbdb. func getANPGressACL(action, anpName, direction string, rulePriority int32, ruleIndex int32, ports *[]anpapi.AdminNetworkPolicyPort, - namedPorts map[string][]libovsdbutil.NamedNetworkPolicyPort, banp bool) []*nbdb.ACL { + namedPorts map[string][]libovsdbutil.NamedNetworkPolicyPort, banp bool, networkName string) []*nbdb.ACL { retACLs := []*nbdb.ACL{} // we are not using BuildACL and instead manually building it on purpose so that the code path for BuildACL is also tested acl := nbdb.ACL{} @@ -105,24 +111,24 @@ func getANPGressACL(action, anpName, direction string, rulePriority int32, acl.Tier = types.DefaultBANPACLTier } acl.ExternalIDs = map[string]string{ - libovsdbops.OwnerControllerKey.String(): DefaultNetworkControllerName, + libovsdbops.OwnerControllerKey.String(): networkName, libovsdbops.ObjectNameKey.String(): anpName, libovsdbops.GressIdxKey.String(): fmt.Sprintf("%d", ruleIndex), libovsdbops.PolicyDirectionKey.String(): direction, libovsdbops.PortPolicyProtocolKey.String(): "None", libovsdbops.OwnerTypeKey.String(): "AdminNetworkPolicy", - libovsdbops.PrimaryIDKey.String(): fmt.Sprintf("%s:AdminNetworkPolicy:%s:%s:%d:None", DefaultNetworkControllerName, anpName, direction, ruleIndex), + libovsdbops.PrimaryIDKey.String(): fmt.Sprintf("%s:AdminNetworkPolicy:%s:%s:%d:None", networkName, anpName, direction, ruleIndex), } acl.Name = utilpointer.String(fmt.Sprintf("ANP:%s:%s:%d", anpName, direction, ruleIndex)) // tests logic for GetACLName if banp { acl.ExternalIDs[libovsdbops.OwnerTypeKey.String()] = "BaselineAdminNetworkPolicy" acl.ExternalIDs[libovsdbops.PrimaryIDKey.String()] = fmt.Sprintf("%s:BaselineAdminNetworkPolicy:%s:%s:%d:None", - DefaultNetworkControllerName, anpName, direction, ruleIndex) + networkName, anpName, direction, ruleIndex) acl.Name = utilpointer.String(fmt.Sprintf("BANP:%s:%s:%d", anpName, direction, ruleIndex)) // tests logic for GetACLName } acl.UUID = fmt.Sprintf("%s_%s_%d-%f-UUID", anpName, direction, ruleIndex, rand.Float64()) // determine ACL match - pgName := libovsdbutil.GetPortGroupName(anpovn.GetANPPortGroupDbIDs(anpName, banp, DefaultNetworkControllerName)) + pgName := libovsdbutil.GetPortGroupName(anpovn.GetANPPortGroupDbIDs(anpName, banp, networkName)) var lPortMatch, l3Match, matchDirection, match string if direction == string(libovsdbutil.ACLIngress) { acl.Direction = nbdb.ACLDirectionToLport @@ -137,7 +143,7 @@ func getANPGressACL(action, anpName, direction string, rulePriority int32, matchDirection = "dst" } asIndex := anpovn.GetANPPeerAddrSetDbIDs(anpName, direction, fmt.Sprintf("%d", ruleIndex), - DefaultNetworkControllerName, banp) + networkName, banp) asv4, asv6 := addressset.GetHashNamesForAS(asIndex) if config.IPv4Mode && config.IPv6Mode { l3Match = fmt.Sprintf("((ip4.%s == $%s || ip6.%s == $%s))", matchDirection, asv4, matchDirection, asv6) @@ -180,10 +186,10 @@ func getANPGressACL(action, anpName, direction string, rulePriority int32, aclCopy.ExternalIDs[libovsdbops.PortPolicyProtocolKey.String()] = protocol aclCopy.Match = match aclCopy.ExternalIDs[libovsdbops.PrimaryIDKey.String()] = fmt.Sprintf("%s:AdminNetworkPolicy:%s:%s:%d:%s", - DefaultNetworkControllerName, anpName, direction, ruleIndex, protocol) + networkName, anpName, direction, ruleIndex, protocol) if banp { aclCopy.ExternalIDs[libovsdbops.PrimaryIDKey.String()] = fmt.Sprintf("%s:BaselineAdminNetworkPolicy:%s:%s:%d:%s", - DefaultNetworkControllerName, anpName, direction, ruleIndex, protocol) + networkName, anpName, direction, ruleIndex, protocol) } aclCopy.UUID = fmt.Sprintf("%s_%s_%d.%s-%f-UUID", anpName, direction, ruleIndex, protocol, rand.Float64()) retACLs = append(retACLs, &aclCopy) @@ -203,10 +209,10 @@ func getANPGressACL(action, anpName, direction string, rulePriority int32, aclCopy.ExternalIDs[libovsdbops.PortPolicyProtocolKey.String()] = protocol + "-namedPort" aclCopy.Match = match aclCopy.ExternalIDs[libovsdbops.PrimaryIDKey.String()] = fmt.Sprintf("%s:AdminNetworkPolicy:%s:%s:%d:%s", - DefaultNetworkControllerName, anpName, direction, ruleIndex, protocol+"-namedPort") + networkName, anpName, direction, ruleIndex, protocol+"-namedPort") if banp { aclCopy.ExternalIDs[libovsdbops.PrimaryIDKey.String()] = fmt.Sprintf("%s:BaselineAdminNetworkPolicy:%s:%s:%d:%s", - DefaultNetworkControllerName, anpName, direction, ruleIndex, protocol+"-namedPort") + networkName, anpName, direction, ruleIndex, protocol+"-namedPort") } retACLs = append(retACLs, &aclCopy) } @@ -218,7 +224,7 @@ func getACLsForANPRulesWithNamedPorts(anp *anpapi.AdminNetworkPolicy, namedIPort ovnBaseANPPriority := getBaseRulePriority(anp.Spec.Priority) for i, ingress := range anp.Spec.Ingress { acls := getANPGressACL(anpovn.GetACLActionForANPRule(ingress.Action), anp.Name, string(libovsdbutil.ACLIngress), - getANPRulePriority(ovnBaseANPPriority, int32(i)), int32(i), ingress.Ports, namedIPorts, false) + getANPRulePriority(ovnBaseANPPriority, int32(i)), int32(i), ingress.Ports, namedIPorts, false, DefaultNetworkControllerName) aclResults = append(aclResults, acls...) } for i, egress := range anp.Spec.Egress { @@ -227,7 +233,7 @@ func getACLsForANPRulesWithNamedPorts(anp *anpapi.AdminNetworkPolicy, namedIPort namedEPorts = nil } acls := getANPGressACL(anpovn.GetACLActionForANPRule(egress.Action), anp.Name, string(libovsdbutil.ACLEgress), - getANPRulePriority(ovnBaseANPPriority, int32(i)), int32(i), egress.Ports, namedEPorts, false) + getANPRulePriority(ovnBaseANPPriority, int32(i)), int32(i), egress.Ports, namedEPorts, false, DefaultNetworkControllerName) aclResults = append(aclResults, acls...) } return aclResults @@ -238,17 +244,52 @@ func getACLsForANPRules(anp *anpapi.AdminNetworkPolicy) []*nbdb.ACL { ovnBaseANPPriority := getBaseRulePriority(anp.Spec.Priority) for i, ingress := range anp.Spec.Ingress { acls := getANPGressACL(anpovn.GetACLActionForANPRule(ingress.Action), anp.Name, string(libovsdbutil.ACLIngress), - getANPRulePriority(ovnBaseANPPriority, int32(i)), int32(i), ingress.Ports, nil, false) + getANPRulePriority(ovnBaseANPPriority, int32(i)), int32(i), ingress.Ports, nil, false, DefaultNetworkControllerName) aclResults = append(aclResults, acls...) } for i, egress := range anp.Spec.Egress { acls := getANPGressACL(anpovn.GetACLActionForANPRule(egress.Action), anp.Name, string(libovsdbutil.ACLEgress), - getANPRulePriority(ovnBaseANPPriority, int32(i)), int32(i), egress.Ports, nil, false) + getANPRulePriority(ovnBaseANPPriority, int32(i)), int32(i), egress.Ports, nil, false, DefaultNetworkControllerName) aclResults = append(aclResults, acls...) } return aclResults } +// pass the networkName to return ACL representations for that network +func getUDNACLsForANPRules(anp *anpapi.AdminNetworkPolicy, networkName string) []*nbdb.ACL { + aclResults := []*nbdb.ACL{} + ovnBaseANPPriority := getBaseRulePriority(anp.Spec.Priority) + for i, ingress := range anp.Spec.Ingress { + acls := getANPGressACL(anpovn.GetACLActionForANPRule(ingress.Action), anp.Name, string(libovsdbutil.ACLIngress), + getANPRulePriority(ovnBaseANPPriority, int32(i)), int32(i), ingress.Ports, nil, false, networkName) + aclResults = append(aclResults, acls...) + } + for i, egress := range anp.Spec.Egress { + acls := getANPGressACL(anpovn.GetACLActionForANPRule(egress.Action), anp.Name, string(libovsdbutil.ACLEgress), + getANPRulePriority(ovnBaseANPPriority, int32(i)), int32(i), egress.Ports, nil, false, networkName) + aclResults = append(aclResults, acls...) + } + return aclResults +} + +func getUDNPG() *nbdb.PortGroup { + pgIDs := libovsdbops.NewDbObjectIDs(libovsdbops.PortGroupUDN, DefaultNetworkControllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: "SecondaryPods", + }) + return &nbdb.PortGroup{ + Name: libovsdbutil.GetPortGroupName(pgIDs), + UUID: "UDN-Isolation-PG-UUID", + } +} + +func buildUDNANPAddressSets(anp *anpapi.AdminNetworkPolicy, index int32, ips []string, + gressPrefix libovsdbutil.ACLDirection, networkName string) (*nbdb.AddressSet, *nbdb.AddressSet) { + asIndex := anpovn.GetANPPeerAddrSetDbIDs(anp.Name, string(gressPrefix), + fmt.Sprintf("%d", index), networkName, false) + return addressset.GetTestDbAddrSets(asIndex, ips) +} + func buildANPAddressSets(anp *anpapi.AdminNetworkPolicy, index int32, ips []string, gressPrefix libovsdbutil.ACLDirection) (*nbdb.AddressSet, *nbdb.AddressSet) { asIndex := anpovn.GetANPPeerAddrSetDbIDs(anp.Name, string(gressPrefix), fmt.Sprintf("%d", index), DefaultNetworkControllerName, false) @@ -257,8 +298,16 @@ func buildANPAddressSets(anp *anpapi.AdminNetworkPolicy, index int32, ips []stri var _ = ginkgo.Describe("OVN ANP Operations", func() { var ( - app *cli.App - fakeOVN *FakeOVN + app *cli.App + fakeOVN *FakeOVN + udnACLs []*nbdb.ACL + udnPG, udnIsolationPG *nbdb.PortGroup + peerUDNASIngressRule0v4, peerUDNASIngressRule0v6 *nbdb.AddressSet + peerUDNASIngressRule1v4, peerUDNASIngressRule1v6 *nbdb.AddressSet + peerUDNASIngressRule2v4, peerUDNASIngressRule2v6 *nbdb.AddressSet + peerUDNASEgressRule0v4, peerUDNASEgressRule0v6 *nbdb.AddressSet + peerUDNASEgressRule1v4, peerUDNASEgressRule1v6 *nbdb.AddressSet + peerUDNASEgressRule2v4, peerUDNASEgressRule2v6 *nbdb.AddressSet ) const ( @@ -286,6 +335,7 @@ var _ = ginkgo.Describe("OVN ANP Operations", func() { node2IPv6Subnet string = "fe00:10:128:2::/64" node2transitIPv4 string = "100.88.0.3" node2transitIPv6 string = "fd97::3" + nadName string = "bluenad" ) ginkgo.BeforeEach(func() { @@ -294,6 +344,9 @@ var _ = ginkgo.Describe("OVN ANP Operations", func() { config.OVNKubernetesFeature.EnableAdminNetworkPolicy = true // IC true or false does not really effect this feature config.OVNKubernetesFeature.EnableInterconnect = true + // We have mix of tests for default network and UDN networks + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true app = cli.NewApp() app.Name = "test" @@ -307,635 +360,985 @@ var _ = ginkgo.Describe("OVN ANP Operations", func() { }) ginkgo.Context("On admin network policy changes", func() { - ginkgo.It("should create/update/delete address-sets, acls, port-groups correctly", func() { - app.Action = func(ctx *cli.Context) error { - anpNamespaceSubject := *newNamespaceWithLabels(anpSubjectNamespaceName, anpLabel) - anpNamespacePeer := *newNamespaceWithLabels(anpPeerNamespaceName, peerDenyLabel) - config.IPv4Mode = true - config.IPv6Mode = true - node1 := nodeFor(node1Name, "100.100.100.0", "fc00:f853:ccd:e793::1", "10.128.1.0/24", "fe00:10:128:1::/64", "", "") - - node1Switch := &nbdb.LogicalSwitch{ - Name: node1Name, - UUID: node1Name + "-UUID", - } - subjectNSASIPv4, subjectNSASIPv6 := buildNamespaceAddressSets(anpSubjectNamespaceName, []string{}) - peerNSASIPv4, peerNSASIPv6 := buildNamespaceAddressSets(anpPeerNamespaceName, []string{}) - dbSetup := libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - node1Switch, - subjectNSASIPv4, - subjectNSASIPv6, - peerNSASIPv4, - peerNSASIPv6, - }, - } - fakeOVN.startWithDBSetup(dbSetup, - &v1.NamespaceList{ - Items: []v1.Namespace{ - anpNamespaceSubject, - anpNamespacePeer, + ginkgo.DescribeTable("should create/update/delete address-sets, acls, port-groups correctly", + func(anpSubjectNetwork, anpPeerNetwork string, isUDNEnabled bool) { + app.Action = func(ctx *cli.Context) error { + anpNamespaceSubject := *newNamespaceWithLabels(anpSubjectNamespaceName, anpLabel) + anpNamespacePeer := *newNamespaceWithLabels(anpPeerNamespaceName, peerDenyLabel) + config.IPv4Mode = true + config.IPv6Mode = true + node1 := nodeFor(node1Name, "100.100.100.0", "fc00:f853:ccd:e793::1", "10.128.1.0/24", "fe00:10:128:1::/64", "", "") + + node1Switch := &nbdb.LogicalSwitch{ + Name: node1Name, + UUID: node1Name + "-UUID", + } + anpSubjectNetworkSwitch := node1Switch + subjectNSASIPv4, subjectNSASIPv6 := buildNamespaceAddressSets(anpSubjectNamespaceName, []string{}) + peerNSASIPv4, peerNSASIPv6 := buildNamespaceAddressSets(anpPeerNamespaceName, []string{}) + dbSetup := libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + node1Switch, + subjectNSASIPv4, + subjectNSASIPv6, + peerNSASIPv4, + peerNSASIPv6, }, - }, - &v1.NodeList{ - Items: []v1.Node{ - *node1, + } + k8sObjects := []runtime.Object{ + &v1.NamespaceList{ + Items: []v1.Namespace{ + anpNamespaceSubject, + anpNamespacePeer, + }, + }, + &v1.NodeList{ + Items: []v1.Node{ + *node1, + }, }, - }, - ) - - fakeOVN.controller.zone = node1Name // ensure we set the controller's zone as the node's zone - err := fakeOVN.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOVN.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fakeOVN.InitAndRunANPController() - fakeOVN.fakeClient.ANPClient.(*anpfake.Clientset).PrependReactor("update", "adminnetworkpolicies", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { - update := action.(clienttesting.UpdateAction) - // Since fake client (NewSimpleClientset) does not differentiate between - // an update and updatestatus, updatestatus in tests updates the spec as - // well causing race conditions. Thus adding a hack here to ensure update - // status is caught and processed by the reactor while update spec is - // delegated to the main code for handling - if action.GetSubresource() == "status" { - klog.Infof("Got an update status action for %v", update.GetObject()) - return true, update.GetObject(), nil } - klog.Infof("Got an update spec action for %v", update.GetObject()) - return false, update.GetObject(), nil - }) + if isUDNEnabled { + if anpSubjectNetwork != types.DefaultNetworkName { + gNAD := ovntest.GenerateNAD(anpSubjectNetwork, nadName, anpNamespaceSubject.Name, + types.Layer3Topology, "100.128.0.0/16/24,2010:100:200::0/60/64", types.NetworkRolePrimary) + k8sObjects = append(k8sObjects, + &nadapi.NetworkAttachmentDefinitionList{ + Items: []nadapi.NetworkAttachmentDefinition{ + *gNAD, + }, + }, + ) + anpSubjectNetworkSwitch = &nbdb.LogicalSwitch{ + Name: anpSubjectNetwork + "_" + node1Name, + UUID: anpSubjectNetwork + "_" + node1Name + "-UUID", + } + // we don't call WatchNodes for ANP tests; so the UDN isolation PG won't be + // created; hence we need to manually add it during setup phase so that pod + // creation on default network succeeds + udnIsolationPG = getUDNPG() + dbSetup.NBData = append(dbSetup.NBData, anpSubjectNetworkSwitch, udnIsolationPG) + } + } - ginkgo.By("1. creating an admin network policy with 0 rules; check if port group is created for the subject") - anpSubject := newANPSubjectObject( - &metav1.LabelSelector{ - MatchLabels: anpLabel, - }, - nil, - ) - anp := newANPObject("harry-potter", 5, anpSubject, []anpapi.AdminNetworkPolicyIngressRule{}, []anpapi.AdminNetworkPolicyEgressRule{}) - anp.ResourceVersion = "1" - anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Create(context.TODO(), anp, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pg := getDefaultPGForANPSubject(anp.Name, nil, nil, false) - expectedDatabaseState := []libovsdbtest.TestData{node1Switch, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6, pg} - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + fakeOVN.startWithDBSetup(dbSetup, k8sObjects...) + if isUDNEnabled { + gomega.Expect(fakeOVN.networkManager.Start()).Should(gomega.Succeed()) + defer fakeOVN.networkManager.Stop() + } + fakeOVN.controller.zone = node1Name // ensure we set the controller's zone as the node's zone + err := fakeOVN.controller.WatchNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOVN.controller.WatchPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("2. creating a pod that will act as subject of admin network policy; check if lsp is added to port-group after retries") - t := newTPod( - node1Name, - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - anpSubjectPodName, - anpPodV4IP, - anpPodMAC, - anpSubjectNamespaceName, - ) - t.portName = util.GetLogicalPortName(t.namespace, t.podName) - t.populateLogicalSwitchCache(fakeOVN) - anpSubjectPod := *newPod(anpSubjectNamespaceName, anpSubjectPodName, node1Name, anpPodV4IP) - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Create(context.TODO(), &anpSubjectPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // The pod takes some time to get created so the first add pod event for ANP does nothing as it waits for LSP to be created - this triggers a retry. - // Next when the pod is scheduled and IP is allocated and LSP is created we get an update event. - // When ANP controller receives the pod update event, it will not trigger an update in unit testing since ResourceVersion and pod.status.Running - // is not updated by the test, hence we need to trigger a manual update in the unit test framework that is accompanied by label update. - // NOTE: This is not an issue in real deployments with actual kapi server; its just a potential flake in the UT framework that we need to protect against. - time.Sleep(time.Second) - anpSubjectPod.ResourceVersion = "5" - // - this is to trigger an actual update technically speaking the pod already matches the namespace of the subject - anpSubjectPod.Labels["rv"] = "resourceVersionUTHack" - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Update(context.TODO(), &anpSubjectPod, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - subjectNSASIPv4, subjectNSASIPv6 = buildNamespaceAddressSets(anpSubjectNamespaceName, []string{t.podIP}) - pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, nil, false) - expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t}, []string{node1Name})...) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + fakeOVN.InitAndRunANPController() + fakeOVN.fakeClient.ANPClient.(*anpfake.Clientset).PrependReactor("update", "adminnetworkpolicies", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + update := action.(clienttesting.UpdateAction) + // Since fake client (NewSimpleClientset) does not differentiate between + // an update and updatestatus, updatestatus in tests updates the spec as + // well causing race conditions. Thus adding a hack here to ensure update + // status is caught and processed by the reactor while update spec is + // delegated to the main code for handling + if action.GetSubresource() == "status" { + klog.Infof("Got an update status action for %v", update.GetObject()) + return true, update.GetObject(), nil + } + klog.Infof("Got an update spec action for %v", update.GetObject()) + return false, update.GetObject(), nil + }) + + ginkgo.By("1. creating an admin network policy with 0 rules; check if port group is created for the subject") + anpSubject := newANPSubjectObject( + &metav1.LabelSelector{ + MatchLabels: anpLabel, + }, + nil, + ) + anp := newANPObject("harry-potter", 5, anpSubject, []anpapi.AdminNetworkPolicyIngressRule{}, []anpapi.AdminNetworkPolicyEgressRule{}) + anp.ResourceVersion = "1" + anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Create(context.TODO(), anp, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pg := getDefaultPGForANPSubject(anp.Name, nil, nil, false) + expectedDatabaseState := []libovsdbtest.TestData{pg, node1Switch, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + if isUDNEnabled { + expectedDatabaseState = append(expectedDatabaseState, anpSubjectNetworkSwitch, udnIsolationPG) + // since there is no namespace in default network that matches this ANP; replace this with blue P-UDN's representation + expectedDatabaseState[0] = getUDNPGForANPSubject(anp.Name, nil, nil, false, anpSubjectNetwork) + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("2. creating a pod that will act as subject of admin network policy; check if lsp is added to port-group after retries") + t := newTPod( + node1Name, + "10.128.1.0/24", + "10.128.1.2", + "10.128.1.1", + anpSubjectPodName, + anpPodV4IP, + anpPodMAC, + anpSubjectNamespaceName, + ) + copyT := t + t.portName = util.GetLogicalPortName(t.namespace, t.podName) + t.populateLogicalSwitchCache(fakeOVN) + if isUDNEnabled { + secondaryNetController, ok := fakeOVN.secondaryControllers[anpSubjectNetwork] + gomega.Expect(ok).To(gomega.BeTrue()) + t.addNetwork(anpSubjectNetwork, util.GetNADName(anpSubjectNamespaceName, nadName), + "100.128.0.0/24", "100.128.0.2", "", "100.128.0.3", anpPodMAC, types.NetworkRolePrimary, 1, nil) + gomega.Expect(secondaryNetController.bnc.WatchNamespaces()).To(gomega.Succeed()) + gomega.Expect(secondaryNetController.bnc.WatchPods()).To(gomega.Succeed()) + t.populateSecondaryNetworkLogicalSwitchCache(fakeOVN, secondaryNetController) + } + anpSubjectPod := *newPod(anpSubjectNamespaceName, anpSubjectPodName, node1Name, anpPodV4IP) + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Create(context.TODO(), &anpSubjectPod, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // The pod takes some time to get created so the first add pod event for ANP does nothing as it waits for LSP to be created - this triggers a retry. + // Next when the pod is scheduled and IP is allocated and LSP is created we get an update event. + // When ANP controller receives the pod update event, it will not trigger an update in unit testing since ResourceVersion and pod.status.Running + // is not updated by the test, hence we need to trigger a manual update in the unit test framework that is accompanied by label update. + // NOTE: This is not an issue in real deployments with actual kapi server; its just a potential flake in the UT framework that we need to protect against. + time.Sleep(time.Second) + anpSubjectPod.ResourceVersion = "5" + // - this is to trigger an actual update technically speaking the pod already matches the namespace of the subject + anpSubjectPod.Labels["rv"] = "resourceVersionUTHack" + if isUDNEnabled { + // unforunately we rely on pod annotations to determine the podIPs for UDN pods since pod status doesn't have them + // and hence we need to patch the pod with the annotation of the P-UDN to simulate that + anpSubjectPod.Annotations = map[string]string{ + "k8s.ovn.org/pod-networks": fmt.Sprintf(`{%q:{"ip_addresses":["%s/24"],"mac_address":"%s"}}`, + util.GetNADName(anpSubjectNamespaceName, nadName), "100.128.0.3", + util.IPAddrToHWAddr(ovntest.MustParseIP("100.128.0.3")).String()), + } + } + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Update(context.TODO(), &anpSubjectPod, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + subjectNSASIPv4, subjectNSASIPv6 = buildNamespaceAddressSets(anpSubjectNamespaceName, []string{t.podIP}) + pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, nil, false) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t}, []string{node1Name})...) + if isUDNEnabled { + netInfo := fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork) + // we need to update the pod's UDN copy so that we can fetch the appropriate items for building the expected database + // for the subject pod's blue P-UDN + copyT.podIP = "100.128.0.3" + copyT.podMAC = util.IPAddrToHWAddr(ovntest.MustParseIP("100.128.0.3")).String() + copyT.portName = util.GetSecondaryNetworkLogicalPortName(anpSubjectNamespaceName, anpSubjectPodName, netInfo.GetNADs()[0]) + copyT.portUUID = copyT.portName + "-UUID" + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + netInfo, []testPod{copyT}, []string{node1Name})...) + udnIsolationPG.Ports = []string{t.portUUID} // add the default network port for this pod + expectedDatabaseState = append(expectedDatabaseState, udnIsolationPG) + // add portgroup with LSP of the pod in the bue P-UDN and address-sets of the blue P-UDN + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{copyT.podIP}) + udnPG = getUDNPGForANPSubject(anp.Name, []string{copyT.portUUID}, nil, false, anpSubjectNetwork) + expectedDatabaseState = append(expectedDatabaseState, + subjectUDNASIPv4, subjectUDNASIPv6, udnPG) + expectedDatabaseState = expectedDatabaseState[1:] // there is no representation for default network yet. + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("3. update the ANP by adding one ingress rule; check if acl is created and added on the port-group") - ingressRules := []anpapi.AdminNetworkPolicyIngressRule{ - { - Name: "deny-traffic-from-slytherin-to-gryffindor", - Action: anpapi.AdminNetworkPolicyRuleActionDeny, - From: []anpapi.AdminNetworkPolicyIngressPeer{ - { - Namespaces: &metav1.LabelSelector{ - MatchLabels: peerDenyLabel, + ginkgo.By("3. update the ANP by adding one ingress rule; check if acl is created and added on the port-group") + ingressRules := []anpapi.AdminNetworkPolicyIngressRule{ + { + Name: "deny-traffic-from-slytherin-to-gryffindor", + Action: anpapi.AdminNetworkPolicyRuleActionDeny, + From: []anpapi.AdminNetworkPolicyIngressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: peerDenyLabel, + }, }, }, }, - }, - } - anp = newANPObject("harry-potter", 5, anpSubject, ingressRules, []anpapi.AdminNetworkPolicyEgressRule{}) - anp.ResourceVersion = "2" - anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - acls := getACLsForANPRules(anp) - pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, acls, false) - expectedDatabaseState[0] = pg // acl should be added to the port group - for _, acl := range acls { - acl := acl - expectedDatabaseState = append(expectedDatabaseState, acl) - } - peerASIngressRule0v4, peerASIngressRule0v6 := buildANPAddressSets(anp, 0, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v6) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - - ginkgo.By("4. creating a pod that will act as peer of admin network policy; check if IP is added to address-set after retries") - t2 := newTPod( - node1Name, - "10.128.1.1/24", - "10.128.1.2", - "10.128.1.1", - anpPeerPodName, - anpPodV4IP2, - anpPodMAC2, - anpPeerNamespaceName, - ) - t2.portName = util.GetLogicalPortName(t2.namespace, t2.podName) - t2.populateLogicalSwitchCache(fakeOVN) - anpPeerPod := *newPod(anpPeerNamespaceName, anpPeerPodName, node1Name, anpPodV4IP2) - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Create(context.TODO(), &anpPeerPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // The pod takes some time to get created so the first add pod event for ANP does nothing as it waits for LSP to be created - this triggers a retry. - // Next when the pod is scheduled and IP is allocated and LSP is created we get an update event. - // When ANP controller receives the pod update event, it will not trigger an update in unit testing since ResourceVersion and pod.status.Running - // is not updated by the test, hence we need to trigger a manual update in the unit test framework that is accompanied by label update. - // NOTE: This is not an issue in real deployments with actual kapi server; its just a potential flake in the UT framework that we need to protect against. - time.Sleep(time.Second) - anpPeerPod.ResourceVersion = "5" - // - this is to trigger an actual update technically speaking the pod already matches the namespace of the subject - anpPeerPod.Labels["rv"] = "resourceVersionUTHack" - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Update(context.TODO(), &anpPeerPod, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - peerNSASIPv4, peerNSASIPv6 = buildNamespaceAddressSets(anpPeerNamespaceName, []string{t2.podIP}) - expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} - for _, acl := range acls { - acl := acl - expectedDatabaseState = append(expectedDatabaseState, acl) - } - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) - peerASIngressRule0v4, peerASIngressRule0v6 = buildANPAddressSets(anp, 0, []string{t2.podIP}, libovsdbutil.ACLIngress) // podIP should be added to v4 address-set - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v6) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + } + anp = newANPObject("harry-potter", 5, anpSubject, ingressRules, []anpapi.AdminNetworkPolicyEgressRule{}) + anp.ResourceVersion = "2" + anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + acls := getACLsForANPRules(anp) + pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, acls, false) + if isUDNEnabled { + // fill in the blue network's representation for this ANP + udnACLs = getUDNACLsForANPRules(anp, anpSubjectNetwork) + udnPG = getUDNPGForANPSubject(anp.Name, []string{copyT.portUUID}, udnACLs, false, anpSubjectNetwork) + expectedDatabaseState[len(expectedDatabaseState)-1] = udnPG + for _, acl := range udnACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + peerUDNASIngressRule0v4, peerUDNASIngressRule0v6 = buildUDNANPAddressSets(anp, 0, []string{}, libovsdbutil.ACLIngress, anpSubjectNetwork) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASIngressRule0v4, peerUDNASIngressRule0v6) + // if pod is UDN then its not part of default network PG + pg.Ports = nil + // since the peer namespace of this ANP matches default network; we will have an empty representation PG in default network + expectedDatabaseState = append(expectedDatabaseState, pg) + } else { + expectedDatabaseState[0] = pg + } + for _, acl := range acls { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + peerASIngressRule0v4, peerASIngressRule0v6 := buildANPAddressSets(anp, 0, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4, peerASIngressRule0v6) + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("4. creating a pod that will act as peer of admin network policy; check if IP is added to address-set after retries") + t2 := newTPod( + node1Name, + "10.128.1.1/24", + "10.128.1.2", + "10.128.1.1", + anpPeerPodName, + anpPodV4IP2, + anpPodMAC2, + anpPeerNamespaceName, + ) + t2.portName = util.GetLogicalPortName(t2.namespace, t2.podName) + t2.populateLogicalSwitchCache(fakeOVN) + anpPeerPod := *newPod(anpPeerNamespaceName, anpPeerPodName, node1Name, anpPodV4IP2) + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Create(context.TODO(), &anpPeerPod, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // The pod takes some time to get created so the first add pod event for ANP does nothing as it waits for LSP to be created - this triggers a retry. + // Next when the pod is scheduled and IP is allocated and LSP is created we get an update event. + // When ANP controller receives the pod update event, it will not trigger an update in unit testing since ResourceVersion and pod.status.Running + // is not updated by the test, hence we need to trigger a manual update in the unit test framework that is accompanied by label update. + // NOTE: This is not an issue in real deployments with actual kapi server; its just a potential flake in the UT framework that we need to protect against. + time.Sleep(time.Second) + anpPeerPod.ResourceVersion = "5" + // - this is to trigger an actual update technically speaking the pod already matches the namespace of the subject + anpPeerPod.Labels["rv"] = "resourceVersionUTHack" + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Update(context.TODO(), &anpPeerPod, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + peerNSASIPv4, peerNSASIPv6 = buildNamespaceAddressSets(anpPeerNamespaceName, []string{t2.podIP}) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + for _, acl := range acls { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + peerASIngressRule0v4, peerASIngressRule0v6 = buildANPAddressSets(anp, 0, []string{t2.podIP}, libovsdbutil.ACLIngress) // podIP should be added to v4 address-set + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v6) + if isUDNEnabled { + // fill in the blue network's representation for this ANP + expectedDatabaseState = append(expectedDatabaseState, udnPG) + for _, acl := range udnACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + //peerUDNASIngressRule0v4, peerUDNASIngressRule0v6 := buildUDNANPAddressSets(anp, 0, []string{}, libovsdbutil.ACLIngress, anpSubjectNetwork) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASIngressRule0v4, peerUDNASIngressRule0v6) + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork), []testPod{copyT}, []string{node1Name})...) + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{copyT.podIP}) + expectedDatabaseState = append(expectedDatabaseState, + subjectUDNASIPv4, subjectUDNASIPv6, udnIsolationPG) + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("5. update the ANP by adding two more ingress rules with allow and pass actions; check if AS-es are created + acls are created and added on the port-group") - ingressRules = append(ingressRules, - anpapi.AdminNetworkPolicyIngressRule{ - Name: "allow-traffic-from-hufflepuff-to-gryffindor", - Action: anpapi.AdminNetworkPolicyRuleActionAllow, - From: []anpapi.AdminNetworkPolicyIngressPeer{ - { - Pods: &anpapi.NamespacedPod{ // test different kind of peer expression - NamespaceSelector: metav1.LabelSelector{ - MatchLabels: peerAllowLabel, - }, - PodSelector: metav1.LabelSelector{ - MatchLabels: peerAllowLabel, + ginkgo.By("5. update the ANP by adding two more ingress rules with allow and pass actions; check if AS-es are created + acls are created and added on the port-group") + ingressRules = append(ingressRules, + anpapi.AdminNetworkPolicyIngressRule{ + Name: "allow-traffic-from-hufflepuff-to-gryffindor", + Action: anpapi.AdminNetworkPolicyRuleActionAllow, + From: []anpapi.AdminNetworkPolicyIngressPeer{ + { + Pods: &anpapi.NamespacedPod{ // test different kind of peer expression + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: peerAllowLabel, + }, + PodSelector: metav1.LabelSelector{ + MatchLabels: peerAllowLabel, + }, }, }, }, }, - }, - anpapi.AdminNetworkPolicyIngressRule{ - Name: "pass-traffic-from-ravenclaw-to-gryffindor", - Action: anpapi.AdminNetworkPolicyRuleActionPass, - From: []anpapi.AdminNetworkPolicyIngressPeer{ - { - Namespaces: &metav1.LabelSelector{ - MatchLabels: peerPassLabel, + anpapi.AdminNetworkPolicyIngressRule{ + Name: "pass-traffic-from-ravenclaw-to-gryffindor", + Action: anpapi.AdminNetworkPolicyRuleActionPass, + From: []anpapi.AdminNetworkPolicyIngressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: peerPassLabel, + }, }, }, - }, - Ports: &[]anpapi.AdminNetworkPolicyPort{ // test different ports combination - { - PortNumber: &anpapi.Port{ - Protocol: v1.ProtocolTCP, - Port: int32(12345), + Ports: &[]anpapi.AdminNetworkPolicyPort{ // test different ports combination + { + PortNumber: &anpapi.Port{ + Protocol: v1.ProtocolTCP, + Port: int32(12345), + }, }, - }, - { - PortNumber: &anpapi.Port{ - Protocol: v1.ProtocolUDP, - Port: int32(1235), + { + PortNumber: &anpapi.Port{ + Protocol: v1.ProtocolUDP, + Port: int32(1235), + }, }, - }, - { - PortNumber: &anpapi.Port{ - Protocol: v1.ProtocolSCTP, - Port: int32(1345), + { + PortNumber: &anpapi.Port{ + Protocol: v1.ProtocolSCTP, + Port: int32(1345), + }, }, }, }, - }, - ) - anp = newANPObject("harry-potter", 5, anpSubject, ingressRules, []anpapi.AdminNetworkPolicyEgressRule{}) - anp.ResourceVersion = "3" - anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - acls = getACLsForANPRules(anp) - pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, acls, false) - expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} - for _, acl := range acls { - acl := acl - expectedDatabaseState = append(expectedDatabaseState, acl) - } - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) - peerASIngressRule0v4, peerASIngressRule0v6 = buildANPAddressSets(anp, - 0, []string{t2.podIP}, libovsdbutil.ACLIngress) // podIP should be added to v4 - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v6) - peerASIngressRule1v4, peerASIngressRule1v6 := buildANPAddressSets(anp, - 1, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v6) - peerASIngressRule2v4, peerASIngressRule2v6 := buildANPAddressSets(anp, - 2, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet - - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule2v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule2v6) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + ) + anp = newANPObject("harry-potter", 5, anpSubject, ingressRules, []anpapi.AdminNetworkPolicyEgressRule{}) + anp.ResourceVersion = "3" + anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + acls = getACLsForANPRules(anp) + pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, acls, false) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + for _, acl := range acls { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + peerASIngressRule0v4, peerASIngressRule0v6 = buildANPAddressSets(anp, + 0, []string{t2.podIP}, libovsdbutil.ACLIngress) // podIP should be added to v4 + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v6) + peerASIngressRule1v4, peerASIngressRule1v6 := buildANPAddressSets(anp, + 1, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v4) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v6) + peerASIngressRule2v4, peerASIngressRule2v6 := buildANPAddressSets(anp, + 2, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet + + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule2v4) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule2v6) + + if isUDNEnabled { + // fill in the blue network's representation for this ANP + udnACLs = getUDNACLsForANPRules(anp, anpSubjectNetwork) + udnPG = getUDNPGForANPSubject(anp.Name, []string{copyT.portUUID}, udnACLs, false, anpSubjectNetwork) + expectedDatabaseState = append(expectedDatabaseState, udnPG) + for _, acl := range udnACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + // peer address-sets will be empty because peers are in default network not BLUE UDN + // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASIngressRule0v4, peerUDNASIngressRule0v6) + peerUDNASIngressRule1v4, peerUDNASIngressRule1v6 = buildUDNANPAddressSets( + anp, 1, []string{}, libovsdbutil.ACLIngress, anpSubjectNetwork) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASIngressRule1v4, peerUDNASIngressRule1v6) + peerUDNASIngressRule2v4, peerUDNASIngressRule2v6 = buildUDNANPAddressSets( + anp, 2, []string{}, libovsdbutil.ACLIngress, anpSubjectNetwork) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASIngressRule2v4, peerUDNASIngressRule2v6) + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork), []testPod{copyT}, []string{node1Name})...) + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{copyT.podIP}) + expectedDatabaseState = append(expectedDatabaseState, + subjectUDNASIPv4, subjectUDNASIPv6, udnIsolationPG) + pg.Ports = nil // since subject pod is part of blue UDN not default network + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("6. update the ANP by adding three egress rules with deny, allow and pass actions;" + - "check if acls are created and added on the port-group") - egressRules := []anpapi.AdminNetworkPolicyEgressRule{ - { - Name: "deny-traffic-to-slytherin-from-gryffindor", - Action: anpapi.AdminNetworkPolicyRuleActionDeny, - To: []anpapi.AdminNetworkPolicyEgressPeer{ - { - Namespaces: &metav1.LabelSelector{ - MatchLabels: peerDenyLabel, + ginkgo.By("6. update the ANP by adding three egress rules with deny, allow and pass actions;" + + "check if acls are created and added on the port-group") + egressRules := []anpapi.AdminNetworkPolicyEgressRule{ + { + Name: "deny-traffic-to-slytherin-from-gryffindor", + Action: anpapi.AdminNetworkPolicyRuleActionDeny, + To: []anpapi.AdminNetworkPolicyEgressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: peerDenyLabel, + }, }, }, }, - }, - { - Name: "allow-traffic-to-hufflepuff-from-gryffindor", - Action: anpapi.AdminNetworkPolicyRuleActionAllow, - To: []anpapi.AdminNetworkPolicyEgressPeer{ - { - Pods: &anpapi.NamespacedPod{ // test different kind of peer expression - NamespaceSelector: metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "house", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"slytherin", "hufflepuff"}, + { + Name: "allow-traffic-to-hufflepuff-from-gryffindor", + Action: anpapi.AdminNetworkPolicyRuleActionAllow, + To: []anpapi.AdminNetworkPolicyEgressPeer{ + { + Pods: &anpapi.NamespacedPod{ // test different kind of peer expression + NamespaceSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "house", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"slytherin", "hufflepuff"}, + }, }, }, - }, - PodSelector: metav1.LabelSelector{ - MatchLabels: peerAllowLabel, + PodSelector: metav1.LabelSelector{ + MatchLabels: peerAllowLabel, + }, }, }, }, }, - }, - { - Name: "pass-traffic-to-ravenclaw-from-gryffindor", - Action: anpapi.AdminNetworkPolicyRuleActionPass, - To: []anpapi.AdminNetworkPolicyEgressPeer{ - { - Namespaces: &metav1.LabelSelector{ - MatchLabels: peerPassLabel, + { + Name: "pass-traffic-to-ravenclaw-from-gryffindor", + Action: anpapi.AdminNetworkPolicyRuleActionPass, + To: []anpapi.AdminNetworkPolicyEgressPeer{ + { + Namespaces: &metav1.LabelSelector{ + MatchLabels: peerPassLabel, + }, }, }, - }, - Ports: &[]anpapi.AdminNetworkPolicyPort{ // test different ports combination - { - PortNumber: &anpapi.Port{ - Protocol: v1.ProtocolTCP, - Port: int32(12345), + Ports: &[]anpapi.AdminNetworkPolicyPort{ // test different ports combination + { + PortNumber: &anpapi.Port{ + Protocol: v1.ProtocolTCP, + Port: int32(12345), + }, }, - }, - { - PortNumber: &anpapi.Port{ - Protocol: v1.ProtocolUDP, - Port: int32(12345), + { + PortNumber: &anpapi.Port{ + Protocol: v1.ProtocolUDP, + Port: int32(12345), + }, }, - }, - { - PortNumber: &anpapi.Port{ - Protocol: v1.ProtocolSCTP, - Port: int32(12345), + { + PortNumber: &anpapi.Port{ + Protocol: v1.ProtocolSCTP, + Port: int32(12345), + }, }, }, }, - }, - } - anp = newANPObject("harry-potter", 5, anpSubject, ingressRules, egressRules) - anp.ResourceVersion = "4" - anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - acls = getACLsForANPRules(anp) - pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, acls, false) - expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} - for _, acl := range acls { - acl := acl - expectedDatabaseState = append(expectedDatabaseState, acl) - } - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) - - // ingressRule AddressSets - nothing has changed - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v6) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v6) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule2v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule2v6) - - // egressRule AddressSets - peerASEgressRule0v4, peerASEgressRule0v6 := buildANPAddressSets(anp, - 0, []string{t2.podIP}, libovsdbutil.ACLEgress) // podIP should be added to v4 - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule0v4) - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule0v6) - peerASEgressRule1v4, peerASEgressRule1v6 := buildANPAddressSets(anp, - 1, []string{}, libovsdbutil.ACLEgress) // address-set will be empty since no pods match it yet - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule1v4) - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule1v6) - peerASEgressRule2v4, peerASEgressRule2v6 := buildANPAddressSets(anp, - 2, []string{}, libovsdbutil.ACLEgress) // address-set will be empty since no pods match it yet - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule2v4) - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule2v6) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - - ginkgo.By("7. update the labels of the pod that matches the peer selector; check if IPs are updated in address-sets") - anpPeerPod.ResourceVersion = "2" - anpPeerPod.Labels = peerAllowLabel // pod is now in namespace {house: slytherin} && pod {house: hufflepuff} - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Update(context.TODO(), &anpPeerPod, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // ingressRule AddressSets - podIP should stay in deny rule's address-set since deny rule matcheS only on nsSelector which has not changed - // egressRule AddressSets - podIP should now be present in both deny and allow rule's address-sets - expectedDatabaseState[len(expectedDatabaseState)-4].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be added to v4 allow address-set - expectedDatabaseState[len(expectedDatabaseState)-6].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be added to v4 deny address-set - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - - ginkgo.By("8. update the labels of the namespace that matches the peer selector; check if IPs are updated in address-sets") - anpNamespacePeer.ResourceVersion = "2" - anpNamespacePeer.Labels = peerPassLabel // pod is now in namespace {house: ravenclaw} && pod {house: hufflepuff} - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), &anpNamespacePeer, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // ingressRule AddressSets - podIP should stay in only pass rule's address-set - expectedDatabaseState[len(expectedDatabaseState)-8].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be added to v4 Pass address-set - expectedDatabaseState[len(expectedDatabaseState)-10].(*nbdb.AddressSet).Addresses = []string{} // podIP should be removed from allow rule AS - expectedDatabaseState[len(expectedDatabaseState)-12].(*nbdb.AddressSet).Addresses = []string{} // podIP should be removed from deny rule AS - - // egressRule AddressSets - podIP should stay in only pass rule's address-set - expectedDatabaseState[len(expectedDatabaseState)-2].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be added to v4 Pass address-set - expectedDatabaseState[len(expectedDatabaseState)-4].(*nbdb.AddressSet).Addresses = []string{} // podIP should be removed from allow rule AS - expectedDatabaseState[len(expectedDatabaseState)-6].(*nbdb.AddressSet).Addresses = []string{} // podIP should be removed from deny rule AS - - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + } + anp = newANPObject("harry-potter", 5, anpSubject, ingressRules, egressRules) + anp.ResourceVersion = "4" + anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + acls = getACLsForANPRules(anp) + pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, acls, false) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + if isUDNEnabled { + // fill in the blue network's representation for this ANP + udnACLs = getUDNACLsForANPRules(anp, anpSubjectNetwork) + udnPG = getUDNPGForANPSubject(anp.Name, []string{copyT.portUUID}, udnACLs, false, anpSubjectNetwork) + expectedDatabaseState = append(expectedDatabaseState, udnPG) + for _, acl := range udnACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + // peer address-sets will be empty because peers are in default network not BLUE UDN + // ingressRule AddressSets - nothing has changed + // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASIngressRule0v4, peerUDNASIngressRule0v6) + expectedDatabaseState = append(expectedDatabaseState, peerUDNASIngressRule1v4, peerUDNASIngressRule1v6) + expectedDatabaseState = append(expectedDatabaseState, peerUDNASIngressRule2v4, peerUDNASIngressRule2v6) + + // new egress rule addresssets + peerUDNASEgressRule0v4, peerUDNASEgressRule0v6 = buildUDNANPAddressSets( + anp, 0, []string{}, libovsdbutil.ACLEgress, anpSubjectNetwork) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASEgressRule0v4, peerUDNASEgressRule0v6) + peerUDNASEgressRule1v4, peerUDNASEgressRule1v6 = buildUDNANPAddressSets( + anp, 1, []string{}, libovsdbutil.ACLEgress, anpSubjectNetwork) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASEgressRule1v4, peerUDNASEgressRule1v6) + peerUDNASEgressRule2v4, peerUDNASEgressRule2v6 = buildUDNANPAddressSets( + anp, 2, []string{}, libovsdbutil.ACLEgress, anpSubjectNetwork) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASEgressRule2v4, peerUDNASEgressRule2v6) + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork), []testPod{copyT}, []string{node1Name})...) + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{copyT.podIP}) + expectedDatabaseState = append(expectedDatabaseState, + subjectUDNASIPv4, subjectUDNASIPv6, udnIsolationPG) + pg.Ports = nil // since subject pod is part of blue UDN not default network + } + for _, acl := range acls { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + // ingressRule AddressSets - nothing has changed + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v6) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v4) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v6) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule2v4) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule2v6) + + // egressRule AddressSets + peerASEgressRule0v4, peerASEgressRule0v6 := buildANPAddressSets(anp, + 0, []string{t2.podIP}, libovsdbutil.ACLEgress) // podIP should be added to v4 + expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule0v4) + expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule0v6) + peerASEgressRule1v4, peerASEgressRule1v6 := buildANPAddressSets(anp, + 1, []string{}, libovsdbutil.ACLEgress) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule1v4) + expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule1v6) + peerASEgressRule2v4, peerASEgressRule2v6 := buildANPAddressSets(anp, + 2, []string{}, libovsdbutil.ACLEgress) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule2v4) + expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule2v6) + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("7. update the labels of the pod that matches the peer selector; check if IPs are updated in address-sets") + anpPeerPod.ResourceVersion = "2" + anpPeerPod.Labels = peerAllowLabel // pod is now in namespace {house: slytherin} && pod {house: hufflepuff} + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Update(context.TODO(), &anpPeerPod, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // ingressRule AddressSets - podIP should stay in deny rule's address-set since deny rule matches only on nsSelector which has not changed + // egressRule AddressSets - podIP should now be present in both deny and allow rule's address-sets + expectedDatabaseState[len(expectedDatabaseState)-4].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be added to v4 allow address-set + expectedDatabaseState[len(expectedDatabaseState)-6].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be added to v4 deny address-set + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("8. update the labels of the namespace that matches the peer selector; check if IPs are updated in address-sets") + anpNamespacePeer.ResourceVersion = "2" + anpNamespacePeer.Labels = peerPassLabel // pod is now in namespace {house: ravenclaw} && pod {house: hufflepuff} + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), &anpNamespacePeer, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // ingressRule AddressSets - podIP should stay in only pass rule's address-set + expectedDatabaseState[len(expectedDatabaseState)-8].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be added to v4 Pass address-set + expectedDatabaseState[len(expectedDatabaseState)-10].(*nbdb.AddressSet).Addresses = []string{} // podIP should be removed from allow rule AS + expectedDatabaseState[len(expectedDatabaseState)-12].(*nbdb.AddressSet).Addresses = []string{} // podIP should be removed from deny rule AS + + // egressRule AddressSets - podIP should stay in only pass rule's address-set + expectedDatabaseState[len(expectedDatabaseState)-2].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be added to v4 Pass address-set + expectedDatabaseState[len(expectedDatabaseState)-4].(*nbdb.AddressSet).Addresses = []string{} // podIP should be removed from allow rule AS + expectedDatabaseState[len(expectedDatabaseState)-6].(*nbdb.AddressSet).Addresses = []string{} // podIP should be removed from deny rule AS + + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("9. update the subject of admin network policy so that subject pod stops matching; check if port group is updated") + anpSubject = newANPSubjectObject( + &metav1.LabelSelector{ // test different match expression + MatchLabels: anpLabel, + }, + &metav1.LabelSelector{ + MatchLabels: peerDenyLabel, // subject pod won't match any more + }, + ) + anp = newANPObject("harry-potter", 5, anpSubject, ingressRules, egressRules) + anp.ResourceVersion = "5" + anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pg = getDefaultPGForANPSubject(anp.Name, nil, acls, false) // no ports in PG + expectedDatabaseState[0] = pg + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("10. update the resource version and labels of the pod that matches the new subject selector; check if port group is updated") + anpSubjectPod.ResourceVersion = "2" + anpSubjectPod.Labels = peerDenyLabel // pod is now in namespace {house: gryffindor} && pod {house: slytherin} + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Update(context.TODO(), &anpSubjectPod, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, acls, false) // port added back to PG + if !isUDNEnabled { + // if pod is part of blue UDN + expectedDatabaseState[0] = pg + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("9. update the subject of admin network policy so that subject pod stops matching; check if port group is updated") - anpSubject = newANPSubjectObject( - &metav1.LabelSelector{ // test different match expression - MatchLabels: anpLabel, - }, - &metav1.LabelSelector{ - MatchLabels: peerDenyLabel, // subject pod won't match any more - }, - ) - anp = newANPObject("harry-potter", 5, anpSubject, ingressRules, egressRules) - anp.ResourceVersion = "5" - anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pg = getDefaultPGForANPSubject(anp.Name, nil, acls, false) // no ports in PG - expectedDatabaseState[0] = pg - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + ginkgo.By("11. update the labels of the namespace that matches the subject selector to stop matching; check if port group is updated") + anpNamespaceSubject.ResourceVersion = "1" + anpNamespaceSubject.Labels = peerPassLabel // pod is now in namespace {house: ravenclaw} && pod {house: slytherin}; stops matching + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), &anpNamespaceSubject, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pg = getDefaultPGForANPSubject(anp.Name, nil, acls, false) // no ports in PG + expectedDatabaseState[0] = pg + if isUDNEnabled { + udnPG.Ports = nil + expectedDatabaseState[5] = udnPG + // subject namespace has started matching as peer label selector for pass rule + // update blue network's representation + expectedDatabaseState[20].(*nbdb.AddressSet).Addresses = []string{copyT.podIP} // podIP should be added to v4 Pass address-set + expectedDatabaseState[26].(*nbdb.AddressSet).Addresses = []string{copyT.podIP} // podIP should be added to v4 Pass address-set + } else { + // Coincidentally the subject pod also ends up matching the peer label for Pass rule, so let's add that IP to the AS + expectedDatabaseState[len(expectedDatabaseState)-8].(*nbdb.AddressSet).Addresses = []string{t.podIP, t2.podIP} // podIP should be added to v4 Pass address-set + expectedDatabaseState[len(expectedDatabaseState)-2].(*nbdb.AddressSet).Addresses = []string{t.podIP, t2.podIP} // podIP should be added to v4 Pass address-set + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("10. update the resource version and labels of the pod that matches the new subject selector; check if port group is updated") - anpSubjectPod.ResourceVersion = "2" - anpSubjectPod.Labels = peerDenyLabel // pod is now in namespace {house: gryffindor} && pod {house: slytherin} - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Update(context.TODO(), &anpSubjectPod, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, acls, false) // port added back to PG - expectedDatabaseState[0] = pg - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + ginkgo.By("12. update the labels of the namespace that matches the subject selector to start matching; check if port group is updated") + anpNamespaceSubject.ResourceVersion = "2" + anpNamespaceSubject.Labels = anpLabel // pod is now in namespace {house: gryffindor} && pod {house: slytherin}; starts matching + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), &anpNamespaceSubject, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, acls, false) + if isUDNEnabled { + udnPG.Ports = []string{copyT.portUUID} + expectedDatabaseState[5] = udnPG + expectedDatabaseState[20].(*nbdb.AddressSet).Addresses = nil // podIP should be removed to v4 Pass address-set + expectedDatabaseState[26].(*nbdb.AddressSet).Addresses = nil // podIP should be removed to v4 Pass address-set + } else { + expectedDatabaseState[0] = pg + // Let us remove the IPs from the peer label AS as it stops matching + expectedDatabaseState[len(expectedDatabaseState)-8].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be removed from v4 Pass address-set + expectedDatabaseState[len(expectedDatabaseState)-2].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be removed from v4 Pass address-set + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("13. update the ANP by changing its priority and deleting 1 ingress rule & 1 egress rule; check if all objects are re-created correctly") + anp.ResourceVersion = "6" + anp.Spec.Priority = 6 + anp.Spec.Ingress = []anpapi.AdminNetworkPolicyIngressRule{ingressRules[1], ingressRules[2]} // deny is gone + anp.Spec.Egress = []anpapi.AdminNetworkPolicyEgressRule{egressRules[0], egressRules[2]} // allow is gone + anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("11. update the labels of the namespace that matches the subject selector to stop matching; check if port group is updated") - anpNamespaceSubject.ResourceVersion = "1" - anpNamespaceSubject.Labels = peerPassLabel // pod is now in namespace {house: ravenclaw} && pod {house: slytherin}; stops matching - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), &anpNamespaceSubject, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pg = getDefaultPGForANPSubject(anp.Name, nil, acls, false) // no ports in PG - expectedDatabaseState[0] = pg - // Coincidentally the subject pod also ends up matching the peer label for Pass rule, so let's add that IP to the AS - expectedDatabaseState[len(expectedDatabaseState)-8].(*nbdb.AddressSet).Addresses = []string{t.podIP, t2.podIP} // podIP should be added to v4 Pass address-set - expectedDatabaseState[len(expectedDatabaseState)-2].(*nbdb.AddressSet).Addresses = []string{t.podIP, t2.podIP} // podIP should be added to v4 Pass address-set - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + newACLs := getACLsForANPRules(anp) + pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, newACLs, false) // only newACLs are hosted + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + for _, acl := range newACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + // ensure address-sets for the rules that were deleted are also gone (index 2 in the list) + // ensure new address-sets have expected IPs + peerASIngressRule0v4, peerASIngressRule0v6 = buildANPAddressSets(anp, + 0, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v6) + peerASIngressRule1v4, peerASIngressRule1v6 = buildANPAddressSets(anp, + 1, []string{t2.podIP}, libovsdbutil.ACLIngress) // podIP should be added to v4 Pass address-set + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v4) + expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v6) + peerASEgressRule0v4, peerASEgressRule0v6 = buildANPAddressSets(anp, + 0, []string{}, libovsdbutil.ACLEgress) // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule0v4) + expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule0v6) + peerASEgressRule1v4, peerASEgressRule1v6 = buildANPAddressSets(anp, + 1, []string{t2.podIP}, libovsdbutil.ACLEgress) // podIP should be added to v4 Pass address-set + expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule1v4) + expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule1v6) + if isUDNEnabled { + // fill in the blue network's representation for this ANP + udnACLs = getUDNACLsForANPRules(anp, anpSubjectNetwork) + udnPG = getUDNPGForANPSubject(anp.Name, []string{copyT.portUUID}, udnACLs, false, anpSubjectNetwork) + expectedDatabaseState = append(expectedDatabaseState, udnPG) + for _, acl := range udnACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + // peer address-sets will be empty because peers are in default network not BLUE UDN + // ingressRule AddressSets - nothing has changed + // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASIngressRule0v4, peerUDNASIngressRule0v6) + expectedDatabaseState = append(expectedDatabaseState, peerUDNASIngressRule1v4, peerUDNASIngressRule1v6) + + // new egress rule addresssets + // address-set will be empty since no pods match it yet + expectedDatabaseState = append(expectedDatabaseState, peerUDNASEgressRule0v4, peerUDNASEgressRule0v6) + expectedDatabaseState = append(expectedDatabaseState, peerUDNASEgressRule1v4, peerUDNASEgressRule1v6) + + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork), []testPod{copyT}, []string{node1Name})...) + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{copyT.podIP}) + expectedDatabaseState = append(expectedDatabaseState, + subjectUDNASIPv4, subjectUDNASIPv6, udnIsolationPG) + pg.Ports = nil // since subject pod is part of blue UDN not default network + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("12. update the labels of the namespace that matches the subject selector to start matching; check if port group is updated") - anpNamespaceSubject.ResourceVersion = "2" - anpNamespaceSubject.Labels = anpLabel // pod is now in namespace {house: gryffindor} && pod {house: slytherin}; starts matching - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), &anpNamespaceSubject, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, acls, false) // no ports in PG - expectedDatabaseState[0] = pg - // Let us remove the IPs from the peer label AS as it stops matching - expectedDatabaseState[len(expectedDatabaseState)-8].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be added to v4 Pass address-set - expectedDatabaseState[len(expectedDatabaseState)-2].(*nbdb.AddressSet).Addresses = []string{t2.podIP} // podIP should be added to v4 Pass address-set - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + ginkgo.By("14. delete pod matching peer selector; check if IPs are updated in address-sets") + anpPeerPod.ResourceVersion = "3" + err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Delete(context.TODO(), anpPeerPod.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + expectedDatabaseState[3].(*nbdb.AddressSet).Addresses = nil // pod is gone from peer namespace address-set + expectedDatabaseState[18].(*nbdb.AddressSet).Addresses = []string{} // pod is gone from peer namespace address-set + expectedDatabaseState[22].(*nbdb.AddressSet).Addresses = []string{} // pod is gone from peer namespace address-set + expectedDatabaseState[7].(*nbdb.LogicalSwitch).Ports = []string{t.portUUID} // remove peer pod's LSP from switch + expectedDatabaseState = append(expectedDatabaseState[:6], expectedDatabaseState[7:]...) // remove peer pod's LSP + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("15. update the subject pod to go into completed state; check if port group and address-set is updated") + anpSubjectPod.ResourceVersion = "3" + anpSubjectPod.Status.Phase = v1.PodSucceeded + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Update(context.TODO(), &anpSubjectPod, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pg = getDefaultPGForANPSubject(anp.Name, nil, newACLs, false) // no ports in PG + expectedDatabaseState[0] = pg + expectedDatabaseState[1].(*nbdb.AddressSet).Addresses = nil + expectedDatabaseState[6].(*nbdb.LogicalSwitch).Ports = nil // remove subject pod's LSP from switch + expectedDatabaseState = append(expectedDatabaseState[:5], expectedDatabaseState[6:]...) // remove subject pod's LSP + if isUDNEnabled { + udnPG.Ports = nil + expectedDatabaseState[22] = udnPG // remove subject pod's LSP from ANP PG + expectedDatabaseState[40].(*nbdb.LogicalSwitch).Ports = nil // remove subject pod's LSP from switch + expectedDatabaseState = append(expectedDatabaseState[:39], expectedDatabaseState[40:]...) // remove subject pod's LSP + udnIsolationPG.Ports = nil + expectedDatabaseState[len(expectedDatabaseState)-1] = udnIsolationPG // remove subject pod's LSP + expectedDatabaseState[len(expectedDatabaseState)-3].(*nbdb.AddressSet).Addresses = nil // remove subject pod's IP from namespace address-set + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Delete(context.TODO(), anpSubjectPod.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("13. update the ANP by changing its priority and deleting 1 ingress rule & 1 egress rule; check if all objects are re-created correctly") - anp.ResourceVersion = "6" - anp.Spec.Priority = 6 - anp.Spec.Ingress = []anpapi.AdminNetworkPolicyIngressRule{ingressRules[1], ingressRules[2]} // deny is gone - anp.Spec.Egress = []anpapi.AdminNetworkPolicyEgressRule{egressRules[0], egressRules[2]} // allow is gone - anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("16. delete the subject and peer selected namespaces; check if port group and address-set's are updated") + // create a new pod in subject and peer namespaces so that we can check namespace deletion properly + anpSubjectPod = *newPodWithLabels(anpSubjectNamespaceName, anpSubjectPodName, node1Name, "10.128.1.5", peerDenyLabel) + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Create(context.TODO(), &anpSubjectPod, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // The pod takes some time to get created so the first add pod event for ANP does nothing as it waits for LSP to be created - this triggers a retry. + // Next when the pod is scheduled and IP is allocated and LSP is created we get an update event. + // When ANP controller receives the pod update event, it will not trigger an update in unit testing since ResourceVersion and pod.status.Running + // is not updated by the test, hence we need to trigger a manual update in the unit test framework that is accompanied by label update. + // NOTE: This is not an issue in real deployments with actual kapi server; its just a potential flake in the UT framework that we need to protect against. + time.Sleep(time.Second) + anpSubjectPod.ResourceVersion = "500" + // - this is to trigger an actual update technically speaking the pod already matches the namespace of the subject + anpSubjectPod.Labels["rv"] = "resourceVersionUTHack" + if isUDNEnabled { + // unforunately we rely on pod annotations to determine the podIPs for UDN pods since pod status doesn't have them + // and hence we need to patch the pod with the annotation of the P-UDN to simulate that + anpSubjectPod.Annotations = map[string]string{ + "k8s.ovn.org/pod-networks": fmt.Sprintf(`{%q:{"ip_addresses":["%s/24"],"mac_address":"%s"}}`, + util.GetNADName(anpSubjectNamespaceName, nadName), "100.128.0.4", + util.IPAddrToHWAddr(ovntest.MustParseIP("100.128.0.4")).String()), + } + } + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Update(context.TODO(), &anpSubjectPod, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + t.podIP = "10.128.1.5" + t.podMAC = "0a:58:0a:80:01:05" + copyT.podIP = "100.128.0.4" + copyT.podMAC = "0a:58:64:80:00:04" + pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, newACLs, false) + // deleting the namespace doesn't mean pod will get deleted in unit tests + subjectNSASIPv4, subjectNSASIPv6 = buildNamespaceAddressSets(anpSubjectNamespaceName, []string{t.podIP}) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + for _, acl := range newACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t}, []string{node1Name})...) + expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerASIngressRule0v4, peerASIngressRule0v6, peerASIngressRule1v4, + peerASIngressRule1v6, peerASEgressRule0v4, peerASEgressRule0v6, peerASEgressRule1v4, peerASEgressRule1v6}...) + if isUDNEnabled { + // fill in the blue network's representation for this ANP + udnACLs = getUDNACLsForANPRules(anp, anpSubjectNetwork) + udnPG = getUDNPGForANPSubject(anp.Name, []string{copyT.portUUID}, udnACLs, false, anpSubjectNetwork) + expectedDatabaseState = append(expectedDatabaseState, udnPG) + for _, acl := range udnACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerUDNASIngressRule0v4, peerUDNASIngressRule0v6, + peerUDNASIngressRule1v4, peerUDNASIngressRule1v6, peerUDNASEgressRule0v4, peerUDNASEgressRule0v6, + peerUDNASEgressRule1v4, peerUDNASEgressRule1v6}...) + + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork), []testPod{copyT}, []string{node1Name})...) + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{copyT.podIP}) + udnIsolationPG.Ports = []string{t.portUUID} + expectedDatabaseState = append(expectedDatabaseState, + subjectUDNASIPv4, subjectUDNASIPv6, udnIsolationPG) + pg.Ports = nil // since subject pod is part of blue UDN not default network + } + gomega.Eventually(fakeOVN.nbClient, "3s").Should(libovsdbtest.HaveData(expectedDatabaseState)) + anpPeerPod = *newPodWithLabels(anpPeerNamespaceName, anpPeerPodName, node1Name, "10.128.1.6", peerAllowLabel) + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Create(context.TODO(), &anpPeerPod, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // The pod takes some time to get created so the first add pod event for ANP does nothing as it waits for LSP to be created - this triggers a retry. + // Next when the pod is scheduled and IP is allocated and LSP is created we get an update event. + // When ANP controller receives the pod update event, it will not trigger an update in unit testing since ResourceVersion and pod.status.Running + // is not updated by the test, hence we need to trigger a manual update in the unit test framework that is accompanied by label update. + // NOTE: This is not an issue in real deployments with actual kapi server; its just a potential flake in the UT framework that we need to protect against. + time.Sleep(time.Second) + anpPeerPod.ResourceVersion = "500" + // - this is to trigger an actual update technically speaking the pod already matches the namespace of the subject + anpPeerPod.Labels["rv"] = "resourceVersionUTHack" + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Update(context.TODO(), &anpPeerPod, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + t2.podIP = "10.128.1.6" + t2.podMAC = "0a:58:0a:80:01:06" + peerNSASIPv4, peerNSASIPv6 = buildNamespaceAddressSets(anpPeerNamespaceName, []string{t2.podIP}) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + for _, acl := range newACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + peerASIngressRule1v4, peerASIngressRule1v6 = buildANPAddressSets(anp, + 1, []string{t2.podIP}, libovsdbutil.ACLIngress) + peerASEgressRule1v4, peerASEgressRule1v6 = buildANPAddressSets(anp, + 1, []string{t2.podIP}, libovsdbutil.ACLEgress) + expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerASIngressRule0v4, peerASIngressRule0v6, peerASIngressRule1v4, + peerASIngressRule1v6, peerASEgressRule0v4, peerASEgressRule0v6, peerASEgressRule1v4, peerASEgressRule1v6}...) + if isUDNEnabled { + // fill in the blue network's representation for this ANP + udnACLs = getUDNACLsForANPRules(anp, anpSubjectNetwork) + udnPG = getUDNPGForANPSubject(anp.Name, []string{copyT.portUUID}, udnACLs, false, anpSubjectNetwork) + expectedDatabaseState = append(expectedDatabaseState, udnPG) + for _, acl := range udnACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerUDNASIngressRule0v4, peerUDNASIngressRule0v6, + peerUDNASIngressRule1v4, peerUDNASIngressRule1v6, peerUDNASEgressRule0v4, peerUDNASEgressRule0v6, + peerUDNASEgressRule1v4, peerUDNASEgressRule1v6}...) + + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork), []testPod{copyT}, []string{node1Name})...) + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{copyT.podIP}) + udnIsolationPG.Ports = []string{t.portUUID} + expectedDatabaseState = append(expectedDatabaseState, + subjectUDNASIPv4, subjectUDNASIPv6, udnIsolationPG) + pg.Ports = nil // since subject pod is part of blue UDN not default network + } + gomega.Eventually(fakeOVN.nbClient, "3s").Should(libovsdbtest.HaveData(expectedDatabaseState)) - newACLs := getACLsForANPRules(anp) - pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, newACLs, false) // only newACLs are hosted - expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} - for _, acl := range newACLs { - acl := acl - expectedDatabaseState = append(expectedDatabaseState, acl) - } - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) - // ensure address-sets for the rules that were deleted are also gone (index 2 in the list) - // ensure new address-sets have expected IPs - peerASIngressRule0v4, peerASIngressRule0v6 = buildANPAddressSets(anp, - 0, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v6) - peerASIngressRule1v4, peerASIngressRule1v6 = buildANPAddressSets(anp, - 1, []string{t2.podIP}, libovsdbutil.ACLIngress) // podIP should be added to v4 Pass address-set - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v6) - peerASEgressRule0v4, peerASEgressRule0v6 = buildANPAddressSets(anp, - 0, []string{}, libovsdbutil.ACLEgress) // address-set will be empty since no pods match it yet - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule0v4) - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule0v6) - peerASEgressRule1v4, peerASEgressRule1v6 = buildANPAddressSets(anp, - 1, []string{t2.podIP}, libovsdbutil.ACLEgress) // podIP should be added to v4 Pass address-set - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule1v4) - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule1v6) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + // now delete the namespaces + anpNamespaceSubject.ResourceVersion = "3" + err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), anpNamespaceSubject.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + anpNamespacePeer.ResourceVersion = "3" + err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), anpNamespacePeer.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // network representation for the ANP is fully gone - only thing remaining is the pod rep given in UTs deleting namespace doesn't delete pods + expectedDatabaseState = []libovsdbtest.TestData{} // port group should be deleted + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + if isUDNEnabled { + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork), []testPod{copyT}, []string{node1Name})...) + expectedDatabaseState = append(expectedDatabaseState, udnIsolationPG) + // in unit tests since pods are left behind, we will still have those address-sets for the leftover peers + // stale Address-sets for namespace in P-UDN network + // TODO: Check with Peri; this looks like a bug + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{copyT.podIP}) + expectedDatabaseState = append(expectedDatabaseState, subjectUDNASIPv4, subjectUDNASIPv6) + } + gomega.Eventually(fakeOVN.nbClient, "25s").Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("17. re-add the namespaces to re-simulate ANP topology for the default network") + // NOTE (tssurya): Prior to UDNs support, when all namespaces that are matching the ANP either in the subject + // OR peers get deleted, it was treated as a peer/subject deletion and hence only ports from portgroup OR IPs from address/sets + // were deleted while retaining the port-group/address-sets themselves + // However now with implementing the UDNs, the semantics have changed internally - if all namespaces matching the ANP + // are deleted then its taken as that network representation for the ANP got deleted - implementation/cache wise + // Hence if all namespaces are gone matching an ANP that belong to a specific network; its counted as the network + // got deleted and we remove all port-group/address-sets associated with that network for this ANP + // The alternative here is to start watching for network addition/deletion which is just an overhead here not worth the effort + // Hence we have this extra step to add back the peer/subject networks so that we can test the next two steps + // now delete the namespaces + anpNamespaceSubject.ResourceVersion = "4" + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &anpNamespaceSubject, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + anpNamespacePeer.ResourceVersion = "4" + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &anpNamespacePeer, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + for _, acl := range newACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + if isUDNEnabled { + // fill in the blue network's representation for this ANP + udnACLs = getUDNACLsForANPRules(anp, anpSubjectNetwork) + udnPG = getUDNPGForANPSubject(anp.Name, []string{copyT.portUUID}, udnACLs, false, anpSubjectNetwork) + expectedDatabaseState = append(expectedDatabaseState, udnPG) + for _, acl := range udnACLs { + acl := acl + expectedDatabaseState = append(expectedDatabaseState, acl) + } + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork), []testPod{copyT}, []string{node1Name})...) + expectedDatabaseState = append(expectedDatabaseState, udnIsolationPG) + // in unit tests since pods are left behind, we will still have those address-sets for the leftover peers + // stale Address-sets for namespace in P-UDN network + // TODO: Check with Peri; this looks like a bug + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{copyT.podIP}) + expectedDatabaseState = append(expectedDatabaseState, subjectUDNASIPv4, subjectUDNASIPv6) + expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerUDNASIngressRule0v4, peerUDNASIngressRule0v6, + peerUDNASIngressRule1v4, peerUDNASIngressRule1v6, peerUDNASEgressRule0v4, peerUDNASEgressRule0v6, peerUDNASEgressRule1v4, + peerUDNASEgressRule1v6}...) + } + peerASIngressRule1v4, peerASIngressRule1v6 = buildANPAddressSets(anp, + 1, []string{t2.podIP}, libovsdbutil.ACLIngress) + peerASEgressRule1v4, peerASEgressRule1v6 = buildANPAddressSets(anp, + 1, []string{t2.podIP}, libovsdbutil.ACLEgress) + expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerASIngressRule0v4, peerASIngressRule0v6, + peerASIngressRule1v4, peerASIngressRule1v6, peerASEgressRule0v4, peerASEgressRule0v6, peerASEgressRule1v4, peerASEgressRule1v6}...) + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + gomega.Eventually(fakeOVN.nbClient, "3s").Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("18. delete all the pods matching the ANP and check if port group and address-set's are updated") + err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectNamespaceName).Delete(context.TODO(), anpSubjectPodName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerNamespaceName).Delete(context.TODO(), anpPeerPodName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + expectedDatabaseState[0].(*nbdb.PortGroup).Ports = nil // no ports in PG + expectedDatabaseState = expectedDatabaseState[:len(expectedDatabaseState)-3] // no LSPs in PG + // Let us remove the IPs from the peer label AS as it stops matching + expectedDatabaseState[1].(*nbdb.AddressSet).Addresses = []string{} // subject podIP should be removed from namespace address-set + expectedDatabaseState[2].(*nbdb.AddressSet).Addresses = []string{} // subject podIP should be removed from namespace address-set + expectedDatabaseState[3].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from namespace address-set + expectedDatabaseState[4].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from namespace-addresset + expectedDatabaseState[len(expectedDatabaseState)-1].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from ANP rule address-set + expectedDatabaseState[len(expectedDatabaseState)-2].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from ANP rule address-set + expectedDatabaseState[len(expectedDatabaseState)-5].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from ANP rule address-set + expectedDatabaseState[len(expectedDatabaseState)-6].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from ANP rule address-set + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{}, []string{node1Name})...) + if isUDNEnabled { + // reset subject represention on blue P-UDN + expectedDatabaseState[25].(*nbdb.AddressSet).Addresses = []string{} // subject podIP should be removed from namespace address-set + expectedDatabaseState[13].(*nbdb.PortGroup).Ports = nil + expectedDatabaseState[24].(*nbdb.PortGroup).Ports = nil + expectedDatabaseState[23].(*nbdb.LogicalSwitch).Ports = nil + expectedDatabaseState = append(expectedDatabaseState[:22], expectedDatabaseState[23:]...) // remove subject pod's LSP + } + gomega.Eventually(fakeOVN.nbClient, "3s").Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("14. delete pod matching peer selector; check if IPs are updated in address-sets") - anpPeerPod.ResourceVersion = "3" - err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Delete(context.TODO(), anpPeerPod.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, newACLs, false) // only newACLs are hosted - peerNSASIPv4, peerNSASIPv6 = buildNamespaceAddressSets(anpPeerNamespaceName, []string{}) // pod is gone from peer namespace address-set - expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} - for _, acl := range newACLs { - acl := acl - expectedDatabaseState = append(expectedDatabaseState, acl) - } - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t}, []string{node1Name})...) - peerASIngressRule0v4, peerASIngressRule0v6 = buildANPAddressSets(anp, - 0, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule0v6) - peerASIngressRule1v4, peerASIngressRule1v6 = buildANPAddressSets(anp, - 1, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v4) - expectedDatabaseState = append(expectedDatabaseState, peerASIngressRule1v6) - peerASEgressRule0v4, peerASEgressRule0v6 = buildANPAddressSets(anp, - 0, []string{}, libovsdbutil.ACLEgress) // address-set will be empty since no pods match it yet - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule0v4) - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule0v6) - peerASEgressRule1v4, peerASEgressRule1v6 = buildANPAddressSets(anp, - 1, []string{}, libovsdbutil.ACLEgress) // address-set will be empty since no pods match it yet - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule1v4) - expectedDatabaseState = append(expectedDatabaseState, peerASEgressRule1v6) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + ginkgo.By("19. update the ANP by deleting all rules; check if all objects are deleted correctly") + anp.Spec.Ingress = []anpapi.AdminNetworkPolicyIngressRule{} + anp.Spec.Egress = []anpapi.AdminNetworkPolicyEgressRule{} + anp.ResourceVersion = "7" + anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pg = getDefaultPGForANPSubject(anp.Name, nil, nil, false) // no ports and acls in PG + subjectNSASIPv4, subjectNSASIPv6 = buildNamespaceAddressSets(anpSubjectNamespaceName, []string{}) + peerNSASIPv4, peerNSASIPv6 = buildNamespaceAddressSets(anpPeerNamespaceName, []string{}) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{}, []string{node1Name})...) + if isUDNEnabled { + udnPG = getUDNPGForANPSubject(anp.Name, nil, nil, false, anpSubjectNetwork) // no ports and acls in PG + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{}) + expectedDatabaseState = append(expectedDatabaseState, udnPG, subjectUDNASIPv4, subjectUDNASIPv6) + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork), []testPod{}, []string{node1Name})...) + expectedDatabaseState = append(expectedDatabaseState, udnIsolationPG) + // given all peers are gone in default network, there won't be ANP rep for default network since there is no matching namespaces + // TODO: Double check this logic because peer namespace still exists... + expectedDatabaseState = expectedDatabaseState[1:] + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("15. update the subject pod to go into completed state; check if port group and address-set is updated") - anpSubjectPod.ResourceVersion = "3" - anpSubjectPod.Status.Phase = v1.PodSucceeded - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Update(context.TODO(), &anpSubjectPod, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pg = getDefaultPGForANPSubject(anp.Name, nil, newACLs, false) // no ports in PG - subjectNSASIPv4, subjectNSASIPv6 = buildNamespaceAddressSets(anpSubjectNamespaceName, []string{}) - expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} - for _, acl := range newACLs { - acl := acl - expectedDatabaseState = append(expectedDatabaseState, acl) - } - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{}, []string{node1Name})...) - expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerASIngressRule0v4, peerASIngressRule0v6, peerASIngressRule1v4, - peerASIngressRule1v6, peerASEgressRule0v4, peerASEgressRule0v6, peerASEgressRule1v4, peerASEgressRule1v6}...) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Delete(context.TODO(), anpSubjectPod.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("20. delete the ANP; check if all objects are deleted correctly") + anp.ResourceVersion = "8" + err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Delete(context.TODO(), anp.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + expectedDatabaseState = []libovsdbtest.TestData{subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} // port group should be deleted + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{}, []string{node1Name})...) + if isUDNEnabled { + expectedDatabaseState = append(expectedDatabaseState, getExpectedPodsAndSwitches( + fakeOVN.controller.networkManager.GetNetwork(anpSubjectNetwork), []testPod{}, []string{node1Name})...) + subjectUDNASIPv4, subjectUDNASIPv6 := buildUDNNamespaceAddressSets(anpSubjectNetwork, anpSubjectNamespaceName, []string{}) + expectedDatabaseState = append(expectedDatabaseState, udnIsolationPG, subjectUDNASIPv4, subjectUDNASIPv6) + } + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("16. delete the subject and peer selected namespaces; check if port group and address-set's are updated") - // create a new pod in subject and peer namespaces so that we can check namespace deletion properly - anpSubjectPod = *newPodWithLabels(anpSubjectNamespaceName, anpSubjectPodName, node1Name, "10.128.1.5", peerDenyLabel) - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Create(context.TODO(), &anpSubjectPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // The pod takes some time to get created so the first add pod event for ANP does nothing as it waits for LSP to be created - this triggers a retry. - // Next when the pod is scheduled and IP is allocated and LSP is created we get an update event. - // When ANP controller receives the pod update event, it will not trigger an update in unit testing since ResourceVersion and pod.status.Running - // is not updated by the test, hence we need to trigger a manual update in the unit test framework that is accompanied by label update. - // NOTE: This is not an issue in real deployments with actual kapi server; its just a potential flake in the UT framework that we need to protect against. - time.Sleep(time.Second) - anpSubjectPod.ResourceVersion = "500" - // - this is to trigger an actual update technically speaking the pod already matches the namespace of the subject - anpSubjectPod.Labels["rv"] = "resourceVersionUTHack" - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpSubjectPod.Namespace).Update(context.TODO(), &anpSubjectPod, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - t.podIP = "10.128.1.5" - t.podMAC = "0a:58:0a:80:01:05" - pg = getDefaultPGForANPSubject(anp.Name, []string{t.portUUID}, newACLs, false) - subjectNSASIPv4, subjectNSASIPv6 = buildNamespaceAddressSets(anpSubjectNamespaceName, []string{t.podIP}) - expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} - for _, acl := range newACLs { - acl := acl - expectedDatabaseState = append(expectedDatabaseState, acl) - } - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t}, []string{node1Name})...) - expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerASIngressRule0v4, peerASIngressRule0v6, peerASIngressRule1v4, - peerASIngressRule1v6, peerASEgressRule0v4, peerASEgressRule0v6, peerASEgressRule1v4, peerASEgressRule1v6}...) - gomega.Eventually(fakeOVN.nbClient, "3s").Should(libovsdbtest.HaveData(expectedDatabaseState)) - anpPeerPod = *newPodWithLabels(anpPeerNamespaceName, anpPeerPodName, node1Name, "10.128.1.6", peerAllowLabel) - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Create(context.TODO(), &anpPeerPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // The pod takes some time to get created so the first add pod event for ANP does nothing as it waits for LSP to be created - this triggers a retry. - // Next when the pod is scheduled and IP is allocated and LSP is created we get an update event. - // When ANP controller receives the pod update event, it will not trigger an update in unit testing since ResourceVersion and pod.status.Running - // is not updated by the test, hence we need to trigger a manual update in the unit test framework that is accompanied by label update. - // NOTE: This is not an issue in real deployments with actual kapi server; its just a potential flake in the UT framework that we need to protect against. - time.Sleep(time.Second) - anpPeerPod.ResourceVersion = "500" - // - this is to trigger an actual update technically speaking the pod already matches the namespace of the subject - anpPeerPod.Labels["rv"] = "resourceVersionUTHack" - _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(anpPeerPod.Namespace).Update(context.TODO(), &anpPeerPod, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - t2.podIP = "10.128.1.6" - t2.podMAC = "0a:58:0a:80:01:06" - peerNSASIPv4, peerNSASIPv6 = buildNamespaceAddressSets(anpPeerNamespaceName, []string{t2.podIP}) - expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} - for _, acl := range newACLs { - acl := acl - expectedDatabaseState = append(expectedDatabaseState, acl) - } - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) - peerASIngressRule1v4, peerASIngressRule1v6 = buildANPAddressSets(anp, - 1, []string{t2.podIP}, libovsdbutil.ACLIngress) - peerASEgressRule1v4, peerASEgressRule1v6 = buildANPAddressSets(anp, - 1, []string{t2.podIP}, libovsdbutil.ACLEgress) - expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerASIngressRule0v4, peerASIngressRule0v6, peerASIngressRule1v4, - peerASIngressRule1v6, peerASEgressRule0v4, peerASEgressRule0v6, peerASEgressRule1v4, peerASEgressRule1v6}...) - gomega.Eventually(fakeOVN.nbClient, "3s").Should(libovsdbtest.HaveData(expectedDatabaseState)) - - // now delete the namespaces - anpNamespaceSubject.ResourceVersion = "3" - err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), anpNamespaceSubject.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - anpNamespacePeer.ResourceVersion = "3" - err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), anpNamespacePeer.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pg = getDefaultPGForANPSubject(anp.Name, nil, newACLs, false) // no ports in PG - expectedDatabaseState = []libovsdbtest.TestData{pg} // namespace address-sets are gone - for _, acl := range newACLs { - acl := acl - expectedDatabaseState = append(expectedDatabaseState, acl) + return nil } - // delete namespace in unit testing doesn't delete pods; we just need to check port groups and address-sets are updated - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) - peerASIngressRule1v4, peerASIngressRule1v6 = buildANPAddressSets(anp, - 1, []string{}, libovsdbutil.ACLIngress) - peerASEgressRule1v4, peerASEgressRule1v6 = buildANPAddressSets(anp, - 1, []string{}, libovsdbutil.ACLEgress) - expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerASIngressRule0v4, peerASIngressRule0v6, peerASIngressRule1v4, - peerASIngressRule1v6, peerASEgressRule0v4, peerASEgressRule0v6, peerASEgressRule1v4, peerASEgressRule1v6}...) - // NOTE: Address set deletion is deferred for 20 seconds... - gomega.Eventually(fakeOVN.nbClient, "25s").Should(libovsdbtest.HaveData(expectedDatabaseState)) - - ginkgo.By("17. update the ANP by deleting all rules; check if all objects are deleted correctly") - anp.Spec.Ingress = []anpapi.AdminNetworkPolicyIngressRule{} - anp.Spec.Egress = []anpapi.AdminNetworkPolicyEgressRule{} - anp.ResourceVersion = "7" - anp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Update(context.TODO(), anp, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pg = getDefaultPGForANPSubject(anp.Name, nil, nil, false) // no ports and acls in PG - expectedDatabaseState = []libovsdbtest.TestData{pg} - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - - ginkgo.By("18. delete the ANP; check if all objects are deleted correctly") - anp.ResourceVersion = "8" - err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Delete(context.TODO(), anp.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedDatabaseState = []libovsdbtest.TestData{} // port group should be deleted - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - - return nil - } - err := app.Run([]string{app.Name}) - gomega.Expect(err).ToNot(gomega.HaveOccurred()) - }) + err := app.Run([]string{app.Name}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }, + ginkgo.Entry("all namespaces are part of default network", + types.DefaultNetworkName, types.DefaultNetworkName, false), + ginkgo.Entry("subject namespace is part of blue P-UDN network while peer namespace is part of default network", + "blue", types.DefaultNetworkName, true), + /*ginkgo.Entry("subject namespace is default network while peer namespace is part of blue P-UDN network", + types.DefaultNetworkName, "blue", true), + ginkgo.Entry("all namespaces are part of blue P-UDN network", + "blue", "blue", true),*/ + ) ginkgo.It("PortNumber, PortRange and NamedPort (no matching pods) Rules for ANP", func() { app.Action = func(ctx *cli.Context) error { config.IPv4Mode = true @@ -1046,6 +1449,12 @@ var _ = ginkgo.Describe("OVN ANP Operations", func() { anp.ResourceVersion = "1" anp, err := fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Create(context.TODO(), anp, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveEmptyData()) + + ginkgo.By("1.1. Create the matching namespace so that network's representation is honoured") + anpSubjectNamespace := *newNamespaceWithLabels(anpSubjectNamespaceName, anpLabel) + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &anpSubjectNamespace, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) acls := getACLsForANPRules(anp) pg := getDefaultPGForANPSubject(anp.Name, []string{}, acls, false) expectedDatabaseState := []libovsdbtest.TestData{pg} @@ -1510,7 +1919,7 @@ var _ = ginkgo.Describe("OVN ANP Operations", func() { klog.Infof("Got an update spec action for %v", update.GetObject()) return false, update.GetObject(), nil }) - ginkgo.By("1. Create ANP with 1 ingress rule and 2 egress rules with the ACL logging annotation and ensure its honoured") + ginkgo.By("1. Create ANP with 1 ingress rule and 2 egress rules with the ACL logging annotation and ensure its honoured when the network representation is present") anpSubject := newANPSubjectObject( &metav1.LabelSelector{ MatchLabels: anpLabel, @@ -1562,6 +1971,12 @@ var _ = ginkgo.Describe("OVN ANP Operations", func() { } anp, err := fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().AdminNetworkPolicies().Create(context.TODO(), anp, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveEmptyData()) + + ginkgo.By("1.1. Create the matching namespace so that network's representation is honoured") + anpSubjectNamespace := *newNamespaceWithLabels(anpSubjectNamespaceName, anpLabel) + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &anpSubjectNamespace, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) acls := getACLsForANPRules(anp) pg := getDefaultPGForANPSubject(anp.Name, []string{}, acls, false) expectedDatabaseState := []libovsdbtest.TestData{pg} diff --git a/go-controller/pkg/ovn/base_network_controller.go b/go-controller/pkg/ovn/base_network_controller.go index b8ac355cbf..b15a00e64b 100644 --- a/go-controller/pkg/ovn/base_network_controller.go +++ b/go-controller/pkg/ovn/base_network_controller.go @@ -20,7 +20,7 @@ import ( libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/observability" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" lsm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager" @@ -29,7 +29,6 @@ import ( ovnretry "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/syncmap" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" - ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" kapi "k8s.io/api/core/v1" @@ -84,7 +83,7 @@ type BaseNetworkController struct { controllerName string // network information - util.NetInfo + util.ReconcilableNetInfo // retry framework for pods retryPods *ovnretry.RetryFramework @@ -165,8 +164,9 @@ type BaseNetworkController struct { // to the cluster router. Please see zone_interconnect/interconnect_handler.go for more details. zoneICHandler *zoneic.ZoneInterconnectHandler - // nadController used for getting network information for UDNs - nadController nad.NADController + // networkManager used for getting network information + networkManager networkmanager.Interface + // releasedPodsBeforeStartup tracks pods per NAD (map of NADs to pods UIDs) // might have been already be released on startup releasedPodsBeforeStartup map[string]sets.Set[string] @@ -192,6 +192,10 @@ type BaseSecondaryNetworkController struct { multiNetPolicyHandler *factory.Handler } +func (oc *BaseSecondaryNetworkController) Reconcile(netInfo util.NetInfo) error { + return util.ReconcileNetInfo(oc.ReconcilableNetInfo, netInfo) +} + func getNetworkControllerName(netName string) string { return netName + "-network-controller" } @@ -368,7 +372,7 @@ func (bnc *BaseNetworkController) createNodeLogicalSwitch(nodeName string, hostS Name: switchName, } - logicalSwitch.ExternalIDs = util.GenerateExternalIDsForSwitchOrRouter(bnc.NetInfo) + logicalSwitch.ExternalIDs = util.GenerateExternalIDsForSwitchOrRouter(bnc.GetNetInfo()) var v4Gateway, v6Gateway net.IP logicalSwitch.OtherConfig = map[string]string{} for _, hostSubnet := range hostSubnets { @@ -669,8 +673,8 @@ func (bnc *BaseNetworkController) syncNodeManagementPort(node *kapi.Node, switch } if bnc.IsSecondary() { lrsr.ExternalIDs = map[string]string{ - ovntypes.NetworkExternalID: bnc.GetNetworkName(), - ovntypes.TopologyExternalID: bnc.TopologyType(), + types.NetworkExternalID: bnc.GetNetworkName(), + types.TopologyExternalID: bnc.TopologyType(), } } p := func(item *nbdb.LogicalRouterStaticRoute) bool { @@ -772,14 +776,14 @@ func (bnc *BaseNetworkController) recordPodErrorEvent(pod *kapi.Pod, podErr erro } func (bnc *BaseNetworkController) doesNetworkRequireIPAM() bool { - return util.DoesNetworkRequireIPAM(bnc.NetInfo) + return util.DoesNetworkRequireIPAM(bnc.GetNetInfo()) } func (bnc *BaseNetworkController) getPodNADNames(pod *kapi.Pod) []string { if !bnc.IsSecondary() { return []string{types.DefaultNetworkName} } - podNadNames, _ := util.PodNadNames(pod, bnc.NetInfo) + podNadNames, _ := util.PodNadNames(pod, bnc.GetNetInfo()) return podNadNames } @@ -835,11 +839,6 @@ func (bnc *BaseNetworkController) isLocalZoneNode(node *kapi.Node) bool { return util.GetNodeZone(node) == bnc.zone } -// and is a wrapper around GetActiveNetworkForNamespace -func (bnc *BaseNetworkController) getActiveNetworkForNamespace(namespace string) (util.NetInfo, error) { - return bnc.nadController.GetActiveNetworkForNamespace(namespace) -} - // GetNetworkRole returns the role of this controller's // network for the given pod // Expected values are: @@ -873,7 +872,7 @@ func (bnc *BaseNetworkController) GetNetworkRole(pod *kapi.Pod) (string, error) } return types.NetworkRoleSecondary, nil } - activeNetwork, err := bnc.getActiveNetworkForNamespace(pod.Namespace) + activeNetwork, err := bnc.networkManager.GetActiveNetworkForNamespace(pod.Namespace) if err != nil { if util.IsUnprocessedActiveNetworkError(err) { bnc.recordPodErrorEvent(pod, err) @@ -893,7 +892,7 @@ func (bnc *BaseNetworkController) GetNetworkRole(pod *kapi.Pod) (string, error) } func (bnc *BaseNetworkController) isLayer2Interconnect() bool { - return config.OVNKubernetesFeature.EnableInterconnect && bnc.NetInfo.TopologyType() == types.Layer2Topology + return config.OVNKubernetesFeature.EnableInterconnect && bnc.TopologyType() == types.Layer2Topology } func (bnc *BaseNetworkController) nodeZoneClusterChanged(oldNode, newNode *kapi.Node, newNodeIsLocalZone bool, netName string) bool { @@ -909,7 +908,7 @@ func (bnc *BaseNetworkController) nodeZoneClusterChanged(oldNode, newNode *kapi. } // NodeGatewayRouterLRPAddrsAnnotationChanged would not affect local, nor localnet secondary network - if !newNodeIsLocalZone && bnc.NetInfo.TopologyType() != types.LocalnetTopology && joinCIDRChanged(oldNode, newNode, netName) { + if !newNodeIsLocalZone && bnc.TopologyType() != types.LocalnetTopology && joinCIDRChanged(oldNode, newNode, netName) { return true } @@ -977,7 +976,7 @@ func (bnc *BaseNetworkController) AddResourceCommon(objType reflect.Type, obj in if !ok { return fmt.Errorf("could not cast %T object to *knet.NetworkPolicy", obj) } - netinfo, err := bnc.getActiveNetworkForNamespace(np.Namespace) + netinfo, err := bnc.networkManager.GetActiveNetworkForNamespace(np.Namespace) if err != nil { return fmt.Errorf("could not get active network for namespace %s: %v", np.Namespace, err) } @@ -1002,7 +1001,7 @@ func (bnc *BaseNetworkController) DeleteResourceCommon(objType reflect.Type, obj if !ok { return fmt.Errorf("could not cast obj of type %T to *knet.NetworkPolicy", obj) } - netinfo, err := bnc.getActiveNetworkForNamespace(knp.Namespace) + netinfo, err := bnc.networkManager.GetActiveNetworkForNamespace(knp.Namespace) if err != nil { return fmt.Errorf("could not get active network for namespace %s: %v", knp.Namespace, err) } @@ -1019,7 +1018,7 @@ func (bnc *BaseNetworkController) DeleteResourceCommon(objType reflect.Type, obj func initLoadBalancerGroups(nbClient libovsdbclient.Client, netInfo util.NetInfo) ( clusterLoadBalancerGroupUUID, switchLoadBalancerGroupUUID, routerLoadBalancerGroupUUID string, err error) { - loadBalancerGroupName := netInfo.GetNetworkScopedLoadBalancerGroupName(ovntypes.ClusterLBGroupName) + loadBalancerGroupName := netInfo.GetNetworkScopedLoadBalancerGroupName(types.ClusterLBGroupName) clusterLBGroup := nbdb.LoadBalancerGroup{Name: loadBalancerGroupName} ops, err := libovsdbops.CreateOrUpdateLoadBalancerGroupOps(nbClient, nil, &clusterLBGroup) if err != nil { @@ -1027,7 +1026,7 @@ func initLoadBalancerGroups(nbClient libovsdbclient.Client, netInfo util.NetInfo return } - loadBalancerGroupName = netInfo.GetNetworkScopedLoadBalancerGroupName(ovntypes.ClusterSwitchLBGroupName) + loadBalancerGroupName = netInfo.GetNetworkScopedLoadBalancerGroupName(types.ClusterSwitchLBGroupName) clusterSwitchLBGroup := nbdb.LoadBalancerGroup{Name: loadBalancerGroupName} ops, err = libovsdbops.CreateOrUpdateLoadBalancerGroupOps(nbClient, ops, &clusterSwitchLBGroup) if err != nil { @@ -1035,7 +1034,7 @@ func initLoadBalancerGroups(nbClient libovsdbclient.Client, netInfo util.NetInfo return } - loadBalancerGroupName = netInfo.GetNetworkScopedLoadBalancerGroupName(ovntypes.ClusterRouterLBGroupName) + loadBalancerGroupName = netInfo.GetNetworkScopedLoadBalancerGroupName(types.ClusterRouterLBGroupName) clusterRouterLBGroup := nbdb.LoadBalancerGroup{Name: loadBalancerGroupName} ops, err = libovsdbops.CreateOrUpdateLoadBalancerGroupOps(nbClient, ops, &clusterRouterLBGroup) if err != nil { diff --git a/go-controller/pkg/ovn/base_network_controller_namespace.go b/go-controller/pkg/ovn/base_network_controller_namespace.go index 41b66b1729..7523ae3643 100644 --- a/go-controller/pkg/ovn/base_network_controller_namespace.go +++ b/go-controller/pkg/ovn/base_network_controller_namespace.go @@ -402,7 +402,7 @@ func (bnc *BaseNetworkController) getAllNamespacePodAddresses(ns string) []net.I ips = make([]net.IP, 0, len(existingPods)) for _, pod := range existingPods { if !util.PodWantsHostNetwork(pod) && !util.PodCompleted(pod) && util.PodScheduled(pod) { - podIPs, err := util.GetPodIPsOfNetwork(pod, bnc.NetInfo) + podIPs, err := util.GetPodIPsOfNetwork(pod, bnc.GetNetInfo()) if err != nil { klog.Warningf(err.Error()) continue @@ -446,7 +446,7 @@ func (bnc *BaseNetworkController) getNamespacePortGroupName(namespace string) st // failure indicates it should be retried later. func (bsnc *BaseNetworkController) removeRemoteZonePodFromNamespaceAddressSet(pod *kapi.Pod) error { podDesc := fmt.Sprintf("pod %s/%s/%s", bsnc.GetNetworkName(), pod.Namespace, pod.Name) - podIfAddrs, err := util.GetPodCIDRsWithFullMask(pod, bsnc.NetInfo) + podIfAddrs, err := util.GetPodCIDRsWithFullMask(pod, bsnc.GetNetInfo()) if err != nil { // maybe the pod is not scheduled yet or addLSP has not happened yet, so it doesn't have IPs. // let us ignore deletion failures for podIPs not found because diff --git a/go-controller/pkg/ovn/base_network_controller_pods.go b/go-controller/pkg/ovn/base_network_controller_pods.go index c00d2d2061..37be0ed77e 100644 --- a/go-controller/pkg/ovn/base_network_controller_pods.go +++ b/go-controller/pkg/ovn/base_network_controller_pods.go @@ -330,7 +330,7 @@ func (bnc *BaseNetworkController) canReleasePodIPs(podIfAddrs []*net.IPNet, node needleIPs = append(needleIPs, podIPNet.IP) } - collidingPod, err := findPodWithIPAddresses(bnc.watchFactory, bnc.NetInfo, needleIPs, nodeName) + collidingPod, err := findPodWithIPAddresses(bnc.watchFactory, bnc.GetNetInfo(), needleIPs, nodeName) if err != nil { return false, fmt.Errorf("unable to determine if pod IPs: %#v are in use by another pod :%w", podIfAddrs, err) @@ -424,7 +424,7 @@ func (bnc *BaseNetworkController) getExpectedSwitchName(pod *kapi.Pod) (string, // controller's zone. func (bnc *BaseNetworkController) ensurePodAnnotation(pod *kapi.Pod, nadName string) (*util.PodAnnotation, bool, error) { if kubevirt.IsPodLiveMigratable(pod) { - podAnnotation, err := kubevirt.EnsurePodAnnotationForVM(bnc.watchFactory, bnc.kube, pod, bnc.NetInfo, nadName) + podAnnotation, err := kubevirt.EnsurePodAnnotationForVM(bnc.watchFactory, bnc.kube, pod, bnc.GetNetInfo(), nadName) if err != nil { return nil, false, err } @@ -898,7 +898,7 @@ func (bnc *BaseNetworkController) allocatePodAnnotation(pod *kapi.Pod, existingL return nil, false, fmt.Errorf("cannot retrieve subnet for assigning gateway routes for pod %s, switch: %s", podDesc, switchName) } - err = util.AddRoutesGatewayIP(bnc.NetInfo, pod, podAnnotation, network) + err = util.AddRoutesGatewayIP(bnc.GetNetInfo(), pod, podAnnotation, network) if err != nil { return nil, false, err } @@ -990,7 +990,7 @@ func (bnc *BaseNetworkController) allocatePodAnnotationForSecondaryNetwork(pod * } func (bnc *BaseNetworkController) allocatesPodAnnotation() bool { - switch bnc.NetInfo.TopologyType() { + switch bnc.TopologyType() { case ovntypes.Layer2Topology: // on layer2 topologies, cluster manager allocates tunnel IDs, allocates // IPs if the network has IPAM, and sets the PodAnnotation diff --git a/go-controller/pkg/ovn/base_network_controller_policy.go b/go-controller/pkg/ovn/base_network_controller_policy.go index 73e18e87e3..63523ceeb7 100644 --- a/go-controller/pkg/ovn/base_network_controller_policy.go +++ b/go-controller/pkg/ovn/base_network_controller_policy.go @@ -998,7 +998,7 @@ func (bnc *BaseNetworkController) createNetworkPolicy(policy *knet.NetworkPolicy for i, ingressJSON := range policy.Spec.Ingress { klog.V(5).Infof("Network policy ingress is %+v", ingressJSON) - ingress := newGressPolicy(knet.PolicyTypeIngress, i, policy.Namespace, policy.Name, bnc.controllerName, statelessNetPol, bnc.NetInfo) + ingress := newGressPolicy(knet.PolicyTypeIngress, i, policy.Namespace, policy.Name, bnc.controllerName, statelessNetPol, bnc.GetNetInfo()) // append ingress policy to be able to cleanup created address set // see cleanupNetworkPolicy for details np.ingressPolicies = append(np.ingressPolicies, ingress) @@ -1024,7 +1024,7 @@ func (bnc *BaseNetworkController) createNetworkPolicy(policy *knet.NetworkPolicy for i, egressJSON := range policy.Spec.Egress { klog.V(5).Infof("Network policy egress is %+v", egressJSON) - egress := newGressPolicy(knet.PolicyTypeEgress, i, policy.Namespace, policy.Name, bnc.controllerName, statelessNetPol, bnc.NetInfo) + egress := newGressPolicy(knet.PolicyTypeEgress, i, policy.Namespace, policy.Name, bnc.controllerName, statelessNetPol, bnc.GetNetInfo()) // append ingress policy to be able to cleanup created address set // see cleanupNetworkPolicy for details np.egressPolicies = append(np.egressPolicies, egress) diff --git a/go-controller/pkg/ovn/base_network_controller_secondary.go b/go-controller/pkg/ovn/base_network_controller_secondary.go index 062c6ac618..1d0d617a93 100644 --- a/go-controller/pkg/ovn/base_network_controller_secondary.go +++ b/go-controller/pkg/ovn/base_network_controller_secondary.go @@ -28,14 +28,13 @@ import ( utilerrors "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" corev1 "k8s.io/api/core/v1" - kapi "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/klog/v2" utilnet "k8s.io/utils/net" "k8s.io/utils/ptr" ) -func (bsnc *BaseSecondaryNetworkController) getPortInfoForSecondaryNetwork(pod *kapi.Pod) map[string]*lpInfo { +func (bsnc *BaseSecondaryNetworkController) getPortInfoForSecondaryNetwork(pod *corev1.Pod) map[string]*lpInfo { if util.PodWantsHostNetwork(pod) { return nil } @@ -48,7 +47,7 @@ func (bsnc *BaseSecondaryNetworkController) getPortInfoForSecondaryNetwork(pod * func (bsnc *BaseSecondaryNetworkController) GetInternalCacheEntryForSecondaryNetwork(objType reflect.Type, obj interface{}) interface{} { switch objType { case factory.PodType: - pod := obj.(*kapi.Pod) + pod := obj.(*corev1.Pod) return bsnc.getPortInfoForSecondaryNetwork(pod) default: return nil @@ -60,14 +59,14 @@ func (bsnc *BaseSecondaryNetworkController) GetInternalCacheEntryForSecondaryNet func (bsnc *BaseSecondaryNetworkController) AddSecondaryNetworkResourceCommon(objType reflect.Type, obj interface{}) error { switch objType { case factory.PodType: - pod, ok := obj.(*kapi.Pod) + pod, ok := obj.(*corev1.Pod) if !ok { return fmt.Errorf("could not cast %T object to *knet.Pod", obj) } return bsnc.ensurePodForSecondaryNetwork(pod, true) case factory.NamespaceType: - ns, ok := obj.(*kapi.Namespace) + ns, ok := obj.(*corev1.Namespace) if !ok { return fmt.Errorf("could not cast %T object to *kapi.Namespace", obj) } @@ -109,13 +108,13 @@ func (bsnc *BaseSecondaryNetworkController) AddSecondaryNetworkResourceCommon(ob func (bsnc *BaseSecondaryNetworkController) UpdateSecondaryNetworkResourceCommon(objType reflect.Type, oldObj, newObj interface{}, inRetryCache bool) error { switch objType { case factory.PodType: - oldPod := oldObj.(*kapi.Pod) - newPod := newObj.(*kapi.Pod) + oldPod := oldObj.(*corev1.Pod) + newPod := newObj.(*corev1.Pod) return bsnc.ensurePodForSecondaryNetwork(newPod, inRetryCache || util.PodScheduled(oldPod) != util.PodScheduled(newPod)) case factory.NamespaceType: - oldNs, newNs := oldObj.(*kapi.Namespace), newObj.(*kapi.Namespace) + oldNs, newNs := oldObj.(*corev1.Namespace), newObj.(*corev1.Namespace) return bsnc.updateNamespaceForSecondaryNetwork(oldNs, newNs) case factory.MultiNetworkPolicyType: @@ -171,7 +170,7 @@ func (bsnc *BaseSecondaryNetworkController) DeleteSecondaryNetworkResourceCommon switch objType { case factory.PodType: var portInfoMap map[string]*lpInfo - pod := obj.(*kapi.Pod) + pod := obj.(*corev1.Pod) if cachedObj != nil { portInfoMap = cachedObj.(map[string]*lpInfo) @@ -179,7 +178,7 @@ func (bsnc *BaseSecondaryNetworkController) DeleteSecondaryNetworkResourceCommon return bsnc.removePodForSecondaryNetwork(pod, portInfoMap) case factory.NamespaceType: - ns := obj.(*kapi.Namespace) + ns := obj.(*corev1.Namespace) return bsnc.deleteNamespace4SecondaryNetwork(ns) case factory.MultiNetworkPolicyType: @@ -225,7 +224,7 @@ func (bsnc *BaseSecondaryNetworkController) DeleteSecondaryNetworkResourceCommon // ensurePodForSecondaryNetwork tries to set up secondary network for a pod. It returns nil on success and error // on failure; failure indicates the pod set up should be retried later. -func (bsnc *BaseSecondaryNetworkController) ensurePodForSecondaryNetwork(pod *kapi.Pod, addPort bool) error { +func (bsnc *BaseSecondaryNetworkController) ensurePodForSecondaryNetwork(pod *corev1.Pod, addPort bool) error { // Try unscheduled pods later if !util.PodScheduled(pod) { @@ -239,7 +238,7 @@ func (bsnc *BaseSecondaryNetworkController) ensurePodForSecondaryNetwork(pod *ka var kubevirtLiveMigrationStatus *kubevirt.LiveMigrationStatus var err error - if kubevirt.IsPodAllowedForMigration(pod, bsnc.NetInfo) { + if kubevirt.IsPodAllowedForMigration(pod, bsnc.GetNetInfo()) { kubevirtLiveMigrationStatus, err = kubevirt.DiscoverLiveMigrationStatus(bsnc.watchFactory, pod) if err != nil { return fmt.Errorf("failed to discover Live-migration status: %w", err) @@ -258,12 +257,12 @@ func (bsnc *BaseSecondaryNetworkController) ensurePodForSecondaryNetwork(pod *ka return err } - activeNetwork, err := bsnc.getActiveNetworkForNamespace(pod.Namespace) + activeNetwork, err := bsnc.networkManager.GetActiveNetworkForNamespace(pod.Namespace) if err != nil { return fmt.Errorf("failed looking for the active network at namespace '%s': %w", pod.Namespace, err) } - on, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork(pod, bsnc.NetInfo, activeNetwork) + on, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork(pod, bsnc.GetNetInfo(), activeNetwork) if err != nil { bsnc.recordPodErrorEvent(pod, err) // configuration error, no need to retry, do not return error @@ -297,7 +296,7 @@ func (bsnc *BaseSecondaryNetworkController) ensurePodForSecondaryNetwork(pod *ka return nil } -func (bsnc *BaseSecondaryNetworkController) addLogicalPortToNetworkForNAD(pod *kapi.Pod, nadName, switchName string, +func (bsnc *BaseSecondaryNetworkController) addLogicalPortToNetworkForNAD(pod *corev1.Pod, nadName, switchName string, network *nadapi.NetworkSelectionElement, kubevirtLiveMigrationStatus *kubevirt.LiveMigrationStatus) error { var libovsdbExecuteTime time.Duration @@ -418,9 +417,9 @@ func (bsnc *BaseSecondaryNetworkController) addLogicalPortToNetworkForNAD(pod *k } if isLocalPod { - bsnc.podRecorder.AddLSP(pod.UID, bsnc.NetInfo) + bsnc.podRecorder.AddLSP(pod.UID, bsnc.GetNetInfo()) if newlyCreated { - metrics.RecordPodCreated(pod, bsnc.NetInfo) + metrics.RecordPodCreated(pod, bsnc.GetNetInfo()) } } @@ -428,7 +427,7 @@ func (bsnc *BaseSecondaryNetworkController) addLogicalPortToNetworkForNAD(pod *k } // addPerPodSNATOps returns the ops that will add the SNAT towards masqueradeIP for this given pod -func (bsnc *BaseSecondaryNetworkController) addPerPodSNATOps(pod *kapi.Pod, podIPs []*net.IPNet) ([]ovsdb.Operation, error) { +func (bsnc *BaseSecondaryNetworkController) addPerPodSNATOps(pod *corev1.Pod, podIPs []*net.IPNet) ([]ovsdb.Operation, error) { if !bsnc.isPodScheduledinLocalZone(pod) { // nothing to do if its a remote zone pod return nil, nil @@ -443,7 +442,7 @@ func (bsnc *BaseSecondaryNetworkController) addPerPodSNATOps(pod *kapi.Pod, podI return nil, fmt.Errorf("failed to get masquerade IPs, network %s (%d): %v", bsnc.GetNetworkName(), networkID, err) } - ops, err := addOrUpdatePodSNATOps(bsnc.nbClient, bsnc.GetNetworkScopedGWRouterName(pod.Spec.NodeName), masqIPs, podIPs, bsnc.NetInfo.GetNetworkScopedClusterSubnetSNATMatch(pod.Spec.NodeName), nil) + ops, err := addOrUpdatePodSNATOps(bsnc.nbClient, bsnc.GetNetworkScopedGWRouterName(pod.Spec.NodeName), masqIPs, podIPs, bsnc.GetNetworkScopedClusterSubnetSNATMatch(pod.Spec.NodeName), nil) if err != nil { return nil, fmt.Errorf("failed to construct SNAT pods for pod %s/%s which is part of network %s, err: %v", pod.Namespace, pod.Name, bsnc.GetNetworkName(), err) @@ -453,7 +452,7 @@ func (bsnc *BaseSecondaryNetworkController) addPerPodSNATOps(pod *kapi.Pod, podI // removePodForSecondaryNetwork tried to tear down a pod. It returns nil on success and error on failure; // failure indicates the pod tear down should be retried later. -func (bsnc *BaseSecondaryNetworkController) removePodForSecondaryNetwork(pod *kapi.Pod, portInfoMap map[string]*lpInfo) error { +func (bsnc *BaseSecondaryNetworkController) removePodForSecondaryNetwork(pod *corev1.Pod, portInfoMap map[string]*lpInfo) error { if util.PodWantsHostNetwork(pod) || !util.PodScheduled(pod) { return nil } @@ -504,7 +503,7 @@ func (bsnc *BaseSecondaryNetworkController) removePodForSecondaryNetwork(pod *ka alreadyProcessed = true } - if kubevirt.IsPodAllowedForMigration(pod, bsnc.NetInfo) { + if kubevirt.IsPodAllowedForMigration(pod, bsnc.GetNetInfo()) { if err = bsnc.enableSourceLSPFailedLiveMigration(pod, nadName); err != nil { return err } @@ -565,7 +564,7 @@ func (bsnc *BaseSecondaryNetworkController) removePodForSecondaryNetwork(pod *ka // hasIPAMClaim determines whether a pod's IPAM is being handled by IPAMClaim CR. // pod passed should already be validated as having a network connection to nadName -func (bsnc *BaseSecondaryNetworkController) hasIPAMClaim(pod *kapi.Pod, nadNamespacedName string) (bool, error) { +func (bsnc *BaseSecondaryNetworkController) hasIPAMClaim(pod *corev1.Pod, nadNamespacedName string) (bool, error) { if !bsnc.AllowsPersistentIPs() { return false, nil } @@ -617,7 +616,7 @@ func (bsnc *BaseSecondaryNetworkController) hasIPAMClaim(pod *kapi.Pod, nadNames } // delPerPodSNAT will delete the SNAT towards masqueradeIP for this given pod -func (bsnc *BaseSecondaryNetworkController) delPerPodSNAT(pod *kapi.Pod, nadName string) error { +func (bsnc *BaseSecondaryNetworkController) delPerPodSNAT(pod *corev1.Pod, nadName string) error { if !bsnc.isPodScheduledinLocalZone(pod) { // nothing to do if its a remote zone pod return nil @@ -648,22 +647,22 @@ func (bsnc *BaseSecondaryNetworkController) delPerPodSNAT(pod *kapi.Pod, nadName } func (bsnc *BaseSecondaryNetworkController) syncPodsForSecondaryNetwork(pods []interface{}) error { - annotatedLocalPods := map[*kapi.Pod]map[string]*util.PodAnnotation{} + annotatedLocalPods := map[*corev1.Pod]map[string]*util.PodAnnotation{} // get the list of logical switch ports (equivalent to pods). Reserve all existing Pod IPs to // avoid subsequent new Pods getting the same duplicate Pod IP. expectedLogicalPorts := make(map[string]bool) for _, podInterface := range pods { - pod, ok := podInterface.(*kapi.Pod) + pod, ok := podInterface.(*corev1.Pod) if !ok { return fmt.Errorf("spurious object in syncPods: %v", podInterface) } - activeNetwork, err := bsnc.getActiveNetworkForNamespace(pod.Namespace) + activeNetwork, err := bsnc.networkManager.GetActiveNetworkForNamespace(pod.Namespace) if err != nil { return fmt.Errorf("failed looking for the active network at namespace '%s': %w", pod.Namespace, err) } - on, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork(pod, bsnc.NetInfo, activeNetwork) + on, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork(pod, bsnc.GetNetInfo(), activeNetwork) if err != nil || !on { if err != nil { bsnc.recordPodErrorEvent(pod, err) @@ -728,7 +727,7 @@ func (bsnc *BaseSecondaryNetworkController) addPodToNamespaceForSecondaryNetwork } // AddNamespaceForSecondaryNetwork creates corresponding addressset in ovn db for secondary network -func (bsnc *BaseSecondaryNetworkController) AddNamespaceForSecondaryNetwork(ns *kapi.Namespace) error { +func (bsnc *BaseSecondaryNetworkController) AddNamespaceForSecondaryNetwork(ns *corev1.Namespace) error { klog.Infof("[%s] adding namespace for network %s", ns.Name, bsnc.GetNetworkName()) // Keep track of how long syncs take. start := time.Now() @@ -747,11 +746,11 @@ func (bsnc *BaseSecondaryNetworkController) AddNamespaceForSecondaryNetwork(ns * // ensureNamespaceLockedForSecondaryNetwork locks namespacesMutex, gets/creates an entry for ns, configures OVN nsInfo, // and returns it with its mutex locked. // ns is the name of the namespace, while namespace is the optional k8s namespace object -func (bsnc *BaseSecondaryNetworkController) ensureNamespaceLockedForSecondaryNetwork(ns string, readOnly bool, namespace *kapi.Namespace) (*namespaceInfo, func(), error) { +func (bsnc *BaseSecondaryNetworkController) ensureNamespaceLockedForSecondaryNetwork(ns string, readOnly bool, namespace *corev1.Namespace) (*namespaceInfo, func(), error) { return bsnc.ensureNamespaceLockedCommon(ns, readOnly, namespace, bsnc.getAllNamespacePodAddresses, bsnc.configureNamespaceCommon) } -func (bsnc *BaseSecondaryNetworkController) updateNamespaceForSecondaryNetwork(old, newer *kapi.Namespace) error { +func (bsnc *BaseSecondaryNetworkController) updateNamespaceForSecondaryNetwork(old, newer *corev1.Namespace) error { var errors []error klog.Infof("[%s] updating namespace for network %s", old.Name, bsnc.GetNetworkName()) @@ -777,7 +776,7 @@ func (bsnc *BaseSecondaryNetworkController) updateNamespaceForSecondaryNetwork(o return utilerrors.Join(errors...) } -func (bsnc *BaseSecondaryNetworkController) deleteNamespace4SecondaryNetwork(ns *kapi.Namespace) error { +func (bsnc *BaseSecondaryNetworkController) deleteNamespace4SecondaryNetwork(ns *corev1.Namespace) error { klog.Infof("[%s] deleting namespace for network %s", ns.Name, bsnc.GetNetworkName()) nsInfo, err := bsnc.deleteNamespaceLocked(ns.Name) @@ -859,8 +858,8 @@ func (bsnc *BaseSecondaryNetworkController) WatchIPAMClaims() error { func (oc *BaseSecondaryNetworkController) allowPersistentIPs() bool { return config.OVNKubernetesFeature.EnablePersistentIPs && - util.DoesNetworkRequireIPAM(oc.NetInfo) && - util.AllowsPersistentIPs(oc.NetInfo) + util.DoesNetworkRequireIPAM(oc.GetNetInfo()) && + util.AllowsPersistentIPs(oc.GetNetInfo()) } func (oc *BaseSecondaryNetworkController) getNetworkID() (int, error) { @@ -870,7 +869,7 @@ func (oc *BaseSecondaryNetworkController) getNetworkID() (int, error) { if err != nil { return util.InvalidID, err } - *oc.networkID, err = util.GetNetworkID(nodes, oc.NetInfo) + *oc.networkID, err = util.GetNetworkID(nodes, oc.GetNetInfo()) if err != nil { return util.InvalidID, err } @@ -881,7 +880,7 @@ func (oc *BaseSecondaryNetworkController) getNetworkID() (int, error) { // buildUDNEgressSNAT is used to build the conditional SNAT required on L3 and L2 UDNs to // steer traffic correctly via mp0 when leaving OVN to the host func (bsnc *BaseSecondaryNetworkController) buildUDNEgressSNAT(localPodSubnets []*net.IPNet, outputPort string, - node *kapi.Node) ([]*nbdb.NAT, error) { + node *corev1.Node) ([]*nbdb.NAT, error) { if len(localPodSubnets) == 0 { return nil, nil // nothing to do } diff --git a/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go b/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go index 71d6df0849..0858b13d84 100644 --- a/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go +++ b/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go @@ -125,7 +125,7 @@ func (oc *BaseSecondaryLayer2NetworkController) initializeLogicalSwitch(switchNa excludeSubnets []*net.IPNet, clusterLoadBalancerGroupUUID, switchLoadBalancerGroupUUID string) (*nbdb.LogicalSwitch, error) { logicalSwitch := nbdb.LogicalSwitch{ Name: switchName, - ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(oc.NetInfo), + ExternalIDs: util.GenerateExternalIDsForSwitchOrRouter(oc.GetNetInfo()), } hostSubnets := make([]*net.IPNet, 0, len(clusterSubnets)) diff --git a/go-controller/pkg/ovn/baseline_admin_network_policy_test.go b/go-controller/pkg/ovn/baseline_admin_network_policy_test.go index 7a570475dc..897925be04 100644 --- a/go-controller/pkg/ovn/baseline_admin_network_policy_test.go +++ b/go-controller/pkg/ovn/baseline_admin_network_policy_test.go @@ -50,12 +50,12 @@ func getACLsForBANPRulesWithNamedPorts(banp *anpapi.BaselineAdminNetworkPolicy, aclResults := []*nbdb.ACL{} for i, ingress := range banp.Spec.Ingress { acls := getANPGressACL(anpovn.GetACLActionForBANPRule(ingress.Action), banp.Name, string(libovsdbutil.ACLIngress), - getBANPRulePriority(int32(i)), int32(i), ingress.Ports, namedIPorts, true) + getBANPRulePriority(int32(i)), int32(i), ingress.Ports, namedIPorts, true, DefaultNetworkControllerName) aclResults = append(aclResults, acls...) } for i, egress := range banp.Spec.Egress { acls := getANPGressACL(anpovn.GetACLActionForBANPRule(egress.Action), banp.Name, string(libovsdbutil.ACLEgress), - getBANPRulePriority(int32(i)), int32(i), egress.Ports, namedEPorts, true) + getBANPRulePriority(int32(i)), int32(i), egress.Ports, namedEPorts, true, DefaultNetworkControllerName) aclResults = append(aclResults, acls...) } return aclResults @@ -65,12 +65,12 @@ func getACLsForBANPRules(banp *anpapi.BaselineAdminNetworkPolicy) []*nbdb.ACL { aclResults := []*nbdb.ACL{} for i, ingress := range banp.Spec.Ingress { acls := getANPGressACL(anpovn.GetACLActionForBANPRule(ingress.Action), banp.Name, string(libovsdbutil.ACLIngress), - getBANPRulePriority(int32(i)), int32(i), ingress.Ports, nil, true) + getBANPRulePriority(int32(i)), int32(i), ingress.Ports, nil, true, DefaultNetworkControllerName) aclResults = append(aclResults, acls...) } for i, egress := range banp.Spec.Egress { acls := getANPGressACL(anpovn.GetACLActionForBANPRule(egress.Action), banp.Name, string(libovsdbutil.ACLEgress), - getBANPRulePriority(int32(i)), int32(i), egress.Ports, nil, true) + getBANPRulePriority(int32(i)), int32(i), egress.Ports, nil, true, DefaultNetworkControllerName) aclResults = append(aclResults, acls...) } return aclResults @@ -703,49 +703,91 @@ var _ = ginkgo.Describe("OVN BANP Operations", func() { banpNamespacePeer.ResourceVersion = "3" err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), banpNamespacePeer.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pg = getDefaultPGForANPSubject(banp.Name, nil, newACLs, true) // no ports in PG - expectedDatabaseState = []libovsdbtest.TestData{pg} // namespace address-sets are gone + // network representation for the ANP is fully gone - only thing remaining is the pod rep given in UTs deleting namespace doesn't delete pods + expectedDatabaseState = []libovsdbtest.TestData{} // port group should be deleted + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + gomega.Eventually(fakeOVN.nbClient, "25s").Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("17. re-add the namespaces to re-simulate BANP topology for the default network") + // NOTE (tssurya): Prior to UDNs support, when all namespaces that are matching the ANP either in the subject + // OR peers get deleted, it was treated as a peer/subject deletion and hence only ports from portgroup OR IPs from address/sets + // were deleted while retaining the port-group/address-sets themselves + // However now with implementing the UDNs, the semantics have changed internally - if all namespaces matching the ANP + // are deleted then its taken as that network representation for the ANP got deleted - implementation/cache wise + // Hence if all namespaces are gone matching an ANP that belong to a specific network; its counted as the network + // got deleted and we remove all port-group/address-sets associated with that network for this ANP + // The alternative here is to start watching for network addition/deletion which is just an overhead here not worth the effort + // Hence we have this extra step to add back the peer/subject networks so that we can test the next two steps + // now delete the namespaces + banpNamespaceSubject.ResourceVersion = "4" + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &banpNamespaceSubject, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + banpNamespacePeer.ResourceVersion = "4" + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &banpNamespacePeer, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} for _, acl := range newACLs { acl := acl expectedDatabaseState = append(expectedDatabaseState, acl) } - // delete namespace in unit testing doesn't delete pods; we just need to check port groups and address-sets are updated - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) - peerASIngressRule1v4, peerASIngressRule1v6 = buildBANPAddressSets(banp, 1, []string{}, libovsdbutil.ACLIngress) // address-set will be empty since no pods match it yet - peerASEgressRule1v4, peerASEgressRule1v6 = buildBANPAddressSets(banp, 1, []string{}, libovsdbutil.ACLEgress) // address-set will be empty since no pods match it yet + peerASIngressRule1v4, peerASIngressRule1v6 = buildBANPAddressSets(banp, + 1, []string{t2.podIP}, libovsdbutil.ACLIngress) + peerASEgressRule1v4, peerASEgressRule1v6 = buildBANPAddressSets(banp, + 1, []string{t2.podIP}, libovsdbutil.ACLEgress) expectedDatabaseState = append(expectedDatabaseState, []libovsdbtest.TestData{peerASIngressRule0v4, peerASIngressRule0v6, peerASIngressRule1v4, peerASIngressRule1v6, peerASEgressRule0v4, peerASEgressRule0v6, peerASEgressRule1v4, peerASEgressRule1v6}...) - // NOTE: Address set deletion is deferred for 20 seconds... - gomega.Eventually(fakeOVN.nbClient, "25s").Should(libovsdbtest.HaveData(expectedDatabaseState)) + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + gomega.Eventually(fakeOVN.nbClient, "3s").Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("17. update the BANP by deleting all rules; check if all objects are re-created correctly") + ginkgo.By("18. delete all the pods matching the BANP and check if port group and address-set's are updated") + err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(banpSubjectNamespaceName).Delete(context.TODO(), banpSubjectPodName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOVN.fakeClient.KubeClient.CoreV1().Pods(banpPeerNamespaceName).Delete(context.TODO(), banpPeerPodName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + expectedDatabaseState[0].(*nbdb.PortGroup).Ports = nil // no ports in PG + expectedDatabaseState = expectedDatabaseState[:len(expectedDatabaseState)-3] // no LSPs in PG + // Let us remove the IPs from the peer label AS as it stops matching + expectedDatabaseState[1].(*nbdb.AddressSet).Addresses = []string{} // subject podIP should be removed from namespace address-set + expectedDatabaseState[2].(*nbdb.AddressSet).Addresses = []string{} // subject podIP should be removed from namespace address-set + expectedDatabaseState[3].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from namespace address-set + expectedDatabaseState[4].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from namespace-addresset + expectedDatabaseState[len(expectedDatabaseState)-1].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from ANP rule address-set + expectedDatabaseState[len(expectedDatabaseState)-2].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from ANP rule address-set + expectedDatabaseState[len(expectedDatabaseState)-5].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from ANP rule address-set + expectedDatabaseState[len(expectedDatabaseState)-6].(*nbdb.AddressSet).Addresses = []string{} // peer podIP should be removed from ANP rule address-set + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{}, []string{node1Name})...) + gomega.Eventually(fakeOVN.nbClient, "3s").Should(libovsdbtest.HaveData(expectedDatabaseState)) + + ginkgo.By("19. update the BANP by deleting all rules; check if all objects are re-created correctly") banp.Spec.Ingress = []anpapi.BaselineAdminNetworkPolicyIngressRule{} banp.Spec.Egress = []anpapi.BaselineAdminNetworkPolicyEgressRule{} banp.ResourceVersion = "7" banp, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().BaselineAdminNetworkPolicies().Update(context.TODO(), banp, metav1.UpdateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) pg = getDefaultPGForANPSubject(banp.Name, nil, nil, true) // no ports and acls in PG - expectedDatabaseState = []libovsdbtest.TestData{pg} - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) - gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + subjectNSASIPv4, subjectNSASIPv6 = buildNamespaceAddressSets(banpSubjectNamespaceName, []string{}) + peerNSASIPv4, peerNSASIPv6 = buildNamespaceAddressSets(banpPeerNamespaceName, []string{}) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{}, []string{node1Name})...) + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState...)) - ginkgo.By("18. delete the BANP; check if all objects are re-created correctly") + ginkgo.By("20. delete the BANP; check if all objects are re-created correctly") banp.ResourceVersion = "8" err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().BaselineAdminNetworkPolicies().Delete(context.TODO(), banp.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedDatabaseState = []libovsdbtest.TestData{} // port group should be deleted - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + expectedDatabaseState = []libovsdbtest.TestData{subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} // port group should be deleted + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{}, []string{node1Name})...) gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - ginkgo.By("19. create new BANP with same name; check if all objects are re-created correctly; test's cache is cleared and then repopulated properly") + ginkgo.By("21. create new BANP with same name; check if all objects are re-created correctly; test's cache is cleared and then repopulated properly") banp2 := newBANPObject("harry-potter", banpSubject, []anpapi.BaselineAdminNetworkPolicyIngressRule{}, []anpapi.BaselineAdminNetworkPolicyEgressRule{}) banp2.ResourceVersion = "1" banp2, err = fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().BaselineAdminNetworkPolicies().Create(context.TODO(), banp, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // TODO: Check if clone methods did the right thing - do the test in the admin_network_policy_package pg = getDefaultPGForANPSubject(banp2.Name, nil, nil, true) - expectedDatabaseState = []libovsdbtest.TestData{pg} // port-group should be recreated - expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{t, t2}, []string{node1Name})...) + expectedDatabaseState = []libovsdbtest.TestData{pg, subjectNSASIPv4, subjectNSASIPv6, peerNSASIPv4, peerNSASIPv6} // port-group should be recreated + expectedDatabaseState = append(expectedDatabaseState, getDefaultNetExpectedPodsAndSwitches([]testPod{}, []string{node1Name})...) gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) return nil @@ -773,7 +815,7 @@ var _ = ginkgo.Describe("OVN BANP Operations", func() { klog.Infof("Got an update spec action for %v", update.GetObject()) return false, update.GetObject(), nil }) - ginkgo.By("1. Create BANP with 1 ingress rule and 1 egress rule with the ACL logging annotation and ensure its honoured") + ginkgo.By("1. Create BANP with 1 ingress rule and 1 egress rule with the ACL logging annotation and ensure its honoured when the network representation is present") anpSubject := newANPSubjectObject( &metav1.LabelSelector{ MatchLabels: anpLabel, @@ -814,6 +856,12 @@ var _ = ginkgo.Describe("OVN BANP Operations", func() { } banp, err := fakeOVN.fakeClient.ANPClient.PolicyV1alpha1().BaselineAdminNetworkPolicies().Create(context.TODO(), banp, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveEmptyData()) + + ginkgo.By("1.1. Create the matching namespace so that network's representation is honoured") + banpSubjectNamespace := *newNamespaceWithLabels(banpSubjectNamespaceName, anpLabel) + _, err = fakeOVN.fakeClient.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &banpSubjectNamespace, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) acls := getACLsForBANPRules(banp) pg := getDefaultPGForANPSubject(banp.Name, []string{}, acls, true) expectedDatabaseState := []libovsdbtest.TestData{pg} diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy.go b/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy.go index 1520b4561e..cdb49919da 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy.go @@ -14,6 +14,7 @@ import ( libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -121,11 +122,9 @@ func (c *Controller) ensureAdminNetworkPolicy(anp *anpapi.AdminNetworkPolicy) er // fetch the anpState from our cache if it exists currentANPState, loaded := c.anpCache[anp.Name] // Based on the latest kapi ANP, namespace and pod objects: - // 1) Construct Port Group name using ANP name and ports of pods in ANP subject - // 2) Construct Address-sets with IPs of the peers in the rules - // 3) Construct ACLs using AS-es and PGs - portGroupName := c.getANPPortGroupName(desiredANPState.name, false) - + // 1) Construct Address-sets with IPs of the peers in the rules + // 2) Construct ACLs using AS-es and PGs + // for all networks in the cluster desiredPorts, err := c.convertANPSubjectToLSPs(desiredANPState) if err != nil { return fmt.Errorf("unable to fetch ports for anp %s: %v", desiredANPState.name, err) @@ -135,7 +134,7 @@ func (c *Controller) ensureAdminNetworkPolicy(anp *anpapi.AdminNetworkPolicy) er return fmt.Errorf("unable to convert peers to addresses for anp %s: %v", desiredANPState.name, err) } atLeastOneRuleUpdated := false - desiredACLs := c.convertANPRulesToACLs(desiredANPState, currentANPState, portGroupName, &atLeastOneRuleUpdated, false) + desiredACLs := c.convertANPRulesToACLs(desiredANPState, currentANPState, &atLeastOneRuleUpdated, false) if !loaded { // this is a fresh ANP create @@ -206,17 +205,16 @@ func (c *Controller) ensureAdminNetworkPolicy(anp *anpapi.AdminNetworkPolicy) er // convertANPRulesToACLs takes all the rules belonging to the ANP and initiates the conversion of rule->acl // if currentANPState exists; then we also see if any of the current v/s desired ACLs had a state change // and if so, we return atLeastOneRuleUpdated=true -func (c *Controller) convertANPRulesToACLs(desiredANPState, currentANPState *adminNetworkPolicyState, pgName string, - atLeastOneRuleUpdated *bool, isBanp bool) []*nbdb.ACL { - acls := []*nbdb.ACL{} +func (c *Controller) convertANPRulesToACLs(desiredANPState, currentANPState *adminNetworkPolicyState, + atLeastOneRuleUpdated *bool, isBanp bool) map[string][]*nbdb.ACL { + acls := make(map[string][]*nbdb.ACL) // isAtLeastOneRuleUpdatedCheckRequired is set to true, if we had an anp already in cache (update) AND the rule lengths are the same // if the rule lengths are different we do a full peer recompute in ensureAdminNetworkPolicy anyways isAtLeastOneRuleUpdatedCheckRequired := (currentANPState != nil && currentANPState.name != "" && len(currentANPState.ingressRules) == len(desiredANPState.ingressRules) && len(currentANPState.egressRules) == len(desiredANPState.egressRules)) for i, ingressRule := range desiredANPState.ingressRules { - acl := c.convertANPRuleToACL(ingressRule, pgName, desiredANPState.name, desiredANPState.aclLoggingParams, isBanp) - acls = append(acls, acl...) + c.convertANPRuleToACL(ingressRule, desiredANPState.name, desiredANPState.managedNetworks, desiredANPState.aclLoggingParams, isBanp, acls) if isAtLeastOneRuleUpdatedCheckRequired && !*atLeastOneRuleUpdated && (ingressRule.action != currentANPState.ingressRules[i].action || @@ -227,8 +225,7 @@ func (c *Controller) convertANPRulesToACLs(desiredANPState, currentANPState *adm } } for i, egressRule := range desiredANPState.egressRules { - acl := c.convertANPRuleToACL(egressRule, pgName, desiredANPState.name, desiredANPState.aclLoggingParams, isBanp) - acls = append(acls, acl...) + c.convertANPRuleToACL(egressRule, desiredANPState.name, desiredANPState.managedNetworks, desiredANPState.aclLoggingParams, isBanp, acls) if isAtLeastOneRuleUpdatedCheckRequired && !*atLeastOneRuleUpdated && (egressRule.action != currentANPState.egressRules[i].action || @@ -244,16 +241,11 @@ func (c *Controller) convertANPRulesToACLs(desiredANPState, currentANPState *adm // convertANPRuleToACL takes the given gressRule and converts it into an ACL(0 ports rule) or // multiple ACLs(ports are set) and returns those ACLs for a given gressRule -func (c *Controller) convertANPRuleToACL(rule *gressRule, pgName, anpName string, aclLoggingParams *libovsdbutil.ACLLoggingLevels, isBanp bool) []*nbdb.ACL { +func (c *Controller) convertANPRuleToACL(rule *gressRule, anpName string, managedNetworks sets.Set[string], + aclLoggingParams *libovsdbutil.ACLLoggingLevels, isBanp bool, acls map[string][]*nbdb.ACL) { klog.V(5).Infof("Creating ACL for rule %d/%s belonging to ANP %s", rule.priority, rule.gressPrefix, anpName) - // create match based on direction and address-set name - asIndex := GetANPPeerAddrSetDbIDs(anpName, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), c.controllerName, isBanp) - l3Match := constructMatchFromAddressSet(rule.gressPrefix, asIndex) - // create match based on rule type (ingress/egress) and port-group - lportMatch := libovsdbutil.GetACLMatch(pgName, "", libovsdbutil.ACLDirection(rule.gressPrefix)) var match string hasNamedPorts := len(rule.namedPorts) > 0 - acls := []*nbdb.ACL{} // We will have // - one single ACL if len(rule.ports) == 0 && len(rule.namedPorts) == 0 // - one ACL per protocol if len(rule.ports) > 0 and len(rule.namedPorts) == 0 @@ -262,44 +254,67 @@ func (c *Controller) convertANPRuleToACL(rule *gressRule, pgName, anpName string // (so max 3 ACLs (tcp,udp,sctp) per rule {namedPort type ports ONLY}) // - one ACL per protocol if len(rule.ports) > 0 and one ACL per protocol if len(rule.namedPorts) > 0 // (so max 6 ACLs (2tcp,2udp,2sctp) per rule {{portNumber, portRange, namedPorts ALL PRESENT}}) + // for each network matching this ANP for protocol, l4Match := range libovsdbutil.GetL4MatchesFromNetworkPolicyPorts(rule.ports) { - if l4Match == libovsdbutil.UnspecifiedL4Match { - if hasNamedPorts { - continue - } // if we have namedPorts we shouldn't add the noneProtocol ACL even if the namedPort doesn't match any pods - match = fmt.Sprintf("%s && %s", lportMatch, l3Match) - } else { - match = fmt.Sprintf("%s && %s && %s", lportMatch, l3Match, l4Match) - } - acl := libovsdbutil.BuildANPACL( - getANPRuleACLDbIDs(anpName, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), protocol, c.controllerName, isBanp), - int(rule.priority), - match, - rule.action, - libovsdbutil.ACLDirectionToACLPipeline(libovsdbutil.ACLDirection(rule.gressPrefix)), - aclLoggingParams, - ) - acls = append(acls, acl) + for netName := range managedNetworks { + // create match based on direction and address-set name + asIndex := GetANPPeerAddrSetDbIDs(anpName, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), netName, isBanp) + l3Match := constructMatchFromAddressSet(rule.gressPrefix, asIndex) + // create match based on rule type (ingress/egress) and port-group + pgName := c.getANPPortGroupName(anpName, netName, isBanp) + lportMatch := libovsdbutil.GetACLMatch(pgName, "", libovsdbutil.ACLDirection(rule.gressPrefix)) + if l4Match == libovsdbutil.UnspecifiedL4Match { + if hasNamedPorts { + continue + } // if we have namedPorts we shouldn't add the noneProtocol ACL even if the namedPort doesn't match any pods + match = fmt.Sprintf("%s && %s", lportMatch, l3Match) + } else { + match = fmt.Sprintf("%s && %s && %s", lportMatch, l3Match, l4Match) + } + acl := libovsdbutil.BuildANPACL( + getANPRuleACLDbIDs(anpName, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), protocol, netName, isBanp), + int(rule.priority), + match, + rule.action, + libovsdbutil.ACLDirectionToACLPipeline(libovsdbutil.ACLDirection(rule.gressPrefix)), + aclLoggingParams, + ) + _, ok := acls[netName] + if !ok { + acls[netName] = []*nbdb.ACL{} + } + acls[netName] = append(acls[netName], acl) + } } // Process match for NamedPorts if any for protocol, l3l4Match := range libovsdbutil.GetL3L4MatchesFromNamedPorts(rule.namedPorts) { - if rule.gressPrefix == string(libovsdbutil.ACLIngress) { - match = fmt.Sprintf("%s && %s", l3Match, l3l4Match) - } else { - match = fmt.Sprintf("%s && %s", lportMatch, l3l4Match) + for netName := range managedNetworks { + // create match based on direction and address-set name + asIndex := GetANPPeerAddrSetDbIDs(anpName, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), netName, isBanp) + l3Match := constructMatchFromAddressSet(rule.gressPrefix, asIndex) + // create match based on rule type (ingress/egress) and port-group + pgName := c.getANPPortGroupName(anpName, netName, isBanp) + lportMatch := libovsdbutil.GetACLMatch(pgName, "", libovsdbutil.ACLDirection(rule.gressPrefix)) + if rule.gressPrefix == string(libovsdbutil.ACLIngress) { + match = fmt.Sprintf("%s && %s", l3Match, l3l4Match) + } else { + match = fmt.Sprintf("%s && %s", lportMatch, l3l4Match) + } + acl := libovsdbutil.BuildANPACL( + getANPRuleACLDbIDs(anpName, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), protocol+libovsdbutil.NamedPortL4MatchSuffix, netName, isBanp), + int(rule.priority), + match, + rule.action, + libovsdbutil.ACLDirectionToACLPipeline(libovsdbutil.ACLDirection(rule.gressPrefix)), + aclLoggingParams, + ) + _, ok := acls[netName] + if !ok { + acls[netName] = []*nbdb.ACL{} + } + acls[netName] = append(acls[netName], acl) } - acl := libovsdbutil.BuildANPACL( - getANPRuleACLDbIDs(anpName, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), protocol+libovsdbutil.NamedPortL4MatchSuffix, c.controllerName, isBanp), - int(rule.priority), - match, - rule.action, - libovsdbutil.ACLDirectionToACLPipeline(libovsdbutil.ACLDirection(rule.gressPrefix)), - aclLoggingParams, - ) - acls = append(acls, acl) } - - return acls } // expandANPRulePeers takes all the peers belonging to each of the ANP rule and initiates the conversion @@ -307,14 +322,14 @@ func (c *Controller) convertANPRuleToACL(rule *gressRule, pgName, anpName string func (c *Controller) expandANPRulePeers(anp *adminNetworkPolicyState) error { var err error for _, ingressRule := range anp.ingressRules { - err = c.expandRulePeers(ingressRule) // namedPorts has to be processed for subject in case of ingress rules + err = c.expandRulePeers(ingressRule, anp.managedNetworks) // namedPorts has to be processed for subject in case of ingress rules if err != nil { return fmt.Errorf("unable to create address set for "+ " rule %s with priority %d: %w", ingressRule.name, ingressRule.priority, err) } } for _, egressRule := range anp.egressRules { - err = c.expandRulePeers(egressRule) + err = c.expandRulePeers(egressRule, anp.managedNetworks) if err != nil { return fmt.Errorf("unable to create address set for "+ " rule %s with priority %d: %w", egressRule.name, egressRule.priority, err) @@ -328,7 +343,8 @@ func (c *Controller) expandANPRulePeers(anp *adminNetworkPolicyState) error { // This function also takes care of populating the adminNetworkPolicyPeer.namespaces cache // It also adds up all the peerAddresses that are supposed to be present in the created AddressSet and returns them on // a per-rule basis so that the actual ops to transact these into the AddressSet can be constructed using that -func (c *Controller) expandRulePeers(rule *gressRule) error { +func (c *Controller) expandRulePeers(rule *gressRule, managedNetworks sets.Set[string]) error { + networkScopedPeerAddresses := make(map[string]sets.Set[string]) for _, peer := range rule.peers { namespaces, err := c.anpNamespaceLister.List(peer.namespaceSelector) if err != nil { @@ -347,12 +363,22 @@ func (c *Controller) expandRulePeers(rule *gressRule) error { if err != nil { return err } + netInfo, err := c.networkManager.GetActiveNetworkForNamespace(namespace.Name) + if err != nil { + return err + } + managedNetworks.Insert(netInfo.GetNetworkName()) + peerAddresses, ok := networkScopedPeerAddresses[netInfo.GetNetworkName()] + if !ok { + peerAddresses = sets.Set[string]{} + networkScopedPeerAddresses[netInfo.GetNetworkName()] = peerAddresses + } for _, pod := range pods { // we don't handle HostNetworked or completed pods; unscheduled pods shall be handled via pod update path if util.PodWantsHostNetwork(pod) || util.PodCompleted(pod) || !util.PodScheduled(pod) { continue } - podIPs, err := util.GetPodIPsOfNetwork(pod, &util.DefaultNetInfo{}) + podIPs, err := util.GetPodIPsOfNetwork(pod, netInfo) if err != nil { if errors.Is(err, util.ErrNoPodIPFound) { // we ignore podIPsNotFound error here because onANPPodUpdate @@ -362,7 +388,7 @@ func (c *Controller) expandRulePeers(rule *gressRule) error { } return err // we won't hit this TBH because the only error that GetPodIPsOfNetwork returns is podIPsNotFound } - rule.peerAddresses.Insert(util.StringSlice(podIPs)...) + peerAddresses.Insert(util.StringSlice(podIPs)...) podCache.Insert(pod.Name) // Process NamedPorts if any if len(rule.namedPorts) == 0 { @@ -398,21 +424,42 @@ func (c *Controller) expandRulePeers(rule *gressRule) error { if err != nil { // Annotation not found errors are ignored, they will come as node updates return err } - rule.peerAddresses.Insert(nodeIPs...) + // for each network's set of peerIPs, add the nodeIPs + for networkName := range managedNetworks { + _, ok := networkScopedPeerAddresses[networkName] + if !ok { + // means a representation exists for this network matching a subject namespace since subject is processed first + // so let's add the nodeIPs as peers + networkScopedPeerAddresses[networkName] = make(sets.Set[string]) + } + networkScopedPeerAddresses[networkName].Insert(nodeIPs...) + } nodeCache.Insert(node.Name) } peer.nodes = nodeCache + // for each network's set of peerIPs, add the CIDR ranges + for networkName := range managedNetworks { + _, ok := networkScopedPeerAddresses[networkName] + if !ok { + // means a representation exists for this network matching a subject namespace since subject is processed first + // so let's add the nodeIPs as peers + networkScopedPeerAddresses[networkName] = make(sets.Set[string]) + } + networkScopedPeerAddresses[networkName].Insert(peer.networks.UnsortedList()...) + } } + rule.peerAddresses = networkScopedPeerAddresses return nil } -// convertANPSubjectToLSPs calculates all the LSP's that match for the provided anp's subject and returns them +// convertANPSubjectToLSPs calculates all the LSP's that match for the provided anp's subject across all networks +// and returns them // It also populates the adminNetworkPolicySubject.namespaces and adminNetworkPolicySubject.podPorts // pieces of the cache // Since we have to loop through all the pods here, we also take the opportunity to update our namedPorts cache -func (c *Controller) convertANPSubjectToLSPs(anp *adminNetworkPolicyState) ([]*nbdb.LogicalSwitchPort, error) { - lsports := []*nbdb.LogicalSwitchPort{} - anp.subject.podPorts = sets.Set[string]{} +func (c *Controller) convertANPSubjectToLSPs(anp *adminNetworkPolicyState) (map[string][]*nbdb.LogicalSwitchPort, error) { + networkScopedLSPs := make(map[string][]*nbdb.LogicalSwitchPort) + networkScopedPodPorts := make(map[string]sets.Set[string]) namespaces, err := c.anpNamespaceLister.List(anp.subject.namespaceSelector) if err != nil { return nil, err @@ -441,11 +488,39 @@ func (c *Controller) convertANPSubjectToLSPs(anp *adminNetworkPolicyState) ([]*n if err != nil { return nil, err } + netInfo, err := c.networkManager.GetActiveNetworkForNamespace(namespace.Name) + if err != nil { + return nil, err + } + anp.managedNetworks.Insert(netInfo.GetNetworkName()) + podPorts, ok := networkScopedPodPorts[netInfo.GetNetworkName()] + if !ok { + podPorts = sets.Set[string]{} + networkScopedPodPorts[netInfo.GetNetworkName()] = podPorts + } + lsPorts, ok := networkScopedLSPs[netInfo.GetNetworkName()] + if !ok { + lsPorts = []*nbdb.LogicalSwitchPort{} + } for _, pod := range pods { if util.PodWantsHostNetwork(pod) || util.PodCompleted(pod) || !util.PodScheduled(pod) || !c.isPodScheduledinLocalZone(pod) { continue } - logicalPortName := util.GetLogicalPortName(pod.Namespace, pod.Name) + var logicalPortName string + if netInfo.IsDefault() { + logicalPortName = util.GetLogicalPortName(pod.Namespace, pod.Name) + } else { + nadNames, err := util.PodNadNames(pod, netInfo) + if err != nil { + return nil, err + } + if len(nadNames) == 0 { + return nil, fmt.Errorf("pod %s/%s must contain network attach definition for its user defined network %s", + pod.Namespace, pod.Name, netInfo.GetNetworkName()) + } + logicalPortName = util.GetSecondaryNetworkLogicalPortName(pod.Namespace, pod.Name, nadNames[0]) + } + lsp := &nbdb.LogicalSwitchPort{Name: logicalPortName} lsp, err = libovsdbops.GetLogicalSwitchPort(c.nbClient, lsp) if err != nil { @@ -463,8 +538,8 @@ func (c *Controller) convertANPSubjectToLSPs(anp *adminNetworkPolicyState) ([]*n return nil, fmt.Errorf("error retrieving logical switch port with name %s "+ " from libovsdb cache: %w", logicalPortName, err) } - lsports = append(lsports, lsp) - anp.subject.podPorts.Insert(lsp.UUID) + lsPorts = append(lsPorts, lsp) + podPorts.Insert(lsp.UUID) podCache.Insert(pod.Name) if len(namedPortMatchingRulesIndexes) == 0 { continue @@ -496,6 +571,7 @@ func (c *Controller) convertANPSubjectToLSPs(anp *adminNetworkPolicyState) ([]*n } } } + networkScopedLSPs[netInfo.GetNetworkName()] = lsPorts } // we have to store the sorted slice here because in convertANPRulesToACLs // we use DeepEqual to compare ports which doesn't do well with unordered slices @@ -505,8 +581,9 @@ func (c *Controller) convertANPSubjectToLSPs(anp *adminNetworkPolicyState) ([]*n } } anp.subject.namespaces = namespaceCache + anp.subject.podPorts = networkScopedPodPorts - return lsports, nil + return networkScopedLSPs, nil } // clearAdminNetworkPolicy will handle the logic for deleting all db objects related @@ -523,12 +600,16 @@ func (c *Controller) clearAdminNetworkPolicy(anpName string) error { // clear NBDB objects for the given ANP (PG, ACLs on that PG, AddrSets used by the ACLs) var err error - // remove PG for Subject (ACLs will get cleaned up automatically) - portGroupName := c.getANPPortGroupName(anp.name, false) + // remove PG for Subject (ACLs will get cleaned up automatically) across all networks + predicateIDs := libovsdbops.NewDbObjectIDsAcrossAllContollers(libovsdbops.AddressSetAdminNetworkPolicy, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: anp.name, + }) + pgPredicate := libovsdbops.GetPredicateAcrossAllControllers[*nbdb.PortGroup](predicateIDs, nil) // no need to batch this with address-set deletes since this itself will contain a bunch of ACLs that need to be deleted which is heavy enough. - err = libovsdbops.DeletePortGroups(c.nbClient, portGroupName) + err = libovsdbops.DeletePortGroupsWithPredicate(c.nbClient, pgPredicate) if err != nil { - return fmt.Errorf("unable to delete PG %s for ANP %s: %w", portGroupName, anp.name, err) + return fmt.Errorf("unable to delete PGs for ANP %s: %w", anp.name, err) } // remove address-sets that were created for the peers of each rule fpr the whole ANP // do this after ACLs are gone so that there is no lingering references @@ -549,11 +630,11 @@ func (c *Controller) clearAdminNetworkPolicy(anpName string) error { // clearASForPeers takes the externalID objectIDs and uses them to delete all the address-sets // that were owned by anpName func (c *Controller) clearASForPeers(anpName string, idType *libovsdbops.ObjectIDsType) error { - predicateIDs := libovsdbops.NewDbObjectIDs(idType, c.controllerName, + predicateIDs := libovsdbops.NewDbObjectIDsAcrossAllContollers(idType, map[libovsdbops.ExternalIDKey]string{ libovsdbops.ObjectNameKey: anpName, }) - asPredicate := libovsdbops.GetPredicate[*nbdb.AddressSet](predicateIDs, nil) + asPredicate := libovsdbops.GetPredicateAcrossAllControllers[*nbdb.AddressSet](predicateIDs, nil) if err := libovsdbops.DeleteAddressSetsWithPredicate(c.nbClient, asPredicate); err != nil { return fmt.Errorf("failed to destroy address-set for ANP %s, err: %v", anpName, err) } @@ -561,26 +642,34 @@ func (c *Controller) clearASForPeers(anpName string, idType *libovsdbops.ObjectI } // createNewANP takes the desired state of the anp and creates the corresponding objects in the NBDB -func (c *Controller) createNewANP(desiredANPState *adminNetworkPolicyState, desiredACLs []*nbdb.ACL, - desiredPorts []*nbdb.LogicalSwitchPort, isBanp bool) error { +func (c *Controller) createNewANP(desiredANPState *adminNetworkPolicyState, desiredACLs map[string][]*nbdb.ACL, + desiredPorts map[string][]*nbdb.LogicalSwitchPort, isBanp bool) error { ops := []ovsdb.Operation{} // now CreateOrUpdate the address-sets; add the right IPs - we treat the rest of the address-set cases as a fresh add or update - addrSetOps, err := c.constructOpsForRuleChanges(desiredANPState, isBanp) + addrSetOps, err := c.constructOpsForRuleChanges(desiredANPState, isBanp, nil) if err != nil { return fmt.Errorf("failed to create address-sets, %v", err) } ops = append(ops, addrSetOps...) - ops, err = libovsdbops.CreateOrUpdateACLsOps(c.nbClient, ops, c.GetSamplingConfig(), desiredACLs...) + var aclsAcrossAllNetworks []*nbdb.ACL + for _, acls := range desiredACLs { + aclsAcrossAllNetworks = append(aclsAcrossAllNetworks, acls...) + } + ops, err = libovsdbops.CreateOrUpdateACLsOps(c.nbClient, ops, c.GetSamplingConfig(), aclsAcrossAllNetworks...) if err != nil { return fmt.Errorf("failed to create ACL ops: %v", err) } - pgDbIDs := GetANPPortGroupDbIDs(desiredANPState.name, isBanp, c.controllerName) - pg := libovsdbutil.BuildPortGroup(pgDbIDs, desiredPorts, desiredACLs) - ops, err = libovsdbops.CreateOrUpdatePortGroupsOps(c.nbClient, ops, pg) - if err != nil { - return fmt.Errorf("failed to create ops to add port to a port group: %v", err) + // For a given ANP there will be 1PG representation for each network in the cluster + for networkName := range desiredANPState.managedNetworks { + pgDbIDs := GetANPPortGroupDbIDs(desiredANPState.name, isBanp, networkName) + pg := libovsdbutil.BuildPortGroup(pgDbIDs, desiredPorts[networkName], desiredACLs[networkName]) + ops, err = libovsdbops.CreateOrUpdatePortGroupsOps(c.nbClient, ops, pg) + if err != nil { + return fmt.Errorf("failed to create ops to add port to a port group: %v", err) + } } + _, err = libovsdbops.TransactAndCheck(c.nbClient, ops) if err != nil { return fmt.Errorf("failed to run ovsdb txn to add ports to port group: %v", err) @@ -589,24 +678,34 @@ func (c *Controller) createNewANP(desiredANPState *adminNetworkPolicyState, desi } func (c *Controller) updateExistingANP(currentANPState, desiredANPState *adminNetworkPolicyState, atLeastOneRuleUpdated, - hasPriorityChanged, isBanp bool, desiredACLs []*nbdb.ACL) error { + hasPriorityChanged, isBanp bool, desiredACLs map[string][]*nbdb.ACL) error { var ops []ovsdb.Operation var err error - portGroupName := c.getANPPortGroupName(desiredANPState.name, isBanp) - // Did ANP.Spec.Ingress Change (rule inserts/deletes)? && || Did ANP.Spec.Egress Change (rule inserts/deletes)? && || - // If yes we need to fully recompute the acls present in our ANP's port group; Let's do a full recompute and return. + // Based on network adds/deletes the port-groups will have to be added/deleted + // Hence we need to transact the ops for port-group changes first so that + // subsequent ACL/Port changes all reference existing port-groups + // NOTE that this will also remove any ACLs for the deleted networks + networksToAdd := desiredANPState.managedNetworks.Difference(currentANPState.managedNetworks) + networksToDelete := currentANPState.managedNetworks.Difference(desiredANPState.managedNetworks) + if err := c.updatePortGroupsForNetworkChanges(networksToAdd, networksToDelete, desiredANPState.name, isBanp); err != nil { + return fmt.Errorf("failed to create or delete port groups for ANP %s, err: %v", desiredANPState.name, err) + } + // Did ANP.Spec.Ingress Change (rule inserts/deletes)? && || Did ANP.Spec.Egress Change (rule inserts/deletes)? && || networkChanges? + // If yes we need to fully recompute the acls present in our ANP's port groups for each network; Let's do a full recompute and return. + // constructOpsForRuleChanges updates all the address-sets for our ANPs across all networks + // NOTE that it is also called to create new address-sets for newly added networks + // and delete address-sets for deleted networks => for rest of the networks it will be no-op // Reason behind a full recompute: Each rule has precedence based on its position and priority of ANP; if any of that changes - // better to delete and recreate ACLs rather than figure out from caches - // rather than always cleaning up everything and recreating them. But this is tricky since rules have precedence - // from their ordering. - // NOTE: Changes to admin policies should be a rare action (can be improved post user feedback) - usually churn would be around namespaces and pods + // better to delete and recreate ACLs rather than figure out from caches a diff of what needs change since that would be more complicated + // NOTE: Changes to admin policies should be a rare action (so this can be improved post user feedback) - usually churn would be around namespaces and pods fullPeerRecompute := (len(currentANPState.egressRules) != len(desiredANPState.egressRules) || len(currentANPState.ingressRules) != len(desiredANPState.ingressRules)) - if fullPeerRecompute { + networkChanges := len(networksToAdd) > 0 || len(networksToDelete) > 0 + if fullPeerRecompute || networkChanges { // full recompute // which means update all ACLs and address-sets klog.V(3).Infof("ANP %s with priority (old %d, new %d) was updated", desiredANPState.name, currentANPState.anpPriority, desiredANPState.anpPriority) - ops, err = c.constructOpsForRuleChanges(desiredANPState, isBanp) + ops, err = c.constructOpsForRuleChanges(desiredANPState, isBanp, networksToDelete) if err != nil { return fmt.Errorf("failed to create update ANP ops %s: %v", desiredANPState.name, err) } @@ -623,10 +722,10 @@ func (c *Controller) updateExistingANP(currentANPState, desiredANPState *adminNe // 2) ANP.Spec.Ingress.Peers.Pods changed && || // 3) A namespace started or stopped matching the peer && || // 4) A pod started or stopped matching the peer - // If yes we need to recompute the IPs present in our ANP's peer's address-sets - if !fullPeerRecompute && !reflect.DeepEqual(desiredANPState.ingressRules, currentANPState.ingressRules) { + // If yes we need to recompute the IPs present in our ANP's peer's address-sets for all networks + if !fullPeerRecompute && !networkChanges && !reflect.DeepEqual(desiredANPState.ingressRules, currentANPState.ingressRules) { addrOps, err := c.constructOpsForPeerChanges(desiredANPState.ingressRules, - currentANPState.ingressRules, desiredANPState.name, isBanp) + currentANPState.ingressRules, desiredANPState.name, isBanp, desiredANPState.managedNetworks) if err != nil { return fmt.Errorf("failed to create ops for changes to ANP ingress peers: %v", err) } @@ -646,10 +745,10 @@ func (c *Controller) updateExistingANP(currentANPState, desiredANPState *adminNe // 4) A namespace started or stopped matching the peer && || // 5) A pod started or stopped matching the peer && || // 6) A node started or stopped matching the peer - // If yes we need to recompute the IPs present in our ANP's peer's address-sets - if !fullPeerRecompute && !reflect.DeepEqual(desiredANPState.egressRules, currentANPState.egressRules) { + // If yes we need to recompute the IPs present in our ANP's peer's address-sets for all networks + if !fullPeerRecompute && !networkChanges && !reflect.DeepEqual(desiredANPState.egressRules, currentANPState.egressRules) { addrOps, err := c.constructOpsForPeerChanges(desiredANPState.egressRules, - currentANPState.egressRules, desiredANPState.name, isBanp) + currentANPState.egressRules, desiredANPState.name, isBanp, desiredANPState.managedNetworks) if err != nil { return fmt.Errorf("failed to create ops for changes to ANP egress peers: %v", err) } @@ -669,18 +768,26 @@ func (c *Controller) updateExistingANP(currentANPState, desiredANPState *adminNe // (2) atLeastOneRuleUpdated=true which means the gress rules were of same lengths but action or ports changed on at least one rule // (3) hasPriorityChanged=true which means we should update acl.Priority for every ACL // (4) hasACLLoggingParamsChanged=true which means we should update acl.Severity/acl.Log for every ACL - if fullPeerRecompute || atLeastOneRuleUpdated || hasPriorityChanged || hasACLLoggingParamsChanged { + // (5) len(networksToAdd) > 0 which means new networks were added (delete case is already covered when PGs were deleted) + if fullPeerRecompute || atLeastOneRuleUpdated || hasPriorityChanged || hasACLLoggingParamsChanged || len(networksToAdd) > 0 { klog.V(3).Infof("ANP %s with priority %d was updated", desiredANPState.name, desiredANPState.anpPriority) // now update the acls to the desired ones - ops, err = libovsdbops.CreateOrUpdateACLsOps(c.nbClient, ops, c.GetSamplingConfig(), desiredACLs...) - if err != nil { - return fmt.Errorf("failed to create new ACL ops for anp %s: %v", desiredANPState.name, err) - } - // since we update the portgroup with the new set of ACLs, any unreferenced set of ACLs - // will be automatically removed - ops, err = libovsdbops.UpdatePortGroupSetACLsOps(c.nbClient, ops, portGroupName, desiredACLs) - if err != nil { - return fmt.Errorf("failed to create ACL-on-PG update ops for anp %s: %v", desiredANPState.name, err) + for networkName := range desiredANPState.managedNetworks { + acls, ok := desiredACLs[networkName] + if !ok { + acls = []*nbdb.ACL{} + } + ops, err = libovsdbops.CreateOrUpdateACLsOps(c.nbClient, ops, c.GetSamplingConfig(), acls...) + if err != nil { + return fmt.Errorf("failed to create new ACL ops for anp %s: %v", desiredANPState.name, err) + } + // since we update the portgroup with the new set of ACLs, any unreferenced set of ACLs + // will be automatically removed + portGroupName := c.getANPPortGroupName(desiredANPState.name, networkName, isBanp) + ops, err = libovsdbops.UpdatePortGroupSetACLsOps(c.nbClient, ops, portGroupName, acls) + if err != nil { + return fmt.Errorf("failed to create ACL-on-PG update ops for anp %s: %v", desiredANPState.name, err) + } } } @@ -689,8 +796,8 @@ func (c *Controller) updateExistingANP(currentANPState, desiredANPState *adminNe // 2) ANP.Spec.Pods changed && || // 3) A namespace started or stopped matching the subject && || // 4) A pod started or stopped matching the subject - // If yes we need to recompute the ports present in our ANP's port group - subjectOps, err := c.constructOpsForSubjectChanges(currentANPState, desiredANPState, portGroupName) + // If yes we need to recompute the ports present in our ANP's port groups for all networks + subjectOps, err := c.constructOpsForSubjectChanges(currentANPState, desiredANPState, isBanp) if err != nil { return fmt.Errorf("failed to create ops for changes to ANP %s subject: %v", desiredANPState.name, err) } @@ -702,8 +809,40 @@ func (c *Controller) updateExistingANP(currentANPState, desiredANPState *adminNe return nil } +// updatePortGroupsForNetworkChanges takes the newly added and deleted networks and +// creates/deletes the correspnding port groups for those networks for the given ANP +func (c *Controller) updatePortGroupsForNetworkChanges(networksToAdd, networksToDelete sets.Set[string], + anpName string, isBanp bool) error { + var pgOps []ovsdb.Operation + var err error + // If there are new networks then it means we need to create new PGs + // before we process any ACL or peer updates + for networkName := range networksToAdd { + pgDbIDs := GetANPPortGroupDbIDs(anpName, isBanp, networkName) + pg := libovsdbutil.BuildPortGroup(pgDbIDs, nil, nil) + pgOps, err = libovsdbops.CreateOrUpdatePortGroupsOps(c.nbClient, pgOps, pg) + if err != nil { + return fmt.Errorf("failed to create ops to add port to a port group: %v", err) + } + } + // If there are stale networks then it means we need to delete the PGs + // before we process any ACL or peer updates + for networkName := range networksToDelete { + portGroupName := c.getANPPortGroupName(anpName, networkName, isBanp) + pgOps, err = libovsdbops.DeletePortGroupsOps(c.nbClient, pgOps, portGroupName) + if err != nil { + return fmt.Errorf("failed to create ops to add port to a port group: %v", err) + } + } + if _, err := libovsdbops.TransactAndCheck(c.nbClient, pgOps); err != nil { + return fmt.Errorf("failed to run ovsdb txn to update portgroups for ANP %s: %v", + anpName, err) + } + return nil +} + // constructOpsForRuleChanges takes the desired state of the anp and returns the corresponding ops for updating NBDB objects -func (c *Controller) constructOpsForRuleChanges(desiredANPState *adminNetworkPolicyState, isBanp bool) ([]ovsdb.Operation, error) { +func (c *Controller) constructOpsForRuleChanges(desiredANPState *adminNetworkPolicyState, isBanp bool, networksToDelete sets.Set[string]) ([]ovsdb.Operation, error) { var ops []ovsdb.Operation var err error // Logic to delete address-sets: @@ -714,7 +853,7 @@ func (c *Controller) constructOpsForRuleChanges(desiredANPState *adminNetworkPol if isBanp { idType = libovsdbops.AddressSetBaselineAdminNetworkPolicy } - predicateIDs := libovsdbops.NewDbObjectIDs(idType, c.controllerName, + predicateIDs := libovsdbops.NewDbObjectIDsAcrossAllContollers(idType, map[libovsdbops.ExternalIDKey]string{ libovsdbops.ObjectNameKey: desiredANPState.name, }) @@ -725,31 +864,62 @@ func (c *Controller) constructOpsForRuleChanges(desiredANPState *adminNetworkPol (as.ExternalIDs[libovsdbops.PolicyDirectionKey.String()] == string(libovsdbutil.ACLIngress) && asIndex >= len(desiredANPState.ingressRules)) } - asPredicate := libovsdbops.GetPredicate[*nbdb.AddressSet](predicateIDs, predicateFunc) + asPredicate := libovsdbops.GetPredicateAcrossAllControllers[*nbdb.AddressSet](predicateIDs, predicateFunc) ops, err = libovsdbops.DeleteAddressSetsWithPredicateOps(c.nbClient, ops, asPredicate) if err != nil { return nil, fmt.Errorf("failed to create address-set destroy ops for ANP %s, err: %v", desiredANPState.name, err) } + if len(networksToDelete) > 0 { + for networkName := range networksToDelete { + if networkName == types.DefaultNetworkName { + networkName = defaultNetworkControllerName + } + // network delete event: so let's remove all the address-sets for this network + predicateIDs := libovsdbops.NewDbObjectIDs(idType, networkName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: desiredANPState.name, + }) + asPredicate := libovsdbops.GetPredicate[*nbdb.AddressSet](predicateIDs, nil) + ops, err = libovsdbops.DeleteAddressSetsWithPredicateOps(c.nbClient, ops, asPredicate) + if err != nil { + return nil, fmt.Errorf("failed to create address-set destroy ops for ANP %s, err: %v", desiredANPState.name, err) + } + } + } // TODO (tssurya): Revisit this logic to see if its better to do one address-set per peer instead of one address-set for all peers // Had briefly discussed this OVN team. We are not yet clear which is better since both have advantages and disadvantages. // Decide this after doing some scale runs. for _, rule := range desiredANPState.ingressRules { - asIndex := GetANPPeerAddrSetDbIDs(desiredANPState.name, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), c.controllerName, isBanp) - _, addrSetOps, err := c.addressSetFactory.NewAddressSetOps(asIndex, rule.peerAddresses.UnsortedList()) - if err != nil { - return nil, fmt.Errorf("failed to create address-sets for ANP %s's"+ - " ingress rule %s/%s/%d: %v", desiredANPState.name, rule.name, rule.gressPrefix, rule.priority, err) + for networkName := range desiredANPState.managedNetworks { + peerAddresses, ok := rule.peerAddresses[networkName] + if !ok { + // network add event: then we should create an empty representation of peers + peerAddresses = sets.Set[string]{} + } + asIndex := GetANPPeerAddrSetDbIDs(desiredANPState.name, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), networkName, isBanp) + _, addrSetOps, err := c.addressSetFactory.NewAddressSetOps(asIndex, peerAddresses.UnsortedList()) + if err != nil { + return nil, fmt.Errorf("failed to create address-sets for ANP %s's"+ + " ingress rule %s/%s/%d: %v as part of network %s", desiredANPState.name, rule.name, rule.gressPrefix, rule.priority, err, networkName) + } + ops = append(ops, addrSetOps...) } - ops = append(ops, addrSetOps...) } for _, rule := range desiredANPState.egressRules { - asIndex := GetANPPeerAddrSetDbIDs(desiredANPState.name, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), c.controllerName, isBanp) - _, addrSetOps, err := c.addressSetFactory.NewAddressSetOps(asIndex, rule.peerAddresses.UnsortedList()) - if err != nil { - return nil, fmt.Errorf("failed to create address-sets for ANP %s's"+ - " egress rule %s/%s/%d: %v", desiredANPState.name, rule.name, rule.gressPrefix, rule.priority, err) + for networkName := range desiredANPState.managedNetworks { + peerAddresses, ok := rule.peerAddresses[networkName] + if !ok { + // If this network has no matching peers, then we should create an empty representation of peers + peerAddresses = sets.Set[string]{} + } + asIndex := GetANPPeerAddrSetDbIDs(desiredANPState.name, rule.gressPrefix, fmt.Sprintf("%d", rule.gressIndex), networkName, isBanp) + _, addrSetOps, err := c.addressSetFactory.NewAddressSetOps(asIndex, peerAddresses.UnsortedList()) + if err != nil { + return nil, fmt.Errorf("failed to create address-sets for ANP %s's"+ + " egress rule %s/%s/%d: %v as part of network %s", desiredANPState.name, rule.name, rule.gressPrefix, rule.priority, err, networkName) + } + ops = append(ops, addrSetOps...) } - ops = append(ops, addrSetOps...) } return ops, nil } @@ -758,41 +928,56 @@ func (c *Controller) constructOpsForRuleChanges(desiredANPState *adminNetworkPol // for updating NBDB AddressSet objects for those peers // This should be called if namespace/pod is being created/updated func (c *Controller) constructOpsForPeerChanges(desiredRules, currentRules []*gressRule, - anpName string, isBanp bool) ([]ovsdb.Operation, error) { + anpName string, isBanp bool, managedNetworks sets.Set[string]) ([]ovsdb.Operation, error) { var ops []ovsdb.Operation for i := range desiredRules { desiredRule := desiredRules[i] currentRule := currentRules[i] - addressesToAdd := desiredRule.peerAddresses.Difference(currentRule.peerAddresses) - asIndex := GetANPPeerAddrSetDbIDs(anpName, desiredRule.gressPrefix, fmt.Sprintf("%d", desiredRule.gressIndex), c.controllerName, isBanp) - if len(addressesToAdd) > 0 { - as, err := c.addressSetFactory.GetAddressSet(asIndex) - if err != nil { - return nil, fmt.Errorf("cannot ensure that addressSet %+v exists: err %v", asIndex.GetExternalIDs(), err) + for networkName := range managedNetworks { + desiredPeerAddresses, ok := desiredRule.peerAddresses[networkName] + if !ok { + // empty peers for this network: so let's set the desired value to empty set for this network + desiredPeerAddresses = sets.Set[string]{} } - klog.V(5).Infof("Adding peerAddresses %+v to address-set %s for ANP %s", addressesToAdd, as.GetName(), anpName) - addrOps, err := as.AddAddressesReturnOps(addressesToAdd.UnsortedList()) - if err != nil { - return nil, fmt.Errorf("failed to construct address-set %s's IP add ops for anp %s's rule"+ - " %s/%s/%d: %v", as.GetName(), anpName, desiredRule.name, - desiredRule.gressPrefix, desiredRule.priority, err) + currentPeerAddresses, ok := currentRule.peerAddresses[networkName] + if !ok { + // network add event or empty network peers: so let's set the current value to empty set for this network + currentPeerAddresses = sets.Set[string]{} } - ops = append(ops, addrOps...) - } - addressesToRemove := currentRule.peerAddresses.Difference(desiredRule.peerAddresses) - if len(addressesToRemove) > 0 { - as, err := c.addressSetFactory.GetAddressSet(asIndex) - if err != nil { - return nil, fmt.Errorf("cannot ensure that addressSet %+v exists: err %v", asIndex.GetExternalIDs(), err) + addressesToAdd := desiredPeerAddresses.Difference(currentPeerAddresses) + asIndex := GetANPPeerAddrSetDbIDs(anpName, desiredRule.gressPrefix, + fmt.Sprintf("%d", desiredRule.gressIndex), networkName, isBanp) + if len(addressesToAdd) > 0 { + as, err := c.addressSetFactory.GetAddressSet(asIndex) + if err != nil { + return nil, fmt.Errorf("cannot ensure that addressSet %+v exists: err %v", asIndex.GetExternalIDs(), err) + } + klog.V(5).Infof("Adding peerAddresses %+v to address-set %s for ANP %s for network %s", + addressesToAdd, as.GetName(), anpName, networkName) + addrOps, err := as.AddAddressesReturnOps(addressesToAdd.UnsortedList()) + if err != nil { + return nil, fmt.Errorf("failed to construct address-set %s's IP add ops for anp %s's rule"+ + " %s/%s/%d: %v for network %s", as.GetName(), anpName, desiredRule.name, + desiredRule.gressPrefix, desiredRule.priority, err, networkName) + } + ops = append(ops, addrOps...) } - klog.V(5).Infof("Deleting peerAddresses %+v from address-set %s for ANP %s", addressesToRemove, as.GetName(), anpName) - addrOps, err := as.DeleteAddressesReturnOps(addressesToRemove.UnsortedList()) - if err != nil { - return nil, fmt.Errorf("failed to construct address-set %s's IP delete ops for anp %s's rule"+ - " %s/%s/%d: %v", as.GetName(), anpName, desiredRule.name, - desiredRule.gressPrefix, desiredRule.priority, err) + addressesToRemove := currentPeerAddresses.Difference(desiredPeerAddresses) + if len(addressesToRemove) > 0 { + as, err := c.addressSetFactory.GetAddressSet(asIndex) + if err != nil { + return nil, fmt.Errorf("cannot ensure that addressSet %+v exists: err %v", asIndex.GetExternalIDs(), err) + } + klog.V(5).Infof("Deleting peerAddresses %+v to address-set %s for ANP %s for network %s", + addressesToRemove, as.GetName(), anpName, networkName) + addrOps, err := as.DeleteAddressesReturnOps(addressesToRemove.UnsortedList()) + if err != nil { + return nil, fmt.Errorf("failed to construct address-set %s's IP delete ops for anp %s's rule"+ + " %s/%s/%d: %v", as.GetName(), anpName, desiredRule.name, + desiredRule.gressPrefix, desiredRule.priority, err) + } + ops = append(ops, addrOps...) } - ops = append(ops, addrOps...) } } return ops, nil @@ -800,23 +985,40 @@ func (c *Controller) constructOpsForPeerChanges(desiredRules, currentRules []*gr // constructOpsForSubjectChanges takes the current and desired cache states for a given ANP and returns the ops // required to construct the transact to insert/delete ports to/from port-groups according to the ANP subject changes -func (c *Controller) constructOpsForSubjectChanges(currentANPState, desiredANPState *adminNetworkPolicyState, portGroupName string) ([]ovsdb.Operation, error) { +func (c *Controller) constructOpsForSubjectChanges(currentANPState, desiredANPState *adminNetworkPolicyState, isBanp bool) ([]ovsdb.Operation, error) { var ops []ovsdb.Operation var err error - portsToAdd := desiredANPState.subject.podPorts.Difference(currentANPState.subject.podPorts).UnsortedList() - portsToDelete := currentANPState.subject.podPorts.Difference(desiredANPState.subject.podPorts).UnsortedList() - if len(portsToAdd) > 0 { - klog.V(5).Infof("Adding ports %+v to port-group %s for ANP %s", portsToAdd, portGroupName, desiredANPState.name) - ops, err = libovsdbops.AddPortsToPortGroupOps(c.nbClient, ops, portGroupName, portsToAdd...) - if err != nil { - return nil, fmt.Errorf("failed to create Port-to-PG add ops for anp %s: %v", desiredANPState.name, err) + // loop through all networks to get the state change across all networks so that we can + // construct ops to update per network portgroup of this ANP's subject + // In the beginning of updateExistingANP we have already taken care of network add/delete + // cases for port groups. So here only updates need to be handled since + for networkName := range desiredANPState.managedNetworks { + desiredANPStatePodPorts, ok := desiredANPState.subject.podPorts[networkName] + if !ok { + // means it is a network with empty subjects + desiredANPStatePodPorts = make(sets.Set[string]) } - } - if len(portsToDelete) > 0 { - klog.V(5).Infof("Deleting ports %+v from port-group %s for ANP %s", portsToDelete, portGroupName, desiredANPState.name) - ops, err = libovsdbops.DeletePortsFromPortGroupOps(c.nbClient, ops, portGroupName, portsToDelete...) - if err != nil { - return nil, fmt.Errorf("failed to create Port-from-PG delete ops for anp %s: %v", desiredANPState.name, err) + currentANPStatePodPorts, ok := currentANPState.subject.podPorts[networkName] + if !ok { + // means it is a new network add OR a network that had empty subjects + currentANPStatePodPorts = make(sets.Set[string]) + } + portsToAdd := desiredANPStatePodPorts.Difference(currentANPStatePodPorts).UnsortedList() + portsToDelete := currentANPStatePodPorts.Difference(desiredANPStatePodPorts).UnsortedList() + portGroupName := c.getANPPortGroupName(desiredANPState.name, networkName, isBanp) + if len(portsToAdd) > 0 { + klog.V(5).Infof("Adding ports %+v to port-group %s for ANP %s", portsToAdd, portGroupName, desiredANPState.name) + ops, err = libovsdbops.AddPortsToPortGroupOps(c.nbClient, ops, portGroupName, portsToAdd...) + if err != nil { + return nil, fmt.Errorf("failed to create Port-to-PG add ops for anp %s: %v", desiredANPState.name, err) + } + } + if len(portsToDelete) > 0 { + klog.V(5).Infof("Deleting ports %+v from port-group %s for ANP %s", portsToDelete, portGroupName, desiredANPState.name) + ops, err = libovsdbops.DeletePortsFromPortGroupOps(c.nbClient, ops, portGroupName, portsToDelete...) + if err != nil { + return nil, fmt.Errorf("failed to create Port-from-PG delete ops for anp %s: %v", desiredANPState.name, err) + } } } return ops, nil diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy_controller.go b/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy_controller.go index ea4299d98c..5de2284266 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy_controller.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/admin_network_policy_controller.go @@ -9,6 +9,7 @@ import ( libovsdbclient "github.com/ovn-org/libovsdb/client" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/observability" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -34,7 +35,8 @@ const ( // sequence of delays between successive queuings of an object. // // 5ms, 10ms, 20ms, 40ms, 80ms, 160ms, 320ms, 640ms, 1.3s, 2.6s, 5.1s, 10.2s, 20.4s, 41s, 82s - maxRetries = 15 + maxRetries = 15 + defaultNetworkControllerName = "default-network-controller" ) // Controller holds the fields required for ANP controller @@ -99,6 +101,9 @@ type Controller struct { anpNodeQueue workqueue.TypedRateLimitingInterface[string] observManager *observability.Manager + + // networkManager used for getting network information of (C)UDNs + networkManager networkmanager.Interface } // NewController returns a new *Controller. @@ -115,7 +120,8 @@ func NewController( isPodScheduledinLocalZone func(*v1.Pod) bool, zone string, recorder record.EventRecorder, - observManager *observability.Manager) (*Controller, error) { + observManager *observability.Manager, + networkManager networkmanager.Interface) (*Controller, error) { c := &Controller{ controllerName: controllerName, @@ -128,6 +134,7 @@ func NewController( anpPriorityMap: make(map[int32]string), banpCache: &adminNetworkPolicyState{}, // safe to initialise pointer to empty struct than nil observManager: observManager, + networkManager: networkManager, } klog.V(5).Info("Setting up event handlers for Admin Network Policy") diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/baseline_admin_network_policy.go b/go-controller/pkg/ovn/controller/admin_network_policy/baseline_admin_network_policy.go index 7337084d42..a76e285aef 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/baseline_admin_network_policy.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/baseline_admin_network_policy.go @@ -7,6 +7,7 @@ import ( libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" apierrors "k8s.io/apimachinery/pkg/api/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" @@ -93,15 +94,20 @@ func (c *Controller) clearBaselineAdminNetworkPolicy(banpName string) error { } // clear NBDB objects for the given BANP (PG, ACLs on that PG, AddrSets used by the ACLs) - // remove PG for Subject (ACLs will get cleaned up automatically) - portGroupName := c.getANPPortGroupName(banp.name, true) + // remove PG for Subject (ACLs will get cleaned up automatically) across all networks + predicateIDs := libovsdbops.NewDbObjectIDsAcrossAllContollers(libovsdbops.AddressSetBaselineAdminNetworkPolicy, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: banp.name, + }) + pgPredicate := libovsdbops.GetPredicateAcrossAllControllers[*nbdb.PortGroup](predicateIDs, nil) // no need to batch this with address-set deletes since this itself will contain a bunch of ACLs that need to be deleted which is heavy enough. - err := libovsdbops.DeletePortGroups(c.nbClient, portGroupName) + err := libovsdbops.DeletePortGroupsWithPredicate(c.nbClient, pgPredicate) if err != nil { - return fmt.Errorf("unable to delete PG %s for BANP %s: %w", portGroupName, banp.name, err) + return fmt.Errorf("unable to delete PGs for BANP %s: %w", banpName, err) } - // remove address-sets that were created for the peers of each rule fpr the whole ANP + // remove address-sets that were created for the peers of each rule fpr the whole BANP // do this after ACLs are gone so that there is no lingering references + // do this across all networks for this BANP err = c.clearASForPeers(banp.name, libovsdbops.AddressSetBaselineAdminNetworkPolicy) if err != nil { return fmt.Errorf("failed to delete address-sets for BANP %s: %w", banp.name, err) @@ -124,10 +130,9 @@ func (c *Controller) ensureBaselineAdminNetworkPolicy(banp *anpapi.BaselineAdmin // fetch the banpState from our cache currentBANPState := c.banpCache // Based on the latest kapi BANP, namespace and pod objects: - // 1) Construct Port Group name using ANP name - // 2) Construct Address-sets with IPs of the peers in the rules - // 3) Construct ACLs using AS-es and PGs - portGroupName := c.getANPPortGroupName(desiredBANPState.name, true) + // 1) Construct Address-sets with IPs of the peers in the rules + // 2) Construct ACLs using AS-es and PGs + // acrss all networks in the cluster desiredPorts, err := c.convertANPSubjectToLSPs(desiredBANPState) if err != nil { return fmt.Errorf("unable to fetch ports for banp %s: %v", desiredBANPState.name, err) @@ -137,7 +142,7 @@ func (c *Controller) ensureBaselineAdminNetworkPolicy(banp *anpapi.BaselineAdmin return fmt.Errorf("unable to convert peers to addresses for banp %s: %v", desiredBANPState.name, err) } atLeastOneRuleUpdated := false - desiredACLs := c.convertANPRulesToACLs(desiredBANPState, currentBANPState, portGroupName, &atLeastOneRuleUpdated, true) + desiredACLs := c.convertANPRulesToACLs(desiredBANPState, currentBANPState, &atLeastOneRuleUpdated, true) // Comparing names for figuring out if cache is populated or not is safe // because the singleton BANP will always be called "default" in any cluster diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/repair.go b/go-controller/pkg/ovn/controller/admin_network_policy/repair.go index bec75d6613..f741951140 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/repair.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/repair.go @@ -41,8 +41,9 @@ func (c *Controller) repairAdminNetworkPolicies() error { // We grab all the port groups that belong to ANP controller using externalIDs // and compare the value with the name of existing ANPs. If no match is found // we delete that port group along with all the acls in it. - predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.PortGroupAdminNetworkPolicy, c.controllerName, nil) - p := libovsdbops.GetPredicate[*nbdb.PortGroup](predicateIDs, func(pg *nbdb.PortGroup) bool { + // This is done for all networks in the cluster + predicateIDs := libovsdbops.NewDbObjectIDsAcrossAllContollers(libovsdbops.PortGroupAdminNetworkPolicy, nil) + p := libovsdbops.GetPredicateAcrossAllControllers[*nbdb.PortGroup](predicateIDs, func(pg *nbdb.PortGroup) bool { _, ok := existingANPs[pg.ExternalIDs[libovsdbops.ObjectNameKey.String()]] return !ok // return if it doesn't exist in the cache }) @@ -60,16 +61,16 @@ func (c *Controller) repairAdminNetworkPolicies() error { } // Deal with Address-Sets Repairs // We grab all the AddressSets that belong to ANP controller using externalIDs - // and compare with the existing ANPs. The ones that don't match + // across all networks and compare with the existing ANPs. The ones that don't match // will be deleted from the DB. // NOTE: When we call syncAdminNetworkPolicy function after this for every ANP on startup, // the right Address-sets will be recreated. - asPredicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.AddressSetAdminNetworkPolicy, c.controllerName, nil) + asPredicateIDs := libovsdbops.NewDbObjectIDsAcrossAllContollers(libovsdbops.AddressSetAdminNetworkPolicy, nil) asPredicateFunc := func(as *nbdb.AddressSet) bool { _, ok := existingANPs[as.ExternalIDs[libovsdbops.ObjectNameKey.String()]] return !ok // if not present in cache then its stale } - asPredicate := libovsdbops.GetPredicate[*nbdb.AddressSet](asPredicateIDs, asPredicateFunc) + asPredicate := libovsdbops.GetPredicateAcrossAllControllers[*nbdb.AddressSet](asPredicateIDs, asPredicateFunc) if err := libovsdbops.DeleteAddressSetsWithPredicate(c.nbClient, asPredicate); err != nil { return fmt.Errorf("failed to remove stale ANP address sets, err: %v", err) } @@ -105,8 +106,9 @@ func (c *Controller) repairBaselineAdminNetworkPolicy() error { // We grab all the port groups that belong to BANP controller using externalIDs // and compare the value with the name of existing BANPs. If no match is found // we delete that port group along with all the acls in it. - predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.PortGroupBaselineAdminNetworkPolicy, c.controllerName, nil) - p := libovsdbops.GetPredicate[*nbdb.PortGroup](predicateIDs, func(pg *nbdb.PortGroup) bool { + // This is done for all networks in the cluster + predicateIDs := libovsdbops.NewDbObjectIDsAcrossAllContollers(libovsdbops.PortGroupBaselineAdminNetworkPolicy, nil) + p := libovsdbops.GetPredicateAcrossAllControllers[*nbdb.PortGroup](predicateIDs, func(pg *nbdb.PortGroup) bool { _, ok := existingBANPs[pg.ExternalIDs[libovsdbops.ObjectNameKey.String()]] return !ok // return if it doesn't exist in the cache }) @@ -124,17 +126,17 @@ func (c *Controller) repairBaselineAdminNetworkPolicy() error { } // Deal with Address-Sets Repairs // We grab all the AddressSets that belong to BANP controller using externalIDs - // and compare with the existing ANPs. The ones that don't match + // across all networks and compare with the existing ANPs. The ones that don't match // will be deleted from the DB. // NOTE: When we call syncBaselineAdminNetworkPolicy function after this for every BANP on startup, // the right Address-sets will be recreated. // Since we clean ACLs before Address-sets we should not run into any referential ingegrity violation - asPredicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.AddressSetBaselineAdminNetworkPolicy, c.controllerName, nil) + asPredicateIDs := libovsdbops.NewDbObjectIDsAcrossAllContollers(libovsdbops.AddressSetBaselineAdminNetworkPolicy, nil) asPredicateFunc := func(as *nbdb.AddressSet) bool { _, ok := existingBANPs[as.ExternalIDs[libovsdbops.ObjectNameKey.String()]] return !ok // if not present in cache then its stale } - asPredicate := libovsdbops.GetPredicate[*nbdb.AddressSet](asPredicateIDs, asPredicateFunc) + asPredicate := libovsdbops.GetPredicateAcrossAllControllers[*nbdb.AddressSet](asPredicateIDs, asPredicateFunc) if err := libovsdbops.DeleteAddressSetsWithPredicate(c.nbClient, asPredicate); err != nil { return fmt.Errorf("failed to remove stale BANP address sets, err: %v", err) } diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/repair_test.go b/go-controller/pkg/ovn/controller/admin_network_policy/repair_test.go index 2f041817d8..5687fb63bf 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/repair_test.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/repair_test.go @@ -349,7 +349,7 @@ func TestBaselineAdminNetworkPolicyRepair(t *testing.T) { } func portGroup(name string, ports []*nbdb.LogicalSwitchPort, acls []*nbdb.ACL, banp bool) *nbdb.PortGroup { - pgDbIDs := GetANPPortGroupDbIDs(name, banp, "default-network-controller") + pgDbIDs := GetANPPortGroupDbIDs(name, banp, defaultNetworkControllerName) pg := libovsdbutil.BuildPortGroup(pgDbIDs, ports, acls) pg.UUID = pgDbIDs.String() + "-UUID" return pg @@ -363,7 +363,7 @@ func stalePGWithoutExtIDs(name string, ports []*nbdb.LogicalSwitchPort, acls []* func accessControlList(name string, gressPrefix libovsdbutil.ACLDirection, priority int32, banp bool) *nbdb.ACL { objIDs := getANPRuleACLDbIDs(name, string(gressPrefix), fmt.Sprintf("%d", priority), "None", - "default-network-controller", banp) + defaultNetworkControllerName, banp) acl := &nbdb.ACL{ UUID: objIDs.String() + "-UUID", Action: nbdb.ACLActionAllow, @@ -381,7 +381,7 @@ func accessControlList(name string, gressPrefix libovsdbutil.ACLDirection, prior func addressSet(name, gressPrefix string, priority int32, banp bool) *nbdb.AddressSet { objIDs := GetANPPeerAddrSetDbIDs(name, gressPrefix, fmt.Sprintf("%d", priority), - "default-network-controller", banp) + defaultNetworkControllerName, banp) dbIDsWithIPFam := objIDs.AddIDs(map[libovsdbops.ExternalIDKey]string{libovsdbops.IPFamilyKey: "ipv4"}) as := &nbdb.AddressSet{ UUID: dbIDsWithIPFam.String() + "-UUID", diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/status_test.go b/go-controller/pkg/ovn/controller/admin_network_policy/status_test.go index 2b26f1c93f..38c6c43271 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/status_test.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/status_test.go @@ -115,7 +115,7 @@ func newANPControllerWithDBSetup(dbSetup libovsdbtest.TestSetup, initANPs anpapi addressSetFactory := addressset.NewOvnAddressSetFactory(nbClient, config.IPv4Mode, config.IPv6Mode) recorder := record.NewFakeRecorder(10) controller, err := NewController( - "default-network-controller", + defaultNetworkControllerName, nbClient, fakeClient.ANPClient, watcher.ANPInformer(), @@ -128,6 +128,7 @@ func newANPControllerWithDBSetup(dbSetup libovsdbtest.TestSetup, initANPs anpapi "targaryen", recorder, nil, + nil, ) gomega.Expect(err).ToNot(gomega.HaveOccurred()) diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/types.go b/go-controller/pkg/ovn/controller/admin_network_policy/types.go index 2b1391180f..3870ae694a 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/types.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/types.go @@ -39,7 +39,8 @@ type adminNetworkPolicySubject struct { // transact ops calculation. If not, for every pod/namespace update // we would need to do a lookup in the libovsdb cache for the ns_name // LSP index. - podPorts sets.Set[string] + // {K: networkName; V: {set of podLSPs matching the provided podSelector}} + podPorts map[string]sets.Set[string] } type adminNetworkPolicyPeer struct { @@ -52,7 +53,8 @@ type adminNetworkPolicyPeer struct { // {K: namespace name; V: {set of pods matching the provided podSelector}} namespaces map[string]sets.Set[string] // set of nodes matching the provided nodeSelector - nodes sets.Set[string] + nodes sets.Set[string] + networks sets.Set[string] } type gressRule struct { @@ -68,7 +70,9 @@ type gressRule struct { peers []*adminNetworkPolicyPeer ports []*libovsdbutil.NetworkPolicyPort // all the peerAddresses of the peer entities (podIPs, nodeIPs, CIDR ranges) selected by this ANP Rule - peerAddresses sets.Set[string] + // for all networks + // {K: networkName; V: {set of IPs}} + peerAddresses map[string]sets.Set[string] // saves NamedPort representation; // key is the name of the Port // value is an array of possible representations of this port (relevance wrt to rule, peers) @@ -94,17 +98,21 @@ type adminNetworkPolicyState struct { // aclLoggingParams stores the log levels for the ACLs created for this ANP // this is based off the "k8s.ovn.org/acl-logging" annotation set on the ANP's aclLoggingParams *libovsdbutil.ACLLoggingLevels + // set of networkNames selected either as part of ANP's + // subject OR peers in any one of the rules + managedNetworks sets.Set[string] } // newAdminNetworkPolicyState takes the provided ANP API object and creates a new corresponding // adminNetworkPolicyState cache object for that API object. func newAdminNetworkPolicyState(raw *anpapi.AdminNetworkPolicy) (*adminNetworkPolicyState, error) { anp := &adminNetworkPolicyState{ - name: raw.Name, - anpPriority: raw.Spec.Priority, - ovnPriority: (ANPFlowStartPriority - raw.Spec.Priority*ANPMaxRulesPerObject), - ingressRules: make([]*gressRule, 0), - egressRules: make([]*gressRule, 0), + name: raw.Name, + anpPriority: raw.Spec.Priority, + ovnPriority: (ANPFlowStartPriority - raw.Spec.Priority*ANPMaxRulesPerObject), + ingressRules: make([]*gressRule, 0), + egressRules: make([]*gressRule, 0), + managedNetworks: make(sets.Set[string]), } var err error anp.subject, err = newAdminNetworkPolicySubject(raw.Spec.Subject) @@ -271,6 +279,14 @@ func newAdminNetworkPolicyEgressPeer(raw anpapi.AdminNetworkPolicyEgressPeer) (* namespaceSelector: labels.Nothing(), // doesn't match any namespaces podSelector: labels.Nothing(), // doesn't match any pods nodeSelector: labels.Nothing(), // doesn't match any nodes + networks: make(sets.Set[string]), + } + for _, cidr := range raw.Networks { + _, ipNet, err := net.ParseCIDR(string(cidr)) + if err != nil { + return nil, err + } + anpPeer.networks.Insert(ipNet.String()) } } return anpPeer, nil @@ -280,15 +296,14 @@ func newAdminNetworkPolicyEgressPeer(raw anpapi.AdminNetworkPolicyEgressPeer) (* // gressRule cache object for that Rule. func newAdminNetworkPolicyIngressRule(raw anpapi.AdminNetworkPolicyIngressRule, index, priority int32) (*gressRule, error) { anpRule := &gressRule{ - name: raw.Name, - priority: priority, - gressIndex: index, - action: GetACLActionForANPRule(raw.Action), - gressPrefix: string(libovsdbutil.ACLIngress), - peers: make([]*adminNetworkPolicyPeer, 0), - ports: make([]*libovsdbutil.NetworkPolicyPort, 0), - namedPorts: make(map[string][]libovsdbutil.NamedNetworkPolicyPort, 0), - peerAddresses: sets.New[string](), + name: raw.Name, + priority: priority, + gressIndex: index, + action: GetACLActionForANPRule(raw.Action), + gressPrefix: string(libovsdbutil.ACLIngress), + peers: make([]*adminNetworkPolicyPeer, 0), + ports: make([]*libovsdbutil.NetworkPolicyPort, 0), + namedPorts: make(map[string][]libovsdbutil.NamedNetworkPolicyPort, 0), } for _, peer := range raw.From { anpPeer, err := newAdminNetworkPolicyIngressPeer(peer) @@ -315,15 +330,14 @@ func newAdminNetworkPolicyIngressRule(raw anpapi.AdminNetworkPolicyIngressRule, // gressRule cache object for that Rule. func newAdminNetworkPolicyEgressRule(raw anpapi.AdminNetworkPolicyEgressRule, index, priority int32) (*gressRule, error) { anpRule := &gressRule{ - name: raw.Name, - priority: priority, - gressIndex: index, - action: GetACLActionForANPRule(raw.Action), - gressPrefix: string(libovsdbutil.ACLEgress), - peers: make([]*adminNetworkPolicyPeer, 0), - ports: make([]*libovsdbutil.NetworkPolicyPort, 0), - namedPorts: make(map[string][]libovsdbutil.NamedNetworkPolicyPort, 0), - peerAddresses: sets.New[string](), + name: raw.Name, + priority: priority, + gressIndex: index, + action: GetACLActionForANPRule(raw.Action), + gressPrefix: string(libovsdbutil.ACLEgress), + peers: make([]*adminNetworkPolicyPeer, 0), + ports: make([]*libovsdbutil.NetworkPolicyPort, 0), + namedPorts: make(map[string][]libovsdbutil.NamedNetworkPolicyPort, 0), } for _, peer := range raw.To { anpPeer, err := newAdminNetworkPolicyEgressPeer(peer) @@ -331,15 +345,6 @@ func newAdminNetworkPolicyEgressRule(raw anpapi.AdminNetworkPolicyEgressRule, in return nil, err } anpRule.peers = append(anpRule.peers, anpPeer) - if len(peer.Networks) > 0 { - for _, cidr := range peer.Networks { - _, ipNet, err := net.ParseCIDR(string(cidr)) - if err != nil { - return nil, err - } - anpRule.peerAddresses.Insert(ipNet.String()) - } - } } if raw.Ports != nil { for _, port := range *raw.Ports { @@ -358,11 +363,12 @@ func newAdminNetworkPolicyEgressRule(raw anpapi.AdminNetworkPolicyEgressRule, in // adminNetworkPolicyState cache object for that API object. func newBaselineAdminNetworkPolicyState(raw *anpapi.BaselineAdminNetworkPolicy) (*adminNetworkPolicyState, error) { banp := &adminNetworkPolicyState{ - name: raw.Name, - anpPriority: 0, // since BANP does not have priority, we hardcode to 0 - ovnPriority: BANPFlowPriority, - ingressRules: make([]*gressRule, 0), - egressRules: make([]*gressRule, 0), + name: raw.Name, + anpPriority: 0, // since BANP does not have priority, we hardcode to 0 + ovnPriority: BANPFlowPriority, + ingressRules: make([]*gressRule, 0), + egressRules: make([]*gressRule, 0), + managedNetworks: make(sets.Set[string]), } var err error banp.subject, err = newAdminNetworkPolicySubject(raw.Spec.Subject) @@ -402,15 +408,14 @@ func newBaselineAdminNetworkPolicyState(raw *anpapi.BaselineAdminNetworkPolicy) // gressRule cache object for that Rule. func newBaselineAdminNetworkPolicyIngressRule(raw anpapi.BaselineAdminNetworkPolicyIngressRule, index, priority int32) (*gressRule, error) { banpRule := &gressRule{ - name: raw.Name, - priority: priority, - gressIndex: index, - action: GetACLActionForBANPRule(raw.Action), - gressPrefix: string(libovsdbutil.ACLIngress), - peers: make([]*adminNetworkPolicyPeer, 0), - ports: make([]*libovsdbutil.NetworkPolicyPort, 0), - namedPorts: make(map[string][]libovsdbutil.NamedNetworkPolicyPort, 0), - peerAddresses: sets.New[string](), + name: raw.Name, + priority: priority, + gressIndex: index, + action: GetACLActionForBANPRule(raw.Action), + gressPrefix: string(libovsdbutil.ACLIngress), + peers: make([]*adminNetworkPolicyPeer, 0), + ports: make([]*libovsdbutil.NetworkPolicyPort, 0), + namedPorts: make(map[string][]libovsdbutil.NamedNetworkPolicyPort, 0), } for _, peer := range raw.From { anpPeer, err := newAdminNetworkPolicyIngressPeer(peer) @@ -437,15 +442,14 @@ func newBaselineAdminNetworkPolicyIngressRule(raw anpapi.BaselineAdminNetworkPol // gressRule cache object for that Rule. func newBaselineAdminNetworkPolicyEgressRule(raw anpapi.BaselineAdminNetworkPolicyEgressRule, index, priority int32) (*gressRule, error) { banpRule := &gressRule{ - name: raw.Name, - priority: priority, - gressIndex: index, - action: GetACLActionForBANPRule(raw.Action), - gressPrefix: string(libovsdbutil.ACLEgress), - peers: make([]*adminNetworkPolicyPeer, 0), - ports: make([]*libovsdbutil.NetworkPolicyPort, 0), - namedPorts: make(map[string][]libovsdbutil.NamedNetworkPolicyPort, 0), - peerAddresses: sets.New[string](), + name: raw.Name, + priority: priority, + gressIndex: index, + action: GetACLActionForBANPRule(raw.Action), + gressPrefix: string(libovsdbutil.ACLEgress), + peers: make([]*adminNetworkPolicyPeer, 0), + ports: make([]*libovsdbutil.NetworkPolicyPort, 0), + namedPorts: make(map[string][]libovsdbutil.NamedNetworkPolicyPort, 0), } for _, peer := range raw.To { banpPeer, err := newAdminNetworkPolicyEgressPeer(peer) @@ -453,15 +457,6 @@ func newBaselineAdminNetworkPolicyEgressRule(raw anpapi.BaselineAdminNetworkPoli return nil, err } banpRule.peers = append(banpRule.peers, banpPeer) - if len(peer.Networks) > 0 { - for _, cidr := range peer.Networks { - _, ipNet, err := net.ParseCIDR(string(cidr)) - if err != nil { - return nil, err - } - banpRule.peerAddresses.Insert(ipNet.String()) - } - } } if raw.Ports != nil { for _, port := range *raw.Ports { diff --git a/go-controller/pkg/ovn/controller/admin_network_policy/utils.go b/go-controller/pkg/ovn/controller/admin_network_policy/utils.go index d319c35d35..25c17535c3 100644 --- a/go-controller/pkg/ovn/controller/admin_network_policy/utils.go +++ b/go-controller/pkg/ovn/controller/admin_network_policy/utils.go @@ -17,6 +17,7 @@ import ( libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" utilerrors "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" ) @@ -25,28 +26,40 @@ var ErrorANPPriorityUnsupported = errors.New("OVNK only supports priority ranges var ANPWithDuplicatePriorityEvent = "ANPWithDuplicatePriority" var ANPWithUnsupportedPriorityEvent = "ANPWithUnsupportedPriority" -func GetANPPortGroupDbIDs(anpName string, isBanp bool, controller string) *libovsdbops.DbObjectIDs { +func GetANPPortGroupDbIDs(anpName string, isBanp bool, networkName string) *libovsdbops.DbObjectIDs { + if networkName == types.DefaultNetworkName { + // backwards compatibility to not change any of the dbIndexes for default network + networkName = defaultNetworkControllerName + } idsType := libovsdbops.PortGroupAdminNetworkPolicy if isBanp { idsType = libovsdbops.PortGroupBaselineAdminNetworkPolicy } - return libovsdbops.NewDbObjectIDs(idsType, controller, + return libovsdbops.NewDbObjectIDs(idsType, networkName, map[libovsdbops.ExternalIDKey]string{ libovsdbops.ObjectNameKey: anpName, }) } -func (c *Controller) getANPPortGroupName(anpName string, isBanp bool) string { - return libovsdbutil.GetPortGroupName(GetANPPortGroupDbIDs(anpName, isBanp, c.controllerName)) +func (c *Controller) getANPPortGroupName(anpName, networkName string, isBanp bool) string { + if networkName == types.DefaultNetworkName { + // backwards compatibility to not change any of the dbIndexes for default network + networkName = c.controllerName + } + return libovsdbutil.GetPortGroupName(GetANPPortGroupDbIDs(anpName, isBanp, networkName)) } // getANPRuleACLDbIDs will return the dbObjectIDs for a given rule's ACLs -func getANPRuleACLDbIDs(name, gressPrefix, gressIndex, protocol, controller string, isBanp bool) *libovsdbops.DbObjectIDs { +func getANPRuleACLDbIDs(name, gressPrefix, gressIndex, protocol, networkName string, isBanp bool) *libovsdbops.DbObjectIDs { + if networkName == types.DefaultNetworkName { + // backwards compatibility to not change any of the dbIndexes for default network + networkName = defaultNetworkControllerName + } idType := libovsdbops.ACLAdminNetworkPolicy if isBanp { idType = libovsdbops.ACLBaselineAdminNetworkPolicy } - return libovsdbops.NewDbObjectIDs(idType, controller, map[libovsdbops.ExternalIDKey]string{ + return libovsdbops.NewDbObjectIDs(idType, networkName, map[libovsdbops.ExternalIDKey]string{ libovsdbops.ObjectNameKey: name, libovsdbops.PolicyDirectionKey: gressPrefix, // gressidx is the unique id for address set within given objectName and gressPrefix @@ -87,12 +100,16 @@ func GetACLActionForBANPRule(action anpapi.BaselineAdminNetworkPolicyRuleAction) } // GetANPPeerAddrSetDbIDs will return the dbObjectIDs for a given rule's address-set -func GetANPPeerAddrSetDbIDs(name, gressPrefix, gressIndex, controller string, isBanp bool) *libovsdbops.DbObjectIDs { +func GetANPPeerAddrSetDbIDs(name, gressPrefix, gressIndex, networkName string, isBanp bool) *libovsdbops.DbObjectIDs { + if networkName == types.DefaultNetworkName { + // backwards compatibility to not change any of the dbIndexes for default network + networkName = defaultNetworkControllerName + } idType := libovsdbops.AddressSetAdminNetworkPolicy if isBanp { idType = libovsdbops.AddressSetBaselineAdminNetworkPolicy } - return libovsdbops.NewDbObjectIDs(idType, controller, map[libovsdbops.ExternalIDKey]string{ + return libovsdbops.NewDbObjectIDs(idType, networkName, map[libovsdbops.ExternalIDKey]string{ libovsdbops.ObjectNameKey: name, libovsdbops.PolicyDirectionKey: gressPrefix, // gressidx is the unique id for address set within given objectName and gressPrefix diff --git a/go-controller/pkg/ovn/controller/egressservice/egressservice_zone.go b/go-controller/pkg/ovn/controller/egressservice/egressservice_zone.go index 316e9df283..bb0eba6970 100644 --- a/go-controller/pkg/ovn/controller/egressservice/egressservice_zone.go +++ b/go-controller/pkg/ovn/controller/egressservice/egressservice_zone.go @@ -44,7 +44,7 @@ const ( ) type InitClusterEgressPoliciesFunc func(client libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, - ni util.NetInfo, clusterSubnets []*net.IPNet, controllerName string) error + ni util.NetInfo, clusterSubnets []*net.IPNet, controllerName, routerName string) error type EnsureNoRerouteNodePoliciesFunc func(client libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, networkName, controllerName, clusterRouter string, nodeLister corelisters.NodeLister, v4, v6 bool) error type DeleteLegacyDefaultNoRerouteNodePoliciesFunc func(nbClient libovsdbclient.Client, clusterRouter, nodeName string) error @@ -60,10 +60,9 @@ type Controller struct { stopCh <-chan struct{} sync.Mutex - initClusterEgressPolicies InitClusterEgressPoliciesFunc - ensureNoRerouteNodePolicies EnsureNoRerouteNodePoliciesFunc - deleteLegacyDefaultNoRerouteNodePolicies DeleteLegacyDefaultNoRerouteNodePoliciesFunc - createDefaultRouteToExternalForIC CreateDefaultRouteToExternalFunc + initClusterEgressPolicies InitClusterEgressPoliciesFunc + ensureNoRerouteNodePolicies EnsureNoRerouteNodePoliciesFunc + createDefaultRouteToExternalForIC CreateDefaultRouteToExternalFunc services map[string]*svcState // svc key -> state, for services that have sourceIPBy LBIP nodes map[string]*nodeState // node name -> state, contains nodes that host an egress service @@ -121,7 +120,6 @@ func NewController( addressSetFactory addressset.AddressSetFactory, initClusterEgressPolicies InitClusterEgressPoliciesFunc, ensureNoRerouteNodePolicies EnsureNoRerouteNodePoliciesFunc, - deleteLegacyDefaultNoRerouteNodePolicies DeleteLegacyDefaultNoRerouteNodePoliciesFunc, createDefaultRouteToExternalForIC CreateDefaultRouteToExternalFunc, stopCh <-chan struct{}, esInformer egressserviceinformer.EgressServiceInformer, @@ -132,20 +130,19 @@ func NewController( klog.Info("Setting up event handlers for Egress Services") c := &Controller{ - NetInfo: netInfo, - controllerName: controllerName, - client: client, - nbClient: nbClient, - addressSetFactory: addressSetFactory, - initClusterEgressPolicies: initClusterEgressPolicies, - ensureNoRerouteNodePolicies: ensureNoRerouteNodePolicies, - deleteLegacyDefaultNoRerouteNodePolicies: deleteLegacyDefaultNoRerouteNodePolicies, - createDefaultRouteToExternalForIC: createDefaultRouteToExternalForIC, - stopCh: stopCh, - services: map[string]*svcState{}, - nodes: map[string]*nodeState{}, - nodesZoneState: map[string]bool{}, - zone: zone, + NetInfo: netInfo, + controllerName: controllerName, + client: client, + nbClient: nbClient, + addressSetFactory: addressSetFactory, + initClusterEgressPolicies: initClusterEgressPolicies, + ensureNoRerouteNodePolicies: ensureNoRerouteNodePolicies, + createDefaultRouteToExternalForIC: createDefaultRouteToExternalForIC, + stopCh: stopCh, + services: map[string]*svcState{}, + nodes: map[string]*nodeState{}, + nodesZoneState: map[string]bool{}, + zone: zone, } c.egressServiceLister = esInformer.Lister() @@ -232,7 +229,7 @@ func (c *Controller) Run(wg *sync.WaitGroup, threadiness int) error { klog.Errorf("Failed to repair Egress Services entries: %v", err) } subnets := util.GetAllClusterSubnetsFromEntries(c.Subnets()) - err = c.initClusterEgressPolicies(c.nbClient, c.addressSetFactory, &util.DefaultNetInfo{}, subnets, c.controllerName) + err = c.initClusterEgressPolicies(c.nbClient, c.addressSetFactory, c, subnets, c.controllerName, c.GetNetworkScopedClusterRouterName()) if err != nil { klog.Errorf("Failed to init Egress Services cluster policies: %v", err) } diff --git a/go-controller/pkg/ovn/controller/egressservice/egressservice_zone_node.go b/go-controller/pkg/ovn/controller/egressservice/egressservice_zone_node.go index c12edfea52..0dd52e309f 100644 --- a/go-controller/pkg/ovn/controller/egressservice/egressservice_zone_node.go +++ b/go-controller/pkg/ovn/controller/egressservice/egressservice_zone_node.go @@ -122,16 +122,11 @@ func (c *Controller) syncNode(key string) error { } else { delete(c.nodesZoneState, nodeName) } - - if err := c.deleteLegacyDefaultNoRerouteNodePolicies(c.nbClient, c.GetNetworkScopedClusterRouterName(), nodeName); err != nil { - return err - } - // We ensure node no re-route policies contemplating possible node IP // address changes regardless of allocated services. network := util.DefaultNetInfo{} networkName := network.GetNetworkName() - err = c.ensureNoRerouteNodePolicies(c.nbClient, c.addressSetFactory, networkName, c.controllerName, c.GetNetworkScopedClusterRouterName(), c.nodeLister, config.IPv4Mode, config.IPv6Mode) + err = c.ensureNoRerouteNodePolicies(c.nbClient, c.addressSetFactory, networkName, c.GetNetworkScopedClusterRouterName(), c.controllerName, c.nodeLister, config.IPv4Mode, config.IPv6Mode) if err != nil { return err } diff --git a/go-controller/pkg/ovn/controller/services/services_controller.go b/go-controller/pkg/ovn/controller/services/services_controller.go index 6ce13ffcc8..9322ae9fbf 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller.go +++ b/go-controller/pkg/ovn/controller/services/services_controller.go @@ -17,7 +17,7 @@ import ( libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -63,7 +63,7 @@ func NewController(client clientset.Interface, serviceInformer coreinformers.ServiceInformer, endpointSliceInformer discoveryinformers.EndpointSliceInformer, nodeInformer coreinformers.NodeInformer, - nadController networkAttachDefController.NADController, + networkManager networkmanager.Interface, recorder record.EventRecorder, netInfo util.NetInfo, ) (*Controller, error) { @@ -83,7 +83,7 @@ func NewController(client clientset.Interface, serviceLister: serviceInformer.Lister(), endpointSliceInformer: endpointSliceInformer, endpointSliceLister: endpointSliceInformer.Lister(), - nadController: nadController, + networkManager: networkManager, eventRecorder: recorder, repair: newRepair(serviceInformer.Lister(), nbClient), @@ -117,7 +117,7 @@ type Controller struct { endpointSliceInformer discoveryinformers.EndpointSliceInformer endpointSliceLister discoverylisters.EndpointSliceLister - nadController networkAttachDefController.NADController + networkManager networkmanager.Interface nodesSynced cache.InformerSynced @@ -569,7 +569,7 @@ func (c *Controller) RequestFullSync(nodeInfos []nodeInfo) { // belong to the network that this service controller is responsible for. func (c *Controller) skipService(name, namespace string) bool { if util.IsNetworkSegmentationSupportEnabled() { - serviceNetwork, err := c.nadController.GetActiveNetworkForNamespace(namespace) + serviceNetwork, err := c.networkManager.GetActiveNetworkForNamespace(namespace) if err != nil { utilruntime.HandleError(fmt.Errorf("failed to retrieve network for service %s/%s: %w", namespace, name, err)) @@ -638,13 +638,14 @@ func (c *Controller) onServiceDelete(obj interface{}) { utilruntime.HandleError(fmt.Errorf("couldn't get key for object %+v: %v", obj, err)) return } - klog.V(4).Infof("Deleting service %s for network=%s", key, c.netInfo.GetNetworkName()) service := obj.(*v1.Service) if c.skipService(service.Name, service.Namespace) { return } + klog.V(4).Infof("Deleting service %s for network=%s", key, c.netInfo.GetNetworkName()) + metrics.GetConfigDurationRecorder().Start("service", service.Namespace, service.Name) c.queue.Add(key) } diff --git a/go-controller/pkg/ovn/controller/services/services_controller_test.go b/go-controller/pkg/ovn/controller/services/services_controller_test.go index 3e82cd40ba..c9d5a42d2b 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller_test.go +++ b/go-controller/pkg/ovn/controller/services/services_controller_test.go @@ -24,14 +24,12 @@ import ( ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" - globalconfig "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" kubetest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -85,18 +83,13 @@ func newControllerWithDBSetupForNetwork(dbSetup libovsdbtest.TestSetup, netInfo return nil, err } } - testNCM := &nad.FakeNetworkControllerManager{} - nadController, err := networkAttachDefController.NewNetAttachDefinitionController("test", testNCM, factoryMock, nil) - if err != nil { - return nil, err - } controller, err := NewController(client.KubeClient, nbClient, factoryMock.ServiceCoreInformer(), factoryMock.EndpointSliceCoreInformer(), factoryMock.NodeCoreInformer(), - nadController, + networkmanager.Default().Interface(), recorder, netInfo, ) @@ -107,7 +100,7 @@ func newControllerWithDBSetupForNetwork(dbSetup libovsdbtest.TestSetup, netInfo if nbZoneFailed { // Delete the NBGlobal row as this function created it. Otherwise many tests would fail while // checking the expectedData in the NBDB. - if err = deleteTestNBGlobal(nbClient, "global"); err != nil { + if err = deleteTestNBGlobal(nbClient); err != nil { return nil, err } } @@ -206,21 +199,21 @@ func TestSyncServices(t *testing.T) { initialLrGroups = []string{types.ClusterLBGroupName, types.ClusterRouterLBGroupName} ) // setup global config - oldGateway := globalconfig.Gateway.Mode - oldClusterSubnet := globalconfig.Default.ClusterSubnets - globalconfig.Kubernetes.OVNEmptyLbEvents = true - globalconfig.IPv4Mode = true - globalconfig.IPv6Mode = true + oldGateway := config.Gateway.Mode + oldClusterSubnet := config.Default.ClusterSubnets + config.Kubernetes.OVNEmptyLbEvents = true + config.IPv4Mode = true + config.IPv6Mode = true defer func() { - globalconfig.Kubernetes.OVNEmptyLbEvents = false - globalconfig.IPv4Mode = false - globalconfig.IPv6Mode = false - globalconfig.Gateway.Mode = oldGateway - globalconfig.Default.ClusterSubnets = oldClusterSubnet + config.Kubernetes.OVNEmptyLbEvents = false + config.IPv4Mode = false + config.IPv6Mode = false + config.Gateway.Mode = oldGateway + config.Default.ClusterSubnets = oldClusterSubnet }() _, cidr4, _ := net.ParseCIDR("10.128.0.0/16") _, cidr6, _ := net.ParseCIDR("fe00:0:0:0:5555::0/64") - globalconfig.Default.ClusterSubnets = []globalconfig.CIDRNetworkEntry{{cidr4, 26}, {cidr6, 26}} + config.Default.ClusterSubnets = []config.CIDRNetworkEntry{{CIDR: cidr4, HostSubnetLength: 26}, {CIDR: cidr6, HostSubnetLength: 26}} l3UDN, err := getSampleUDNNetInfo(ns, "layer3") if err != nil { t.Fatalf("Error creating UDNNetInfo: %v", err) @@ -1473,14 +1466,14 @@ func TestSyncServices(t *testing.T) { } if tt.gatewayMode != "" { - globalconfig.Gateway.Mode = globalconfig.GatewayMode(tt.gatewayMode) + config.Gateway.Mode = config.GatewayMode(tt.gatewayMode) } else { - globalconfig.Gateway.Mode = globalconfig.GatewayModeShared + config.Gateway.Mode = config.GatewayModeShared } if tt.enableIPv6 { - globalconfig.IPv6Mode = true - defer func() { globalconfig.IPv6Mode = false }() + config.IPv6Mode = true + defer func() { config.IPv6Mode = false }() } // Create services controller @@ -1491,10 +1484,6 @@ func TestSyncServices(t *testing.T) { if err != nil { t.Fatalf("Error creating controller: %v", err) } - if err := controller.nadController.Start(); err != nil { - t.Fatalf("Error starting NAD controller: %v", err) - } - defer controller.nadController.Stop() defer controller.close() // Add k8s objects @@ -1654,33 +1643,6 @@ func clusterWideTCPServiceLoadBalancerNameForNetwork(ns string, serviceName stri return netInfo.GetNetworkScopedLoadBalancerName(baseName) } -func nodeSwitchRouterLoadBalancerNameForNetwork(nodeName string, serviceNamespace string, serviceName string, netInfo util.NetInfo) string { - baseName := fmt.Sprintf( - "Service_%s/%s_TCP_node_router+switch_%s", - serviceNamespace, - serviceName, - nodeName) - return netInfo.GetNetworkScopedLoadBalancerName(baseName) -} - -func nodeSwitchTemplateLoadBalancerNameForNetwork(serviceNamespace string, serviceName string, addressFamily v1.IPFamily, netInfo util.NetInfo) string { - baseName := fmt.Sprintf( - "Service_%s/%s_TCP_node_switch_template_%s", - serviceNamespace, - serviceName, - addressFamily) - return netInfo.GetNetworkScopedLoadBalancerName(baseName) -} - -func nodeRouterTemplateLoadBalancerNameForNetwork(serviceNamespace string, serviceName string, addressFamily v1.IPFamily, netInfo util.NetInfo) string { - baseName := fmt.Sprintf( - "Service_%s/%s_TCP_node_router_template_%s", - serviceNamespace, - serviceName, - addressFamily) - return netInfo.GetNetworkScopedLoadBalancerName(baseName) -} - func nodeMergedTemplateLoadBalancerName(serviceNamespace string, serviceName string, addressFamily v1.IPFamily) string { return nodeMergedTemplateLoadBalancerNameForNetwork(serviceNamespace, serviceName, addressFamily, &util.DefaultNetInfo{}) } @@ -1726,12 +1688,6 @@ func templateServicesOptionsV6() map[string]string { return opts } -func tcpGatewayRouterExternalIDs() map[string]string { - return map[string]string{ - "TCP_lb_gateway_router": "", - } -} - func getExternalIDsForNetwork(network string) map[string]string { if network == types.DefaultNetworkName { return map[string]string{} @@ -1857,7 +1813,7 @@ func createTestNBGlobal(nbClient libovsdbclient.Client, zone string) error { return nil } -func deleteTestNBGlobal(nbClient libovsdbclient.Client, zone string) error { +func deleteTestNBGlobal(nbClient libovsdbclient.Client) error { p := func(nbGlobal *nbdb.NBGlobal) bool { return true } diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index e61ec29c2c..16c78d7364 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -14,7 +14,7 @@ import ( egressqoslisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/listers/egressqos/v1" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/observability" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" anpcontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/admin_network_policy" @@ -34,7 +34,6 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/syncmap" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" - ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" utilerrors "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" @@ -145,17 +144,28 @@ type DefaultNetworkController struct { // NewDefaultNetworkController creates a new OVN controller for creating logical network // infrastructure and policy for default l3 network -func NewDefaultNetworkController(cnci *CommonNetworkControllerInfo, nadController *nad.NetAttachDefinitionController, - observManager *observability.Manager, portCache *PortCache, eIPController *EgressIPController) (*DefaultNetworkController, error) { +func NewDefaultNetworkController( + cnci *CommonNetworkControllerInfo, + observManager *observability.Manager, + networkManager networkmanager.Interface, + eIPController *EgressIPController, + portCache *PortCache, +) (*DefaultNetworkController, error) { stopChan := make(chan struct{}) wg := &sync.WaitGroup{} - return newDefaultNetworkControllerCommon(cnci, stopChan, wg, nil, nadController, observManager, portCache, eIPController) + return newDefaultNetworkControllerCommon(cnci, stopChan, wg, nil, networkManager, observManager, eIPController, portCache) } -func newDefaultNetworkControllerCommon(cnci *CommonNetworkControllerInfo, - defaultStopChan chan struct{}, defaultWg *sync.WaitGroup, - addressSetFactory addressset.AddressSetFactory, nadController *nad.NetAttachDefinitionController, - observManager *observability.Manager, portCache *PortCache, eIPController *EgressIPController) (*DefaultNetworkController, error) { +func newDefaultNetworkControllerCommon( + cnci *CommonNetworkControllerInfo, + defaultStopChan chan struct{}, + defaultWg *sync.WaitGroup, + addressSetFactory addressset.AddressSetFactory, + networkManager networkmanager.Interface, + observManager *observability.Manager, + eIPController *EgressIPController, + portCache *PortCache, +) (*DefaultNetworkController, error) { if addressSetFactory == nil { addressSetFactory = addressset.NewOvnAddressSetFactory(cnci.nbClient, config.IPv4Mode, config.IPv6Mode) @@ -166,7 +176,7 @@ func newDefaultNetworkControllerCommon(cnci *CommonNetworkControllerInfo, cnci.watchFactory.ServiceCoreInformer(), cnci.watchFactory.EndpointSliceCoreInformer(), cnci.watchFactory.NodeCoreInformer(), - nadController, + networkManager, cnci.recorder, &util.DefaultNetInfo{}, ) @@ -199,7 +209,7 @@ func newDefaultNetworkControllerCommon(cnci *CommonNetworkControllerInfo, BaseNetworkController: BaseNetworkController{ CommonNetworkControllerInfo: *cnci, controllerName: DefaultNetworkControllerName, - NetInfo: &util.DefaultNetInfo{}, + ReconcilableNetInfo: &util.DefaultNetInfo{}, lsManager: lsm.NewLogicalSwitchManager(), logicalPortCache: portCache, namespaces: make(map[string]*namespaceInfo), @@ -214,7 +224,7 @@ func newDefaultNetworkControllerCommon(cnci *CommonNetworkControllerInfo, zoneICHandler: zoneICHandler, cancelableCtx: util.NewCancelableContext(), observManager: observManager, - nadController: nadController, + networkManager: networkManager, }, externalGatewayRouteInfo: apbExternalRouteController.ExternalGWRouteInfoCache, eIPC: eIPController, @@ -228,7 +238,7 @@ func newDefaultNetworkControllerCommon(cnci *CommonNetworkControllerInfo, // allocate the first IPs in the join switch subnets. gwLRPIfAddrs, err := oc.getOVNClusterRouterPortToJoinSwitchIfAddrs() if err != nil { - return nil, fmt.Errorf("failed to allocate join switch IP address connected to %s: %v", ovntypes.OVNClusterRouter, err) + return nil, fmt.Errorf("failed to allocate join switch IP address connected to %s: %v", types.OVNClusterRouter, err) } oc.ovnClusterLRPToJoinIfAddrs = gwLRPIfAddrs @@ -381,7 +391,7 @@ func (oc *DefaultNetworkController) Init(ctx context.Context) error { if _, _, err := util.RunOVNNbctl("--columns=_uuid", "list", "Load_Balancer_Group"); err != nil { klog.Warningf("Load Balancer Group support enabled, however version of OVN in use does not support Load Balancer Groups.") } else { - clusterLBGroupUUID, switchLBGroupUUID, routerLBGroupUUID, err := initLoadBalancerGroups(oc.nbClient, oc.NetInfo) + clusterLBGroupUUID, switchLBGroupUUID, routerLBGroupUUID, err := initLoadBalancerGroups(oc.nbClient, oc.GetNetInfo()) if err != nil { return err } @@ -551,7 +561,7 @@ func (oc *DefaultNetworkController) Run(ctx context.Context) error { // on ovnkube-controller side and not on ovnkube-node side, since they are run in the // same process. TODO(tssurya): In upstream ovnk, its possible to run these as different processes // in which case this flushing feature is not supported. - if config.OVNKubernetesFeature.EnableInterconnect && oc.zone != ovntypes.OvnDefaultZone { + if config.OVNKubernetesFeature.EnableInterconnect && oc.zone != types.OvnDefaultZone { // every minute cleanup stale conntrack entries if any go wait.Until(func() { oc.checkAndDeleteStaleConntrackEntries() @@ -585,6 +595,54 @@ func (oc *DefaultNetworkController) Run(ctx context.Context) error { return nil } +func (oc *DefaultNetworkController) Reconcile(netInfo util.NetInfo) error { + // gather some information first + var err error + var retryNodes []*kapi.Node + oc.localZoneNodes.Range(func(key, value any) bool { + nodeName := key.(string) + wasAdvertised := util.IsPodNetworkAdvertisedAtNode(oc, nodeName) + isAdvertised := util.IsPodNetworkAdvertisedAtNode(netInfo, nodeName) + if wasAdvertised == isAdvertised { + // noop + return true + } + var node *kapi.Node + node, err = oc.watchFactory.GetNode(nodeName) + if err != nil { + return false + } + retryNodes = append(retryNodes, node) + return true + }) + if err != nil { + return fmt.Errorf("failed to reconcile network %s: %w", oc.GetNetworkName(), err) + } + + // update network information, point of no return + err = util.ReconcileNetInfo(oc.ReconcilableNetInfo, netInfo) + if err != nil { + klog.Errorf("Failed to reconcile network %s: %v", oc.GetNetworkName(), err) + } + + for _, node := range retryNodes { + oc.gatewaysFailed.Store(node.Name, true) + err = oc.retryNodes.AddRetryObjWithAddNoBackoff(node) + if err != nil { + klog.Errorf("Failed to retry node %s for network %s: %v", node.Name, oc.GetNetworkName(), err) + } + } + if len(retryNodes) > 0 { + oc.retryNodes.RequestRetryObjs() + } + + return nil +} + +func (oc *DefaultNetworkController) isPodNetworkAdvertisedAtNode(node string) bool { + return util.IsPodNetworkAdvertisedAtNode(oc, node) +} + func WithSyncDurationMetric(resourceName string, f func() error) error { start := time.Now() defer func() { diff --git a/go-controller/pkg/ovn/egressip.go b/go-controller/pkg/ovn/egressip.go index f160840d95..53f20e2edd 100644 --- a/go-controller/pkg/ovn/egressip.go +++ b/go-controller/pkg/ovn/egressip.go @@ -25,7 +25,7 @@ import ( libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + networkmanager "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" egresssvc "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/egressservice" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/udnenabledsvc" @@ -190,8 +190,8 @@ type EgressIPController struct { // A cache that maintains all nodes in the cluster, // value will be true if local to this zone and false otherwise nodeZoneState *syncmap.SyncMap[bool] - // nadController used for getting network information for UDNs - nadController nad.NADController + // networkManager used for getting network information for UDNs + networkManager networkmanager.Interface // An address set factory that creates address sets addressSetFactory addressset.AddressSetFactory // Northbound database zone name to which this Controller is connected to - aka local zone @@ -202,8 +202,19 @@ type EgressIPController struct { controllerName string } -func NewEIPController(nbClient libovsdbclient.Client, kube *kube.KubeOVN, watchFactory *factory.WatchFactory, recorder record.EventRecorder, - portCache *PortCache, nadController nad.NADController, addressSetFactor addressset.AddressSetFactory, v4, v6 bool, zone, controllerName string) *EgressIPController { +func NewEIPController( + nbClient libovsdbclient.Client, + kube *kube.KubeOVN, + watchFactory *factory.WatchFactory, + recorder record.EventRecorder, + portCache *PortCache, + networkmanager networkmanager.Interface, + addressSetFactor addressset.AddressSetFactory, + v4 bool, + v6 bool, + zone string, + controllerName string, +) *EgressIPController { e := &EgressIPController{ nbClient: nbClient, kube: kube, @@ -215,7 +226,7 @@ func NewEIPController(nbClient libovsdbclient.Client, kube *kube.KubeOVN, watchF logicalPortCache: portCache, nodeZoneState: syncmap.NewSyncMap[bool](), controllerName: controllerName, - nadController: nadController, + networkManager: networkmanager, addressSetFactory: addressSetFactor, zone: zone, v4: v4, @@ -340,7 +351,7 @@ func (e *EgressIPController) reconcileEgressIP(old, new *egressipv1.EgressIP) (e for _, namespace := range namespaces { namespaceLabels := labels.Set(namespace.Labels) if !newNamespaceSelector.Matches(namespaceLabels) && oldNamespaceSelector.Matches(namespaceLabels) { - ni, err := e.nadController.GetActiveNetworkForNamespace(namespace.Name) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespace.Name) if err != nil { return fmt.Errorf("failed to get active network for namespace %s: %v", namespace.Name, err) } @@ -349,7 +360,7 @@ func (e *EgressIPController) reconcileEgressIP(old, new *egressipv1.EgressIP) (e } } if newNamespaceSelector.Matches(namespaceLabels) && !oldNamespaceSelector.Matches(namespaceLabels) { - ni, err := e.nadController.GetActiveNetworkForNamespace(namespace.Name) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespace.Name) if err != nil { return fmt.Errorf("failed to get active network for namespace %s: %v", namespace.Name, err) } @@ -375,7 +386,7 @@ func (e *EgressIPController) reconcileEgressIP(old, new *egressipv1.EgressIP) (e for _, pod := range pods { podLabels := labels.Set(pod.Labels) if !newPodSelector.Matches(podLabels) && oldPodSelector.Matches(podLabels) { - ni, err := e.nadController.GetActiveNetworkForNamespace(namespace.Name) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespace.Name) if err != nil { return fmt.Errorf("failed to get active network for namespace %s: %v", namespace.Name, err) } @@ -387,7 +398,7 @@ func (e *EgressIPController) reconcileEgressIP(old, new *egressipv1.EgressIP) (e continue } if newPodSelector.Matches(podLabels) && !oldPodSelector.Matches(podLabels) { - ni, err := e.nadController.GetActiveNetworkForNamespace(namespace.Name) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespace.Name) if err != nil { return fmt.Errorf("failed to get active network for namespace %s: %v", namespace.Name, err) } @@ -410,7 +421,7 @@ func (e *EgressIPController) reconcileEgressIP(old, new *egressipv1.EgressIP) (e namespaceLabels := labels.Set(namespace.Labels) // If the namespace does not match anymore then there's no // reason to look at the pod selector. - ni, err := e.nadController.GetActiveNetworkForNamespace(namespace.Name) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespace.Name) if err != nil { return fmt.Errorf("failed to get active network for namespace %s: %v", namespace.Name, err) } @@ -508,7 +519,7 @@ func (e *EgressIPController) reconcileEgressIPNamespace(old, new *corev1.Namespa return err } if namespaceSelector.Matches(oldLabels) && !namespaceSelector.Matches(newLabels) { - ni, err := e.nadController.GetActiveNetworkForNamespace(namespaceName) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespaceName) if err != nil { return fmt.Errorf("failed to get active network for namespace %s: %v", namespaceName, err) } @@ -518,7 +529,7 @@ func (e *EgressIPController) reconcileEgressIPNamespace(old, new *corev1.Namespa } if !namespaceSelector.Matches(oldLabels) && namespaceSelector.Matches(newLabels) { mark := getEgressIPPktMark(egressIP.Name, egressIP.Annotations) - ni, err := e.nadController.GetActiveNetworkForNamespace(namespaceName) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespaceName) if err != nil { return fmt.Errorf("failed to get active network for namespace %s: %v", namespaceName, err) } @@ -598,7 +609,7 @@ func (e *EgressIPController) reconcileEgressIPPod(old, new *corev1.Pod) (err err if err != nil { return err } - ni, err := e.nadController.GetActiveNetworkForNamespace(namespace.Name) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespace.Name) if err != nil { return fmt.Errorf("failed to get active network for namespace %s: %v", namespace.Name, err) } @@ -660,7 +671,7 @@ func (e *EgressIPController) addEgressIPAssignments(name string, statusAssignmen return err } for _, namespace := range namespaces { - ni, err := e.nadController.GetActiveNetworkForNamespace(namespace.Name) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespace.Name) if err != nil { return fmt.Errorf("failed to get active network for namespace %s: %v", namespace.Name, err) } @@ -854,7 +865,7 @@ func (e *EgressIPController) deleteEgressIPAssignments(name string, statusesToRe continue } podNamespace, podName := getPodNamespaceAndNameFromKey(podKey) - ni, err := e.nadController.GetActiveNetworkForNamespace(podNamespace) + ni, err := e.networkManager.GetActiveNetworkForNamespace(podNamespace) if err != nil { return fmt.Errorf("failed to get active network for namespace %s", podNamespace) } @@ -866,7 +877,7 @@ func (e *EgressIPController) deleteEgressIPAssignments(name string, statusesToRe if err := e.deleteEgressIPStatusSetup(ni, name, statusToRemove); err != nil { return fmt.Errorf("failed to delete EgressIP %s status setup for network %s: %v", name, ni.GetNetworkName(), err) } - if cachedNetwork != nil && !cachedNetwork.Equals(ni) { + if cachedNetwork != nil && util.AreNetworksCompatible(cachedNetwork, ni) { if err := e.deleteEgressIPStatusSetup(cachedNetwork, name, statusToRemove); err != nil { klog.Errorf("Failed to delete EgressIP %s status setup for network %s: %v", name, cachedNetwork.GetNetworkName(), err) } @@ -1005,7 +1016,7 @@ func (e *EgressIPController) deletePodEgressIPAssignments(ni util.NetInfo, name func (e *EgressIPController) deletePreviousNetworkPodEgressIPAssignments(ni util.NetInfo, name string, statusesToRemove []egressipv1.EgressIPStatusItem, pod *corev1.Pod) { cachedNetwork := e.getNetworkFromPodAssignmentWithLock(getPodKey(pod)) if cachedNetwork != nil { - if !cachedNetwork.Equals(ni) { + if util.AreNetworksCompatible(cachedNetwork, ni) { if err := e.deletePodEgressIPAssignments(cachedNetwork, name, statusesToRemove, pod); err != nil { // no error is returned because high probability network is deleted klog.Errorf("Failed to delete EgressIP %s assignment for pod %s/%s attached to network %s: %v", @@ -1170,6 +1181,18 @@ func (e *EgressIPController) SyncLocalNodeZonesCache() error { return nil } +// getALocalZoneNodeName fetches the first local OVN zone Node. Support for multiple Nodes per OVN zone is not supported +// and neither is changing a Nodes OVN zone. This function supports said assumptions. +func (e *EgressIPController) getALocalZoneNodeName() (string, error) { + nodeNames := e.nodeZoneState.GetKeys() + for _, nodeName := range nodeNames { + if isLocal, ok := e.nodeZoneState.Load(nodeName); ok && isLocal { + return nodeName, nil + } + } + return "", fmt.Errorf("failed to find a local OVN zone Node") +} + func (e *EgressIPController) syncStaleAddressSetIPs(egressIPCache egressIPCache) error { for _, networkPodCache := range egressIPCache.egressIPNameToPods { for networkName, podCache := range networkPodCache { @@ -1198,12 +1221,12 @@ func (e *EgressIPController) syncStaleAddressSetIPs(egressIPCache egressIPCache) } // syncStaleGWMarkRules removes stale or invalid LRP that packet mark. They are attached to egress nodes gateway router. -// It adds expected LRPs that packet mark. +// It adds expected LRPs that packet mark. This is valid only for L3 networks. func (e *EgressIPController) syncStaleGWMarkRules(egressIPCache egressIPCache) error { // Delete all stale LRPs then add missing LRPs // This func assumes one node per zone. It determines if an LRP is a valid local LRP. It doesn't determine if the // LRP is attached to the correct GW router - if !util.IsNetworkSegmentationSupportEnabled() || !config.OVNKubernetesFeature.EnableInterconnect { + if !isEgressIPForUDNSupported() { return nil } for _, networkPodCache := range egressIPCache.egressIPNameToPods { @@ -1411,13 +1434,22 @@ func (e *EgressIPController) syncPodAssignmentCache(egressIPCache egressIPCache) item.ExternalIDs[libovsdbops.OwnerControllerKey.String()] == getNetworkControllerName(networkName) && strings.HasPrefix(item.ExternalIDs[libovsdbops.ObjectNameKey.String()], egressIPName+dbIDEIPNamePodDivider) } - ni, err := e.nadController.GetNetwork(networkName) - if err != nil { - return fmt.Errorf("failed to get active network for network name %q: %v", networkName, err) + ni := e.networkManager.GetNetwork(networkName) + if ni == nil { + return fmt.Errorf("failed to get active network for network name %q", networkName) + } + routerName := ni.GetNetworkScopedClusterRouterName() + if ni.TopologyType() == types.Layer2Topology { + // no support for multiple Nodes per OVN zone, therefore pick the first local zone node + localNodeName, err := e.getALocalZoneNodeName() + if err != nil { + return err + } + routerName = ni.GetNetworkScopedGWRouterName(localNodeName) } - reRoutePolicies, err := libovsdbops.FindALogicalRouterPoliciesWithPredicate(e.nbClient, ni.GetNetworkScopedClusterRouterName(), p1) + reRoutePolicies, err := libovsdbops.FindALogicalRouterPoliciesWithPredicate(e.nbClient, routerName, p1) if err != nil { - return err + return fmt.Errorf("failed to retrieve a logical router polices attached to router %s: %w", routerName, err) } // not scoped by network since all NATs selected by the follow predicate select only CDN NATs p2 := func(item *nbdb.NAT) bool { @@ -1614,8 +1646,8 @@ func (e *EgressIPController) syncStaleSNATRules(egressIPCache egressIPCache) err klog.Infof("syncStaleSNATRules will delete %s due to logical ip: %v", egressIPName, item) return true } - ni, err := e.nadController.GetNetwork(types.DefaultNetworkName) - if err != nil { + ni := e.networkManager.GetNetwork(types.DefaultNetworkName) + if ni == nil { klog.Errorf("syncStaleSNATRules failed to find default network in networks cache") return false } @@ -1709,7 +1741,7 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { cache.networkToRouter = map[string]string{} // build a map of networks -> nodes -> redirect IP for _, namespace := range namespaces { - ni, err := e.nadController.GetActiveNetworkForNamespace(namespace.Name) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespace.Name) if err != nil { klog.Errorf("Failed to get active network for namespace %s, stale objects may remain: %v", namespace.Name, err) continue @@ -1719,7 +1751,17 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { continue } redirectCache[ni.GetNetworkName()] = map[string]redirectIPs{} - cache.networkToRouter[ni.GetNetworkName()] = ni.GetNetworkScopedClusterRouterName() + var localNodeName string + if localZoneNodes.Len() > 0 { + localNodeName = localZoneNodes.UnsortedList()[0] + } + routerName, err := getTopologyScopedRouterName(ni, localNodeName) + if err != nil { + klog.Errorf("Failed to get network topology scoped router name for network %s attached to namespace %s, stale objects may remain: %v", + ni.GetNetworkName(), namespace.Name, err) + continue + } + cache.networkToRouter[ni.GetNetworkName()] = routerName for _, node := range nodes { r := redirectIPs{} mgmtPort := &nbdb.LogicalSwitchPort{Name: ni.GetNetworkScopedK8sMgmtIntfName(node.Name)} @@ -1752,14 +1794,14 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { if localZoneNodes.Has(node.Name) { if e.v4 { - if gatewayRouterIP, err := e.getGatewayRouterJoinIP(ni, node.Name, false); err != nil { + if gatewayRouterIP, err := e.getGatewayNextHop(ni, node.Name, false); err != nil { klog.V(5).Infof("Unable to retrieve gateway IP for node: %s, protocol is IPv4: err: %v", node.Name, err) } else { r.v4Gateway = gatewayRouterIP.String() } } if e.v6 { - if gatewayRouterIP, err := e.getGatewayRouterJoinIP(ni, node.Name, true); err != nil { + if gatewayRouterIP, err := e.getGatewayNextHop(ni, node.Name, true); err != nil { klog.V(5).Infof("Unable to retrieve gateway IP for node: %s, protocol is IPv6: err: %v", node.Name, err) } else { r.v6Gateway = gatewayRouterIP.String() @@ -1827,7 +1869,7 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { klog.Errorf("Error building egress IP sync cache, cannot retrieve pods for namespace: %s and egress IP: %s, err: %v", namespace.Name, egressIP.Name, err) continue } - ni, err := e.nadController.GetActiveNetworkForNamespace(namespace.Name) + ni, err := e.networkManager.GetActiveNetworkForNamespace(namespace.Name) if err != nil { klog.Errorf("Failed to get active network for namespace %s, skipping sync: %v", namespace.Name, err) continue @@ -1939,27 +1981,26 @@ func (e *EgressIPController) addEgressNode(node *corev1.Node) error { // on IC if the node is gone, then the ovn_cluster_router is also gone along with all // the routes on it. processNetworkFn := func(ni util.NetInfo) error { - clusterSubnetsNetEntry := ni.Subnets() - if len(clusterSubnetsNetEntry) == 0 { + if ni.TopologyType() == types.Layer2Topology || len(ni.Subnets()) == 0 { return nil } if err := libovsdbutil.CreateDefaultRouteToExternal(e.nbClient, ni.GetNetworkScopedClusterRouterName(), - ni.GetNetworkScopedGWRouterName(node.Name), clusterSubnetsNetEntry); err != nil { + ni.GetNetworkScopedGWRouterName(node.Name), ni.Subnets()); err != nil { return fmt.Errorf("failed to create route to external for network %s: %v", ni.GetNetworkName(), err) } return nil } - ni, err := e.nadController.GetNetwork(types.DefaultNetworkName) - if err != nil { + ni := e.networkManager.GetNetwork(types.DefaultNetworkName) + if ni == nil { return fmt.Errorf("failed to get default network from NAD controller") } - if err = processNetworkFn(ni); err != nil { + if err := processNetworkFn(ni); err != nil { return fmt.Errorf("failed to process default network: %v", err) } - if !util.IsNetworkSegmentationSupportEnabled() { + if !isEgressIPForUDNSupported() { return nil } - if err = e.nadController.DoWithLock(processNetworkFn); err != nil { + if err := e.networkManager.DoWithLock(processNetworkFn); err != nil { return fmt.Errorf("failed to process all user defined networks route to external: %v", err) } } @@ -1977,33 +2018,30 @@ func (e *EgressIPController) addEgressNode(node *corev1.Node) error { // continue to do so without any issues. func (e *EgressIPController) initClusterEgressPolicies(nodes []interface{}) error { // Init default network - defaultNetInfo, err := e.nadController.GetNetwork(types.DefaultNetworkName) + defaultNetInfo := e.networkManager.GetNetwork(types.DefaultNetworkName) + localNodeName, err := e.getALocalZoneNodeName() if err != nil { - return fmt.Errorf("failed to get default network: %v", err) + klog.Warningf(err.Error()) } subnets := util.GetAllClusterSubnetsFromEntries(defaultNetInfo.Subnets()) - if err := InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, defaultNetInfo, subnets, e.controllerName); err != nil { + if err := InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, defaultNetInfo, subnets, e.controllerName, defaultNetInfo.GetNetworkScopedClusterRouterName()); err != nil { return fmt.Errorf("failed to initialize networks cluster logical router egress policies for the default network: %v", err) } - for _, node := range nodes { - node := node.(*kapi.Node) - if err := DeleteLegacyDefaultNoRerouteNodePolicies(e.nbClient, defaultNetInfo.GetNetworkScopedClusterRouterName(), node.Name); err != nil { - return fmt.Errorf("failed to delete legacy default no reroute to nodes for node %s: %v", node.Name, err) - } - } - return e.nadController.DoWithLock(func(network util.NetInfo) error { + + return e.networkManager.DoWithLock(func(network util.NetInfo) error { if network.GetNetworkName() == types.DefaultNetworkName { return nil } subnets = util.GetAllClusterSubnetsFromEntries(network.Subnets()) - if err := InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, network, subnets, e.controllerName); err != nil { - return fmt.Errorf("failed to initialize networks cluster logical router egress policies for network %s: %v", network.GetNetworkName(), err) + if len(subnets) == 0 { + return nil } - for _, node := range nodes { - node := node.(*kapi.Node) - if err := DeleteLegacyDefaultNoRerouteNodePolicies(e.nbClient, network.GetNetworkScopedClusterRouterName(), node.Name); err != nil { - return fmt.Errorf("failed to delete legacy default no reroute node policies for node %s and network %s: %v", node.Name, network.GetNetworkName(), err) - } + routerName, err := getTopologyScopedRouterName(network, localNodeName) + if err != nil { + return err + } + if err = InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, network, subnets, e.controllerName, routerName); err != nil { + return fmt.Errorf("failed to initialize networks cluster logical router egress policies for network %s: %v", network.GetNetworkName(), err) } return nil }) @@ -2012,7 +2050,10 @@ func (e *EgressIPController) initClusterEgressPolicies(nodes []interface{}) erro // InitClusterEgressPolicies creates the global no reroute policies and address-sets // required by the egressIP and egressServices features. func InitClusterEgressPolicies(nbClient libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, ni util.NetInfo, - clusterSubnets []*net.IPNet, controllerName string) error { + clusterSubnets []*net.IPNet, controllerName, routerName string) error { + if len(clusterSubnets) == 0 { + return nil + } var v4ClusterSubnet, v6ClusterSubnet []*net.IPNet for _, subnet := range clusterSubnets { if utilnet.IsIPv6CIDR(subnet) { @@ -2041,15 +2082,14 @@ func InitClusterEgressPolicies(nbClient libovsdbclient.Client, addressSetFactory return fmt.Errorf("network %s: failed to parse IPv6 join subnet: %v", ni.GetNetworkName(), err) } } - router := ni.GetNetworkScopedClusterRouterName() - if err = createDefaultNoReroutePodPolicies(nbClient, ni.GetNetworkName(), controllerName, router, v4ClusterSubnet, v6ClusterSubnet); err != nil { + if err = createDefaultNoReroutePodPolicies(nbClient, ni.GetNetworkName(), controllerName, routerName, v4ClusterSubnet, v6ClusterSubnet); err != nil { return fmt.Errorf("failed to create no reroute policies for pods on network %s: %v", ni.GetNetworkName(), err) } - if err = createDefaultNoRerouteServicePolicies(nbClient, ni.GetNetworkName(), controllerName, router, v4ClusterSubnet, v6ClusterSubnet, + if err = createDefaultNoRerouteServicePolicies(nbClient, ni.GetNetworkName(), controllerName, routerName, v4ClusterSubnet, v6ClusterSubnet, v4JoinSubnet, v6JoinSubnet); err != nil { return fmt.Errorf("failed to create no reroute policies for services on network %s: %v", ni.GetNetworkName(), err) } - if err = createDefaultNoRerouteReplyTrafficPolicy(nbClient, ni.GetNetworkName(), controllerName, router); err != nil { + if err = createDefaultNoRerouteReplyTrafficPolicy(nbClient, ni.GetNetworkName(), controllerName, routerName); err != nil { return fmt.Errorf("failed to create no reroute reply traffic policy for network %s: %v", ni.GetNetworkName(), err) } @@ -2074,9 +2114,9 @@ func InitClusterEgressPolicies(nbClient libovsdbclient.Client, addressSetFactory return fmt.Errorf("cannot ensure that addressSet for egressService pods %s exists %v", egresssvc.EgressServiceServedPodsAddrSetName, err) } - if !ni.IsDefault() && util.IsNetworkSegmentationSupportEnabled() { + if !ni.IsDefault() && isEgressIPForUDNSupported() { v4, v6 := len(v4ClusterSubnet) > 0, len(v6ClusterSubnet) > 0 - if err = ensureDefaultNoRerouteUDNEnabledSvcPolicies(nbClient, addressSetFactory, ni, controllerName, v4, v6); err != nil { + if err = ensureDefaultNoRerouteUDNEnabledSvcPolicies(nbClient, addressSetFactory, ni, controllerName, routerName, v4, v6); err != nil { return fmt.Errorf("failed to ensure no reroute for UDN enabled services for network %s: %v", ni.GetNetworkName(), err) } } @@ -2234,44 +2274,72 @@ func (e *EgressIPController) addPodEgressIPAssignment(ni util.NetInfo, egressIPN } isOVNNetwork := util.IsOVNNetwork(parsedNodeEIPConfig, eIPIP) nextHopIP, err := e.getNextHop(ni, status.Node, status.EgressIP, egressIPName, isLocalZoneEgressNode, isOVNNetwork) - if err != nil || nextHopIP == "" { + if err != nil { return fmt.Errorf("failed to determine next hop for pod %s/%s when configuring egress IP %s"+ " IP %s: %v", pod.Namespace, pod.Name, egressIPName, status.EgressIP, err) } + if nextHopIP == "" { + return fmt.Errorf("could not calculate the next hop for pod %s/%s when configuring egress IP %s"+ + " IP %s", pod.Namespace, pod.Name, egressIPName, status.EgressIP) + } var ops []ovsdb.Operation if loadedEgressNode && isLocalZoneEgressNode { + // create NATs for CDNs only + // create LRPs with allow action (aka GW policy marks) only for L3 UDNs. + // L2 UDNs require LRPs with reroute action with a pkt_mark option attached to GW router. if isOVNNetwork { if ni.IsDefault() { ops, err = e.createNATRuleOps(ni, nil, podIPs, status, egressIPName, pod.Namespace, pod.Name) if err != nil { return fmt.Errorf("unable to create NAT rule ops for status: %v, err: %v", status, err) } - } else if ni.IsSecondary() { + + } else if ni.IsSecondary() && ni.TopologyType() == types.Layer3Topology { + // not required for L2 because we always have LRPs using reroute action to pkt mark ops, err = e.createGWMarkPolicyOps(ni, ops, podIPs, status, mark, pod.Namespace, pod.Name, egressIPName) if err != nil { return fmt.Errorf("unable to create GW router LRP ops to packet mark pod %s/%s: %v", pod.Namespace, pod.Name, err) } } } - if config.OVNKubernetesFeature.EnableInterconnect && !isOVNNetwork && (loadedPodNode && !isLocalZonePod) { - // configure reroute for non-local-zone pods on egress nodes - ops, err = e.createReroutePolicyOps(ni, ops, podIPs, status, mark, egressIPName, nextHopIP, pod.Namespace, pod.Name) + if config.OVNKubernetesFeature.EnableInterconnect && ni.IsDefault() && !isOVNNetwork && (loadedPodNode && !isLocalZonePod) { + // For CDNs, configure LRP with reroute action for non-local-zone pods on egress nodes to support redirect to local management port + // when the egress IP is assigned to a host secondary interface + routerName, err := getTopologyScopedRouterName(ni, pod.Spec.NodeName) + if err != nil { + return err + } + ops, err = e.createReroutePolicyOps(ni, ops, podIPs, status, mark, egressIPName, nextHopIP, routerName, pod.Namespace, pod.Name) if err != nil { return fmt.Errorf("unable to create logical router policy ops %v, err: %v", status, err) } } } - // exec when node is local OR when pods are local + // For L2, we always attach an LRP with reroute action to the Nodes gateway router. If the pod is remote, use the local zone Node name to generate the GW router name. + nodeName := pod.Spec.NodeName + if loadedEgressNode && loadedPodNode && !isLocalZonePod && isLocalZoneEgressNode && ni.IsSecondary() && ni.TopologyType() == types.Layer2Topology { + nodeName = status.Node + } + routerName, err := getTopologyScopedRouterName(ni, nodeName) + if err != nil { + return err + } + + // exec when node is local OR when pods are local or L2 UDN // don't add a reroute policy if the egress node towards which we are adding this doesn't exist - if loadedEgressNode && loadedPodNode && isLocalZonePod { - ops, err = e.createReroutePolicyOps(ni, ops, podIPs, status, mark, egressIPName, nextHopIP, pod.Namespace, pod.Name) - if err != nil { - return fmt.Errorf("unable to create logical router policy ops, err: %v", err) + if loadedEgressNode && loadedPodNode { + if isLocalZonePod || (isLocalZoneEgressNode && ni.IsSecondary() && ni.TopologyType() == types.Layer2Topology) { + ops, err = e.createReroutePolicyOps(ni, ops, podIPs, status, mark, egressIPName, nextHopIP, routerName, pod.Namespace, pod.Name) + if err != nil { + return fmt.Errorf("unable to create logical router policy ops, err: %v", err) + } } - ops, err = e.deleteExternalGWPodSNATOps(ni, ops, pod, podIPs, status, isOVNNetwork) - if err != nil { - return err + if isLocalZonePod { + ops, err = e.deleteExternalGWPodSNATOps(ni, ops, pod, podIPs, status, isOVNNetwork) + if err != nil { + return err + } } } _, err = libovsdbops.TransactAndCheck(e.nbClient, ops) @@ -2316,13 +2384,22 @@ func (e *EgressIPController) deletePodEgressIPAssignment(ni util.NetInfo, egress } } } + // For L2, we always attach an LRP with reroute action to the Nodes gateway router. If the pod is remote, use the local zone Node name to generate the GW router name. + nodeName := pod.Spec.NodeName + if !isLocalZonePod && isLocalZoneEgressNode && ni.IsSecondary() && ni.TopologyType() == types.Layer2Topology { + nodeName = status.Node + } + routerName, err := getTopologyScopedRouterName(ni, nodeName) + if err != nil { + return err + } var ops []ovsdb.Operation if !loadedPodNode || isLocalZonePod { // node is deleted (we can't determine zone so we always try and nuke OR pod is local to zone) ops, err = e.addExternalGWPodSNATOps(ni, nil, pod.Namespace, pod.Name, status) if err != nil { return err } - ops, err = e.deleteReroutePolicyOps(ni, ops, status, egressIPName, nextHopIP, pod.Namespace, pod.Name) + ops, err = e.deleteReroutePolicyOps(ni, ops, status, egressIPName, nextHopIP, routerName, pod.Namespace, pod.Name) if errors.Is(err, libovsdbclient.ErrNotFound) { // if the gateway router join IP setup is already gone, then don't count it as error. klog.Warningf("Unable to delete logical router policy, err: %v", err) @@ -2332,9 +2409,9 @@ func (e *EgressIPController) deletePodEgressIPAssignment(ni util.NetInfo, egress } if loadedEgressNode && isLocalZoneEgressNode { - if config.OVNKubernetesFeature.EnableInterconnect && !isOVNNetwork && (!loadedPodNode || !isLocalZonePod) { // node is deleted (we can't determine zone so we always try and nuke OR pod is remote to zone) - // delete reroute for non-local-zone pods on egress nodes - ops, err = e.deleteReroutePolicyOps(ni, ops, status, egressIPName, nextHopIP, pod.Namespace, pod.Name) + if config.OVNKubernetesFeature.EnableInterconnect && ni.IsDefault() && !isOVNNetwork && (!loadedPodNode || !isLocalZonePod) { // node is deleted (we can't determine zone so we always try and nuke OR pod is remote to zone) + // For CDNs, delete reroute for non-local-zone pods on egress nodes when the egress IP is assigned to a secondary host interface + ops, err = e.deleteReroutePolicyOps(ni, ops, status, egressIPName, nextHopIP, routerName, pod.Namespace, pod.Name) if err != nil { return fmt.Errorf("unable to delete logical router static route ops %v, err: %v", status, err) } @@ -2344,7 +2421,7 @@ func (e *EgressIPController) deletePodEgressIPAssignment(ni util.NetInfo, egress if err != nil { return fmt.Errorf("unable to delete NAT rule for status: %v, err: %v", status, err) } - } else if ni.IsSecondary() { + } else if ni.IsSecondary() && ni.TopologyType() == types.Layer3Topology { ops, err = e.deleteGWMarkPolicyOps(ni, ops, status, pod.Namespace, pod.Name, egressIPName) if err != nil { return fmt.Errorf("unable to create GW router packet mark LRPs delete ops for pod %s/%s: %v", pod.Namespace, pod.Name, err) @@ -2396,6 +2473,12 @@ func (e *EgressIPController) addExternalGWPodSNATOps(ni util.NetInfo, ops []ovsd if err != nil { return nil, nil // nothing to do. } + + if util.IsPodNetworkAdvertisedAtNode(ni, pod.Spec.NodeName) { + // network is advertised so don't setup the SNAT + return ops, nil + } + isLocalZonePod, loadedPodNode := e.nodeZoneState.Load(pod.Spec.NodeName) if pod.Spec.NodeName == status.Node && loadedPodNode && isLocalZonePod && util.PodNeedsSNAT(pod) { // if the pod still exists, add snats to->nodeIP (on the node where the pod exists) for these podIPs after deleting the snat to->egressIP @@ -2442,13 +2525,100 @@ func (e *EgressIPController) deleteExternalGWPodSNATOps(ni util.NetInfo, ops []o return ops, nil } -func (e *EgressIPController) getGatewayRouterJoinIP(ni util.NetInfo, node string, wantsIPv6 bool) (net.IP, error) { - gatewayIPs, err := libovsdbutil.GetLRPAddrs(e.nbClient, types.GWRouterToJoinSwitchPrefix+ni.GetNetworkScopedGWRouterName(node)) +// getGatewayNextHop determines the next hop for a given Node considering the network topology type +// For layer 3, next hop is gateway routers 'router to join' port IP +// For layer 2, it's the callers responsibility to ensure that the egress node is remote because a LRP should not be created +func (e *EgressIPController) getGatewayNextHop(ni util.NetInfo, nodeName string, isIPv6 bool) (net.IP, error) { + // fetch gateway router 'router to join' port IP + if ni.TopologyType() == types.Layer3Topology { + return e.getRouterPortIP(types.GWRouterToJoinSwitchPrefix+ni.GetNetworkScopedGWRouterName(nodeName), isIPv6) + } + + // If egress node is local, retrieve the external default gateway next hops from the Node L3 gateway annotation. + // We must pick one of the next hops to add to the LRP reroute next hops to not break ECMP. + // If an egress node is remote, retrieve the remote Nodes gateway router 'router to switch' port IP + // from the Node annotation. + // FIXME: remove gathering the required information from a Node annotations as this approach does not scale + // FIXME: we do not respect multiple default gateway next hops and instead pick the first IP that matches the IP family of the EIP + if ni.TopologyType() == types.Layer2Topology { + node, err := e.watchFactory.GetNode(nodeName) + if err != nil { + return nil, fmt.Errorf("failed to retrive node %s: %w", nodeName, err) + } + localNode, err := e.getALocalZoneNodeName() + if err != nil { + return nil, err + } + // Node is local + if localNode == nodeName { + nextHopIPs, err := util.ParseNodeL3GatewayAnnotation(node) + if err != nil { + if util.IsAnnotationNotSetError(err) { + // remote node may not have the annotation yet, suppress it + return nil, types.NewSuppressedError(err) + } + return nil, fmt.Errorf("failed to get the node %s L3 gateway annotation: %w", node.Name, err) + } + if len(nextHopIPs.NextHops) == 0 { + return nil, fmt.Errorf("l3 gateway annotation on the node %s has empty next hop IPs. Next hop is required for Layer 2 networks", node.Name) + } + ip, err := util.MatchFirstIPFamily(isIPv6, nextHopIPs.NextHops) + if err != nil { + return nil, fmt.Errorf("unable to find a next hop IP from the node %s L3 gateway annotation that is equal "+ + "to the EgressIP IP family (is IPv6: %v): %v", node.Name, isIPv6, err) + } + return ip, nil + } + // Node is remote + // fetch Node gateway routers 'router to switch' port IP + if isIPv6 { + return util.ParseNodeGatewayRouterJoinIPv6(node, ni.GetNetworkName()) + } + return util.ParseNodeGatewayRouterJoinIPv4(node, ni.GetNetworkName()) + } + return nil, fmt.Errorf("unsupported network topology %s", ni.TopologyType()) +} + +// getLocalMgmtPortNextHop attempts the given networks local management port. If the information is not available because +// of management port deletion, no error will be returned. +func (e *EgressIPController) getLocalMgmtPortNextHop(ni util.NetInfo, egressNodeName, egressIPName, egressIPIP string, isIPv6 bool) (string, error) { + mgmtPort := &nbdb.LogicalSwitchPort{Name: ni.GetNetworkScopedK8sMgmtIntfName(egressNodeName)} + mgmtPort, err := libovsdbops.GetLogicalSwitchPort(e.nbClient, mgmtPort) + if err != nil && !errors.Is(err, libovsdbclient.ErrNotFound) { + return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s "+ + "because unable to get management port: %v", egressIPName, egressNodeName, err) + } else if err != nil { + klog.Warningf("While attempting to get next hop for Egress IP %s (%s), unable to get management switch port: %v", + egressIPName, egressIPIP, err) + return "", nil + } + allMgmtPortAddresses := mgmtPort.GetAddresses() + if len(allMgmtPortAddresses) == 0 { + return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s"+ + "because failed to retrieve a MAC and IP address entry from management switch port %s", egressIPName, egressNodeName, mgmtPort.Name) + } + // select first MAC & IP(s) entry + mgmtPortAddresses := strings.Fields(allMgmtPortAddresses[0]) + if len(mgmtPortAddresses) < 2 { + return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s"+ + "because management switch port %s does not contain expected MAC address and one or more IP addresses", egressIPName, egressNodeName, mgmtPort.Name) + } + // filter out the MAC address which is always the first entry within the slice + mgmtPortAddresses = mgmtPortAddresses[1:] + nextHopIP, err := util.MatchIPStringFamily(isIPv6, mgmtPortAddresses) if err != nil { - return nil, fmt.Errorf("attempt at finding node gateway router network information failed, err: %w", err) + return "", fmt.Errorf("failed to find a management port %s IP matching the IP family of the EgressIP: %v", mgmtPort.Name, err) + } + return nextHopIP, nil +} + +func (e *EgressIPController) getRouterPortIP(portName string, wantsIPv6 bool) (net.IP, error) { + gatewayIPs, err := libovsdbutil.GetLRPAddrs(e.nbClient, portName) + if err != nil { + return nil, fmt.Errorf("failed to retrieve port %s IP(s): %w", portName, err) } if gatewayIP, err := util.MatchFirstIPNetFamily(wantsIPv6, gatewayIPs); err != nil { - return nil, fmt.Errorf("could not find gateway IP for node %s with family %v: %v", node, wantsIPv6, err) + return nil, fmt.Errorf("failed to find IP for port %s for IP family %v: %v", portName, wantsIPv6, err) } else { return gatewayIP.IP, nil } @@ -2479,65 +2649,48 @@ func (e *EgressIPController) getTransitIP(nodeName string, wantsIPv6 bool) (stri return nodeTransitIP.IP.String(), nil } -// getNextHop attempts to determine whether an egress IP should be routed through the primary OVN network or through -// a secondary host network. If we failed to look up the information required to determine this, an error will be returned -// however if we are able to lookup the information, but it doesnt exist, called must be able to tolerate a blank next hop -// and no error returned. This means we searched successfully but could not find the information required to generate the next hop. +// getNextHop attempts to determine whether an egress IP should be routed through the Nodes primary network interface (isOVNetwork = true) +// or through a secondary host network (isOVNNetwork = false). If we failed to look up the information required to determine this, an error will be returned +// however if the information to determine the next hop IP doesn't exist, caller must be able to tolerate a empty next hop +// and no error returned. This means we searched successfully but could not find the information required to generate the next hop IP. func (e *EgressIPController) getNextHop(ni util.NetInfo, egressNodeName, egressIP, egressIPName string, isLocalZoneEgressNode, isOVNNetwork bool) (string, error) { - var nextHopIP string - var err error isEgressIPv6 := utilnet.IsIPv6String(egressIP) - // NOTE: No need to check if status.node exists or not in the cache, we are calling this function only if it - // is present in the nodeZoneState cache. Since we call it with lock on cache, we are safe here. - if isLocalZoneEgressNode { + if isLocalZoneEgressNode || ni.TopologyType() == types.Layer2Topology { + // isOVNNetwork is true when an EgressIP is "assigned" to the Nodes primary interface (breth0). Ext traffic will egress breth0. + // is OVNNetwork is false when the EgressIP is assigned to a host secondary interface (not breth0). Ext traffic will egress this interface. if isOVNNetwork { - gatewayRouterIP, err := e.getGatewayRouterJoinIP(ni, egressNodeName, isEgressIPv6) + gatewayRouterIP, err := e.getGatewayNextHop(ni, egressNodeName, isEgressIPv6) + // return error only when we failed to retrieve the gateway IP. Do not return error when we can never get this IP (gw deleted) if err != nil && !errors.Is(err, libovsdbclient.ErrNotFound) { return "", fmt.Errorf("unable to retrieve gateway IP for node: %s, protocol is IPv6: %v, err: %w", egressNodeName, isEgressIPv6, err) } else if err != nil { - klog.Warningf("While attempting to get next hop for Egress IP %s (%s), unable to get gateway "+ - "router join IP: %v", egressIPName, egressIP, err) + klog.Warningf("While attempting to get next hop for Egress IP %s (%s), unable to get Node %s gateway "+ + "router IP: %v", egressIPName, egressIP, egressNodeName, err) return "", nil } - nextHopIP = gatewayRouterIP.String() + return gatewayRouterIP.String(), nil } else { - mgmtPort := &nbdb.LogicalSwitchPort{Name: ni.GetNetworkScopedK8sMgmtIntfName(egressNodeName)} - mgmtPort, err := libovsdbops.GetLogicalSwitchPort(e.nbClient, mgmtPort) - if err != nil && !errors.Is(err, libovsdbclient.ErrNotFound) { - return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s "+ - "because unable to get management port: %v", egressIPName, egressNodeName, err) - } else if err != nil { - klog.Warningf("While attempting to get next hop for Egress IP %s (%s), unable to get management switch port: %v", - egressIPName, egressIP, err) - return "", nil - } - mgmtPortAddresses := mgmtPort.GetAddresses() - if len(mgmtPortAddresses) == 0 { - return "", fmt.Errorf("failed to get next hop IP for secondary host network and egress IP %s for node %s"+ - "because management switch port does not contain any addresses", egressIPName, egressNodeName) - } - for _, mgmtPortAddress := range mgmtPortAddresses { - mgmtPortAddressesStr := strings.Fields(mgmtPortAddress) - mgmtPortIP := net.ParseIP(mgmtPortAddressesStr[1]) - if isEgressIPv6 && utilnet.IsIPv6(mgmtPortIP) { - nextHopIP = mgmtPortIP.To16().String() - } else { - nextHopIP = mgmtPortIP.To4().String() - } + // for an egress IP assigned to a host secondary interface, next hop IP is the networks management port IP + if ni.IsSecondary() { + return "", fmt.Errorf("egress IP assigned to a host secondary interface for a user defined network (network name %s) is unsupported", ni.GetNetworkName()) } + return e.getLocalMgmtPortNextHop(ni, egressNodeName, egressIPName, egressIP, isEgressIPv6) } - } else if config.OVNKubernetesFeature.EnableInterconnect { - // fetch node annotation of the egress node - nextHopIP, err = e.getTransitIP(egressNodeName, isEgressIPv6) + } + + if config.OVNKubernetesFeature.EnableInterconnect { + nextHopIP, err := e.getTransitIP(egressNodeName, isEgressIPv6) if err != nil && !errors.Is(err, libovsdbclient.ErrNotFound) { return "", fmt.Errorf("unable to fetch transit switch IP for node %s: %v", egressNodeName, err) } else if err != nil { klog.Warningf("While attempting to get next hop for Egress IP %s (%s), unable to get transit switch IP: %v", egressIPName, egressIP, err) + return "", nil } + return nextHopIP, nil } - return nextHopIP, nil + return "", nil } // createReroutePolicyOps creates an operation that does idempotent updates of the @@ -2553,7 +2706,7 @@ func (e *EgressIPController) getNextHop(ni util.NetInfo, egressNodeName, egressI // enabled, the appropriate transit switch port. // This function should be called with lock on nodeZoneState cache key status.Node func (e *EgressIPController) createReroutePolicyOps(ni util.NetInfo, ops []ovsdb.Operation, podIPNets []*net.IPNet, status egressipv1.EgressIPStatusItem, - mark util.EgressIPMark, egressIPName, nextHopIP, podNamespace, podName string) ([]ovsdb.Operation, error) { + mark util.EgressIPMark, egressIPName, nextHopIP, routerName, podNamespace, podName string) ([]ovsdb.Operation, error) { isEgressIPv6 := utilnet.IsIPv6String(status.EgressIP) ipFamily := getEIPIPFamily(isEgressIPv6) options := make(map[string]string) @@ -2563,10 +2716,10 @@ func (e *EgressIPController) createReroutePolicyOps(ni util.NetInfo, ops []ovsdb } addPktMarkToLRPOptions(options, mark.String()) } - var err error dbIDs := getEgressIPLRPReRouteDbIDs(egressIPName, podNamespace, podName, ipFamily, ni.GetNetworkName(), e.controllerName) p := libovsdbops.GetPredicate[*nbdb.LogicalRouterPolicy](dbIDs, nil) // Handle all pod IPs that match the egress IP address family + var err error for _, podIPNet := range util.MatchAllIPNetFamily(isEgressIPv6, podIPNets) { lrp := nbdb.LogicalRouterPolicy{ @@ -2577,9 +2730,9 @@ func (e *EgressIPController) createReroutePolicyOps(ni util.NetInfo, ops []ovsdb ExternalIDs: dbIDs.GetExternalIDs(), Options: options, } - ops, err = libovsdbops.CreateOrAddNextHopsToLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), &lrp, p) + ops, err = libovsdbops.CreateOrAddNextHopsToLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, routerName, &lrp, p) if err != nil { - return nil, fmt.Errorf("error creating logical router policy %+v on router %s: %v", lrp, ni.GetNetworkScopedClusterRouterName(), err) + return nil, fmt.Errorf("error creating logical router policy %+v on router %s: %v", lrp, routerName, err) } } return ops, nil @@ -2596,7 +2749,7 @@ func (e *EgressIPController) createReroutePolicyOps(ni util.NetInfo, ops []ovsdb // which will break HA momentarily // This function should be called with lock on nodeZoneState cache key status.Node func (e *EgressIPController) deleteReroutePolicyOps(ni util.NetInfo, ops []ovsdb.Operation, status egressipv1.EgressIPStatusItem, - egressIPName, nextHopIP, podNamespace, podName string) ([]ovsdb.Operation, error) { + egressIPName, nextHopIP, routerName, podNamespace, podName string) ([]ovsdb.Operation, error) { isEgressIPv6 := utilnet.IsIPv6String(status.EgressIP) ipFamily := getEIPIPFamily(isEgressIPv6) var err error @@ -2604,19 +2757,19 @@ func (e *EgressIPController) deleteReroutePolicyOps(ni util.NetInfo, ops []ovsdb dbIDs := getEgressIPLRPReRouteDbIDs(egressIPName, podNamespace, podName, ipFamily, ni.GetNetworkName(), e.controllerName) p := libovsdbops.GetPredicate[*nbdb.LogicalRouterPolicy](dbIDs, nil) if nextHopIP != "" { - ops, err = libovsdbops.DeleteNextHopFromLogicalRouterPoliciesWithPredicateOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), p, nextHopIP) + ops, err = libovsdbops.DeleteNextHopFromLogicalRouterPoliciesWithPredicateOps(e.nbClient, ops, routerName, p, nextHopIP) if err != nil { return nil, fmt.Errorf("error removing nexthop IP %s from egress ip %s policies on router %s: %v", - nextHopIP, egressIPName, ni.GetNetworkScopedClusterRouterName(), err) + nextHopIP, egressIPName, routerName, err) } } else { klog.Errorf("Caller failed to pass next hop for EgressIP %s and IP %s. Deleting all LRPs. This will break HA momentarily", egressIPName, status.EgressIP) // since next hop was not found, delete everything to ensure no stale entries however this will break load // balancing between hops, but we offer no guarantees except one of the EIPs will work - ops, err = libovsdbops.DeleteLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), p) + ops, err = libovsdbops.DeleteLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, routerName, p) if err != nil { - return nil, fmt.Errorf("failed to create logical router policy delete operations on %s: %v", ni.GetNetworkScopedClusterRouterName(), err) + return nil, fmt.Errorf("failed to create logical router policy delete operations on %s: %v", routerName, err) } } return ops, nil @@ -2725,12 +2878,20 @@ func (e *EgressIPController) deleteEgressIPStatusSetup(ni util.NetInfo, name str } if nextHopIP != "" { - ops, err = libovsdbops.DeleteNextHopFromLogicalRouterPoliciesWithPredicateOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), policyPredNextHop, nextHopIP) + router := ni.GetNetworkScopedClusterRouterName() + if ni.TopologyType() == types.Layer2Topology { + nodeName, err := e.getALocalZoneNodeName() + if err != nil { + return err + } + router = ni.GetNetworkScopedGWRouterName(nodeName) + } + ops, err = libovsdbops.DeleteNextHopFromLogicalRouterPoliciesWithPredicateOps(e.nbClient, ops, router, policyPredNextHop, nextHopIP) if err != nil { return fmt.Errorf("error removing nexthop IP %s from egress ip %s policies on router %s: %v", - nextHopIP, name, ni.GetNetworkScopedClusterRouterName(), err) + nextHopIP, name, router, err) } - } else if ops, err = e.ensureOnlyValidNextHops(ni, name, ops); err != nil { + } else if ops, err = e.ensureOnlyValidNextHops(ni, name, status.Node, ops); err != nil { return err } @@ -2762,7 +2923,7 @@ func (e *EgressIPController) deleteEgressIPStatusSetup(ni util.NetInfo, name str return nil } -func (e *EgressIPController) ensureOnlyValidNextHops(ni util.NetInfo, name string, ops []libovsdb.Operation) ([]libovsdb.Operation, error) { +func (e *EgressIPController) ensureOnlyValidNextHops(ni util.NetInfo, name, nodeName string, ops []libovsdb.Operation) ([]libovsdb.Operation, error) { // When no nextHopIP is found, This may happen when node object is already deleted. // So compare validNextHopIPs associated with current eIP.Status and Nexthops present // in the LogicalRouterPolicy, then delete nexthop(s) from LogicalRouterPolicy if @@ -2772,15 +2933,19 @@ func (e *EgressIPController) ensureOnlyValidNextHops(ni util.NetInfo, name strin strings.HasPrefix(item.ExternalIDs[libovsdbops.ObjectNameKey.String()], name+dbIDEIPNamePodDivider) && item.ExternalIDs[libovsdbops.NetworkKey.String()] == ni.GetNetworkName() } + routerName, err := getTopologyScopedRouterName(ni, nodeName) + if err != nil { + return ops, err + } eIP, err := e.watchFactory.GetEgressIP(name) if err != nil && !apierrors.IsNotFound(err) { return ops, fmt.Errorf("error retrieving EgressIP %s object for updating logical router policy nexthops, err: %w", name, err) } else if err != nil && apierrors.IsNotFound(err) { // EgressIP object is not found, so delete LRP associated with it. - ops, err = libovsdbops.DeleteLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), policyPred) + ops, err = libovsdbops.DeleteLogicalRouterPolicyWithPredicateOps(e.nbClient, ops, routerName, policyPred) if err != nil { return ops, fmt.Errorf("error creating ops to remove logical router policy for EgressIP %s from router %s: %v", - name, ni.GetNetworkScopedClusterRouterName(), err) + name, routerName, err) } } else { validNextHopIPs := make(sets.Set[string]) @@ -2798,10 +2963,10 @@ func (e *EgressIPController) ensureOnlyValidNextHops(ni util.NetInfo, name strin return ops, fmt.Errorf("error finding logical router policy for EgressIP %s: %v", name, err) } if len(validNextHopIPs) == 0 { - ops, err = libovsdbops.DeleteLogicalRouterPoliciesOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), reRoutePolicies...) + ops, err = libovsdbops.DeleteLogicalRouterPoliciesOps(e.nbClient, ops, routerName, reRoutePolicies...) if err != nil { return ops, fmt.Errorf("error creating ops to remove logical router policy for EgressIP %s from router %s: %v", - name, ni.GetNetworkScopedClusterRouterName(), err) + name, routerName, err) } return ops, nil } @@ -2810,10 +2975,10 @@ func (e *EgressIPController) ensureOnlyValidNextHops(ni util.NetInfo, name strin if validNextHopIPs.Has(nextHop) { continue } - ops, err = libovsdbops.DeleteNextHopsFromLogicalRouterPolicyOps(e.nbClient, ops, ni.GetNetworkScopedClusterRouterName(), []*nbdb.LogicalRouterPolicy{policy}, nextHop) + ops, err = libovsdbops.DeleteNextHopsFromLogicalRouterPolicyOps(e.nbClient, ops, routerName, []*nbdb.LogicalRouterPolicy{policy}, nextHop) if err != nil { return ops, fmt.Errorf("error creating ops to remove stale next hop IP %s from logical router policy for EgressIP %s from router %s: %v", - nextHop, name, ni.GetNetworkScopedClusterRouterName(), err) + nextHop, name, routerName, err) } } } @@ -2903,34 +3068,40 @@ func createDefaultNoRerouteServicePolicies(nbClient libovsdbclient.Client, netwo return nil } -func (e *EgressIPController) ensureL3ClusterRouterPoliciesForNetwork(ni util.NetInfo) error { +func (e *EgressIPController) ensureRouterPoliciesForNetwork(ni util.NetInfo) error { e.nodeUpdateMutex.Lock() defer e.nodeUpdateMutex.Unlock() - subnets := util.GetAllClusterSubnetsFromEntries(ni.Subnets()) - if err := InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, ni, subnets, e.controllerName); err != nil { + subnetEntries := ni.Subnets() + subnets := util.GetAllClusterSubnetsFromEntries(subnetEntries) + if len(subnets) == 0 { + return nil + } + localNode, err := e.getALocalZoneNodeName() + if err != nil { + return err + } + routerName, err := getTopologyScopedRouterName(ni, localNode) + if err != nil { + return err + } + if err := InitClusterEgressPolicies(e.nbClient, e.addressSetFactory, ni, subnets, e.controllerName, routerName); err != nil { return fmt.Errorf("failed to initialize networks cluster logical router egress policies for the default network: %v", err) } - err := ensureDefaultNoRerouteNodePolicies(e.nbClient, e.addressSetFactory, ni.GetNetworkName(), ni.GetNetworkScopedClusterRouterName(), + err = ensureDefaultNoRerouteNodePolicies(e.nbClient, e.addressSetFactory, ni.GetNetworkName(), routerName, e.controllerName, listers.NewNodeLister(e.watchFactory.NodeInformer().GetIndexer()), e.v4, e.v6) if err != nil { return fmt.Errorf("failed to ensure no reroute node policies for network %s: %v", ni.GetNetworkName(), err) } - if !config.OVNKubernetesFeature.EnableInterconnect { - return nil - } - nodes := e.nodeZoneState.GetKeys() - for _, node := range nodes { - if isLocal, ok := e.nodeZoneState.Load(node); ok && isLocal { - if err := libovsdbutil.CreateDefaultRouteToExternal(e.nbClient, ni.GetNetworkScopedClusterRouterName(), - ni.GetNetworkScopedGWRouterName(node), ni.Subnets()); err != nil { - return fmt.Errorf("failed to create route to external for network %s: %v", ni.GetNetworkName(), err) - } + if config.OVNKubernetesFeature.EnableInterconnect && ni.TopologyType() == types.Layer3Topology { + if err := libovsdbutil.CreateDefaultRouteToExternal(e.nbClient, routerName, + ni.GetNetworkScopedGWRouterName(localNode), subnetEntries); err != nil { + return fmt.Errorf("failed to create route to external for network %s: %v", ni.GetNetworkName(), err) } } return nil } -func (e *EgressIPController) ensureL3SwitchPoliciesForNode(ni util.NetInfo, nodeName string) error { +func (e *EgressIPController) ensureSwitchPoliciesForNode(ni util.NetInfo, nodeName string) error { e.nodeUpdateMutex.Lock() defer e.nodeUpdateMutex.Unlock() ops, err := e.ensureDefaultNoReRouteQosRulesForNode(ni, nodeName, nil) @@ -2946,10 +3117,10 @@ func (e *EgressIPController) ensureL3SwitchPoliciesForNode(ni util.NetInfo, node // createDefaultNoRerouteReplyTrafficPolicies ensures any traffic which is a response/reply from the egressIP pods // will not be re-routed to egress-nodes. This ensures EIP can work well with ETP=local // this policy is ipFamily neutral -func createDefaultNoRerouteReplyTrafficPolicy(nbClient libovsdbclient.Client, network, controller, clusterRouter string) error { +func createDefaultNoRerouteReplyTrafficPolicy(nbClient libovsdbclient.Client, network, controller, routerName string) error { match := fmt.Sprintf("pkt.mark == %d", types.EgressIPReplyTrafficConnectionMark) dbIDs := getEgressIPLRPNoReRouteDbIDs(types.DefaultNoRereoutePriority, ReplyTrafficNoReroute, IPFamilyValue, network, controller) - if err := createLogicalRouterPolicy(nbClient, clusterRouter, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { + if err := createLogicalRouterPolicy(nbClient, routerName, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { return fmt.Errorf("unable to create no-reroute reply traffic policies, err: %v", err) } return nil @@ -2957,18 +3128,18 @@ func createDefaultNoRerouteReplyTrafficPolicy(nbClient libovsdbclient.Client, ne // createDefaultNoReroutePodPolicies ensures egress pods east<->west traffic with regular pods, // i.e: ensuring that an egress pod can still communicate with a regular pod / service backed by regular pods -func createDefaultNoReroutePodPolicies(nbClient libovsdbclient.Client, network, controller, clusterRouter string, v4ClusterSubnet, v6ClusterSubnet []*net.IPNet) error { +func createDefaultNoReroutePodPolicies(nbClient libovsdbclient.Client, network, controller, routerName string, v4ClusterSubnet, v6ClusterSubnet []*net.IPNet) error { for _, v4Subnet := range v4ClusterSubnet { match := fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Subnet.String(), v4Subnet.String()) dbIDs := getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, network, controller) - if err := createLogicalRouterPolicy(nbClient, clusterRouter, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { + if err := createLogicalRouterPolicy(nbClient, routerName, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { return fmt.Errorf("unable to create IPv4 no-reroute pod policies, err: %v", err) } } for _, v6Subnet := range v6ClusterSubnet { match := fmt.Sprintf("ip6.src == %s && ip6.dst == %s", v6Subnet.String(), v6Subnet.String()) dbIDs := getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV6, network, controller) - if err := createLogicalRouterPolicy(nbClient, clusterRouter, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { + if err := createLogicalRouterPolicy(nbClient, routerName, match, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { return fmt.Errorf("unable to create IPv6 no-reroute pod policies, err: %v", err) } } @@ -3028,16 +3199,13 @@ func createDefaultReRouteQoSRuleOps(nbClient libovsdbclient.Client, addressSetFa func (e *EgressIPController) ensureDefaultNoRerouteQoSRules(nodeName string) error { e.nodeUpdateMutex.Lock() defer e.nodeUpdateMutex.Unlock() - defaultNetInfo, err := e.nadController.GetNetwork(types.DefaultNetworkName) - if err != nil { - return fmt.Errorf("failed to get default network from NAD controller: %v", err) - } + defaultNetInfo := e.networkManager.GetNetwork(types.DefaultNetworkName) var ops []libovsdb.Operation - ops, err = e.ensureDefaultNoReRouteQosRulesForNode(defaultNetInfo, nodeName, ops) + ops, err := e.ensureDefaultNoReRouteQosRulesForNode(defaultNetInfo, nodeName, ops) if err != nil { return fmt.Errorf("failed to process default network: %v", err) } - if err = e.nadController.DoWithLock(func(network util.NetInfo) error { + if err = e.networkManager.DoWithLock(func(network util.NetInfo) error { if network.GetNetworkName() == types.DefaultNetworkName { return nil } @@ -3133,23 +3301,29 @@ func (e *EgressIPController) ensureDefaultNoRerouteNodePolicies() error { defer e.nodeUpdateMutex.Unlock() nodeLister := listers.NewNodeLister(e.watchFactory.NodeInformer().GetIndexer()) // ensure default network is processed - defaultNetInfo, err := e.nadController.GetNetwork(types.DefaultNetworkName) - if err != nil { - return fmt.Errorf("failed to get default network: %v", err) - } - err = ensureDefaultNoRerouteNodePolicies(e.nbClient, e.addressSetFactory, defaultNetInfo.GetNetworkName(), defaultNetInfo.GetNetworkScopedClusterRouterName(), + defaultNetInfo := e.networkManager.GetNetwork(types.DefaultNetworkName) + err := ensureDefaultNoRerouteNodePolicies(e.nbClient, e.addressSetFactory, defaultNetInfo.GetNetworkName(), defaultNetInfo.GetNetworkScopedClusterRouterName(), e.controllerName, nodeLister, e.v4, e.v6) if err != nil { return fmt.Errorf("failed to ensure default no reroute policies for nodes for default network: %v", err) } - if !util.IsNetworkSegmentationSupportEnabled() { + if !isEgressIPForUDNSupported() { return nil } - if err = e.nadController.DoWithLock(func(network util.NetInfo) error { + if err = e.networkManager.DoWithLock(func(network util.NetInfo) error { if network.GetNetworkName() == types.DefaultNetworkName { return nil } - err = ensureDefaultNoRerouteNodePolicies(e.nbClient, e.addressSetFactory, network.GetNetworkName(), network.GetNetworkScopedClusterRouterName(), + routerName := network.GetNetworkScopedClusterRouterName() + if network.TopologyType() == types.Layer2Topology { + // assume one node per zone only. Multi nodes per zone not supported. + nodeName, err := e.getALocalZoneNodeName() + if err != nil { + return err + } + routerName = network.GetNetworkScopedGWRouterName(nodeName) + } + err = ensureDefaultNoRerouteNodePolicies(e.nbClient, e.addressSetFactory, network.GetNetworkName(), routerName, e.controllerName, nodeLister, e.v4, e.v6) if err != nil { return fmt.Errorf("failed to ensure default no reroute policies for nodes for network %s: %v", network.GetNetworkName(), err) @@ -3311,23 +3485,6 @@ func createLogicalRouterPolicy(nbClient libovsdbclient.Client, clusterRouter, ma return nil } -// DeleteLegacyDefaultNoRerouteNodePolicies deletes the older EIP node reroute policies -// called from syncFunction and is a one time operation -// sample: 101 ip4.src == 10.244.0.0/16 && ip4.dst == 172.18.0.2/32 allow -func DeleteLegacyDefaultNoRerouteNodePolicies(nbClient libovsdbclient.Client, clusterRouter, node string) error { - p := func(item *nbdb.LogicalRouterPolicy) bool { - if item.Priority != types.DefaultNoRereoutePriority { - return false - } - nodeName, ok := item.ExternalIDs["node"] - if !ok { - return false - } - return nodeName == node - } - return libovsdbops.DeleteLogicalRouterPoliciesWithPredicate(nbClient, clusterRouter, p) -} - func (e *EgressIPController) buildSNATFromEgressIPStatus(ni util.NetInfo, podIP net.IP, status egressipv1.EgressIPStatusItem, egressIPName, podNamespace, podName string) (*nbdb.NAT, error) { logicalIP := &net.IPNet{ IP: podIP, @@ -3406,7 +3563,7 @@ func (e *EgressIPController) getNetworkFromPodAssignment(podKey string) util.Net } func ensureDefaultNoRerouteUDNEnabledSvcPolicies(nbClient libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, - ni util.NetInfo, controllerName string, v4, v6 bool) error { + ni util.NetInfo, controllerName, routerName string, v4, v6 bool) error { var err error var as addressset.AddressSet // fetch the egressIP pods address-set @@ -3460,14 +3617,14 @@ func ensureDefaultNoRerouteUDNEnabledSvcPolicies(nbClient libovsdbclient.Client, // Create global allow policy for UDN enabled service traffic if v4 && matchV4 != "" { dbIDs = getEgressIPLRPNoReRouteDbIDs(types.DefaultNoRereoutePriority, NoReRouteUDNPodToCDNSvc, IPFamilyValueV4, ni.GetNetworkName(), controllerName) - if err := createLogicalRouterPolicy(nbClient, ni.GetNetworkScopedClusterRouterName(), matchV4, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { + if err := createLogicalRouterPolicy(nbClient, routerName, matchV4, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { return fmt.Errorf("unable to create IPv4 no-rerouteUDN pod to CDN svc, err: %v", err) } } if v6 && matchV6 != "" { dbIDs = getEgressIPLRPNoReRouteDbIDs(types.DefaultNoRereoutePriority, NoReRouteUDNPodToCDNSvc, IPFamilyValueV6, ni.GetNetworkName(), controllerName) - if err := createLogicalRouterPolicy(nbClient, ni.GetNetworkScopedClusterRouterName(), matchV6, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { + if err := createLogicalRouterPolicy(nbClient, routerName, matchV6, types.DefaultNoRereoutePriority, nil, dbIDs); err != nil { return fmt.Errorf("unable to create IPv6 no-reroute UDN pod to CDN svc policies, err: %v", err) } } @@ -3486,7 +3643,7 @@ func getPodNamespaceAndNameFromKey(podKey string) (string, string) { func getEgressIPPktMark(eipName string, annotations map[string]string) util.EgressIPMark { var err error var mark util.EgressIPMark - if util.IsNetworkSegmentationSupportEnabled() && util.IsEgressIPMarkSet(annotations) { + if isEgressIPForUDNSupported() && util.IsEgressIPMarkSet(annotations) { mark, err = util.ParseEgressIPMark(annotations) if err != nil { klog.Errorf("Failed to get EgressIP %s packet mark from annotations: %v", eipName, err) @@ -3517,3 +3674,20 @@ func getEIPIPFamily(isIPv6 bool) egressIPFamilyValue { func addPktMarkToLRPOptions(options map[string]string, mark string) { options["pkt_mark"] = mark } + +// getTopologyScopedRouterName returns the router name that we attach polices to support EgressIP depending on network topology +// For Layer 3, we return the network scoped OVN "cluster router" name. For layer 2, we return a Nodes network scoped OVN gateway router name. +func getTopologyScopedRouterName(ni util.NetInfo, nodeName string) (string, error) { + if ni.TopologyType() == types.Layer2Topology { + if nodeName == "" { + return "", fmt.Errorf("node name is required to determine the Nodes gateway router name") + } + return ni.GetNetworkScopedGWRouterName(nodeName), nil + } + return ni.GetNetworkScopedClusterRouterName(), nil +} + +func isEgressIPForUDNSupported() bool { + return config.OVNKubernetesFeature.EnableInterconnect && + config.OVNKubernetesFeature.EnableNetworkSegmentation +} diff --git a/go-controller/pkg/ovn/egressip_test.go b/go-controller/pkg/ovn/egressip_test.go index c42e858489..02296a38cc 100644 --- a/go-controller/pkg/ovn/egressip_test.go +++ b/go-controller/pkg/ovn/egressip_test.go @@ -18,7 +18,6 @@ import ( libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" - ginkgotable "github.com/onsi/ginkgo/extensions/table" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -631,7 +630,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) }) - ginkgotable.DescribeTable("[OVN network] should perform proper OVN transactions when pod is created after node egress label switch", + ginkgo.DescribeTable("[OVN network] should perform proper OVN transactions when pod is created after node egress label switch", func(interconnect bool) { app.Action = func(ctx *cli.Context) error { config.OVNKubernetesFeature.EnableInterconnect = interconnect @@ -924,11 +923,11 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled", false), - ginkgotable.Entry("interconnect enabled", true), + ginkgo.Entry("interconnect disabled", false), + ginkgo.Entry("interconnect enabled", true), ) - ginkgotable.DescribeTable("[OVN network] using EgressNode retry should perform proper OVN transactions when pod is created after node egress label switch", + ginkgo.DescribeTable("[OVN network] using EgressNode retry should perform proper OVN transactions when pod is created after node egress label switch", func(interconnect bool) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -1312,11 +1311,11 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled", false), - ginkgotable.Entry("interconnect enabled", true), // all 3 nodes in same zone, so behaves like non-ic + ginkgo.Entry("interconnect disabled", false), + ginkgo.Entry("interconnect enabled", true), // all 3 nodes in same zone, so behaves like non-ic ) - ginkgotable.DescribeTable("[secondary host network] using EgressNode retry should perform proper OVN transactions when pod is created after node egress label switch", + ginkgo.DescribeTable("[secondary host network] using EgressNode retry should perform proper OVN transactions when pod is created after node egress label switch", func(interconnect bool) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -1724,11 +1723,11 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled", false), - ginkgotable.Entry("interconnect enabled", true), // all 3 nodes in same zone, so behaves like non-ic + ginkgo.Entry("interconnect disabled", false), + ginkgo.Entry("interconnect enabled", true), // all 3 nodes in same zone, so behaves like non-ic ) - ginkgotable.DescribeTable("[secondary host network] should perform proper OVN transactions when namespace and pod is created after node egress label switch", + ginkgo.DescribeTable("[secondary host network] should perform proper OVN transactions when namespace and pod is created after node egress label switch", func(interconnect bool, node1Zone, node2Zone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -1925,7 +1924,12 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" } egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) - egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) + servedPodIPs := []string{podV4IP} + // pod is located on node1, therefore if remote, we dont expect to see its IPs in the served pods address set + if node1Zone == "remote" { + servedPodIPs = nil + } + egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets(servedPodIPs, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) expectedDatabaseState := []libovsdbtest.TestData{ getReRoutePolicy(egressPod.Status.PodIP, "4", "reroute-UUID", reroutePolicyNextHop, getEgressIPLRPReRouteDbIDs(eIP.Name, egressPod.Namespace, egressPod.Name, IPFamilyValueV4, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName).GetExternalIDs()), @@ -2061,17 +2065,17 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), - ginkgotable.Entry("interconnect enabled; node1 and node2 in global zones", true, "global", "global"), + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), + ginkgo.Entry("interconnect enabled; node1 and node2 in global zones", true, "global", "global"), // will showcase localzone setup - master is in pod's zone where pod's reroute policy towards egressNode will be done. // NOTE: SNAT won't be visible because its in remote zone - ginkgotable.Entry("interconnect enabled; node1 in global and node2 in remote zones", true, "global", "remote"), + ginkgo.Entry("interconnect enabled; node1 in global and node2 in remote zones", true, "global", "remote"), // will showcase localzone setup - master is in egress node's zone where pod's SNAT policy and static route will be done. // NOTE: reroute policy won't be visible because its in remote zone (pod is in remote zone) - ginkgotable.Entry("interconnect enabled; node1 in remote and node2 in global zones", true, "remote", "global"), + ginkgo.Entry("interconnect enabled; node1 in remote and node2 in global zones", true, "remote", "global"), ) - ginkgotable.DescribeTable("[mixed networks] should perform proper OVN transactions when namespace and pod is created after node egress label switch", + ginkgo.DescribeTable("[mixed networks] should perform proper OVN transactions when namespace and pod is created after node egress label switch", func(interconnect bool, node1Zone, node2Zone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -2347,7 +2351,14 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" } nodeName := "k8s-node1" egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) - egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP, podV4IP2, podV4IP3, podV4IP4}, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) + servedPodIPs := []string{} + if node1Zone == "global" { + servedPodIPs = append(servedPodIPs, podV4IP, podV4IP2) + } + if node2Zone == "global" { + servedPodIPs = append(servedPodIPs, podV4IP3, podV4IP4) + } + egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets(servedPodIPs, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) ipNets, _ := util.ParseIPNets(append(node1IPv4Addresses, node2IPv4Addresses...)) egressNodeIPs := []string{} @@ -2497,21 +2508,21 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), - ginkgotable.Entry("interconnect enabled; node1 and node2 in global zones", true, "global", "global"), + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), + ginkgo.Entry("interconnect enabled; node1 and node2 in global zones", true, "global", "global"), // will showcase localzone setup - master is in pod's zone where pod's reroute policy towards egressNode will be done. // NOTE: SNAT won't be visible because its in remote zone - ginkgotable.Entry("interconnect enabled; node1 in global and node2 in remote zones", true, "global", "remote"), + ginkgo.Entry("interconnect enabled; node1 in global and node2 in remote zones", true, "global", "remote"), // will showcase localzone setup - master is in egress node's zone where pod's SNAT policy and static route will be done. // NOTE: reroute policy won't be visible because its in remote zone (pod is in remote zone) - ginkgotable.Entry("interconnect enabled; node1 in remote and node2 in global zones", true, "remote", "global"), + ginkgo.Entry("interconnect enabled; node1 in remote and node2 in global zones", true, "remote", "global"), ) }) ginkgo.Context("On node DELETE", func() { - ginkgotable.DescribeTable("should perform proper OVN transactions when node's gateway objects are already deleted", + ginkgo.DescribeTable("should perform proper OVN transactions when node's gateway objects are already deleted", func(interconnect bool, node1Zone, node2Zone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -2697,7 +2708,12 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" primarySNAT := getEIPSNAT(podV4IP, egressPod.Namespace, egressPod.Name, egressIP, expectedNatLogicalPort, DefaultNetworkControllerName) primarySNAT.UUID = "egressip-nat1-UUID" egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) - egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) + servedPodIPs := []string{podV4IP} + // pod is located on node1, therefore if remote, we don't expect to see its IPs in the served pods address set + if node1Zone == "remote" { + servedPodIPs = nil + } + egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets(servedPodIPs, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) ipNets, _ := util.ParseIPNets([]string{node1IPv4, node2IPv4}) egressNodeIPs := []string{} @@ -2951,18 +2967,18 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), - ginkgotable.Entry("interconnect enabled; node1 and node2 in global zones", true, "global", "global"), + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), + ginkgo.Entry("interconnect enabled; node1 and node2 in global zones", true, "global", "global"), // will showcase localzone setup - master is in pod's zone where pod's reroute policy towards egressNode will be done. // NOTE: SNAT won't be visible because its in remote zone - ginkgotable.Entry("interconnect enabled; node1 in global and node2 in remote zones", true, "global", "remote"), + ginkgo.Entry("interconnect enabled; node1 in global and node2 in remote zones", true, "global", "remote"), // will showcase localzone setup - master is in egress node's zone where pod's SNAT policy and static route will* be done. // * the static route won't be visible because the pod's node node1 is getting deleted in this test // NOTE: reroute policy won't be visible because its in remote zone (pod is in remote zone) - ginkgotable.Entry("interconnect enabled; node1 in remote and node2 in global zones", true, "remote", "global"), + ginkgo.Entry("interconnect enabled; node1 in remote and node2 in global zones", true, "remote", "global"), ) - ginkgotable.DescribeTable("[secondary host network] should perform proper OVN transactions when namespace and pod is created after node egress label switch", + ginkgo.DescribeTable("[secondary host network] should perform proper OVN transactions when namespace and pod is created after node egress label switch", func(interconnect bool, node1Zone, node2Zone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -3148,7 +3164,12 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" reroutePolicyNextHop = []string{"100.88.0.3"} // node2's transit switch portIP } egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) - egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) + servedPodIPs := []string{podV4IP} + // pod is located on node1, therefore if remote, we dont expect to see its IPs in the served pods address set + if node1Zone == "remote" { + servedPodIPs = nil + } + egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets(servedPodIPs, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) ipNets, _ := util.ParseIPNets(append(node1IPv4Addresses, node2IPv4OVN)) egressNodeIPs := []string{} for _, ipNet := range ipNets { @@ -3280,20 +3301,20 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), - ginkgotable.Entry("interconnect enabled; node1 and node2 in global zones", true, "global", "global"), + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), + ginkgo.Entry("interconnect enabled; node1 and node2 in global zones", true, "global", "global"), // will showcase localzone setup - master is in pod's zone where pod's reroute policy towards egressNode will be done. // NOTE: SNAT won't be visible because its in remote zone - ginkgotable.Entry("interconnect enabled; node1 in global and node2 in remote zones", true, "global", "remote"), + ginkgo.Entry("interconnect enabled; node1 in global and node2 in remote zones", true, "global", "remote"), // will showcase localzone setup - master is in egress node's zone where pod's SNAT policy and static route will be done. // NOTE: reroute policy won't be visible because its in remote zone (pod is in remote zone) - ginkgotable.Entry("interconnect enabled; node1 in remote and node2 in global zones", true, "remote", "global"), + ginkgo.Entry("interconnect enabled; node1 in remote and node2 in global zones", true, "remote", "global"), ) }) ginkgo.Context("IPv6 on pod UPDATE", func() { - ginkgotable.DescribeTable("should remove OVN pod egress setup when EgressIP stops matching pod label", + ginkgo.DescribeTable("should remove OVN pod egress setup when EgressIP stops matching pod label", func(interconnect, isnode1Local, isnode2Local bool) { config.OVNKubernetesFeature.EnableInterconnect = interconnect config.IPv6Mode = true @@ -3584,13 +3605,13 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, true, true), - ginkgotable.Entry("interconnect enabled; pod and egressnode are in local zone", true, true, true), - ginkgotable.Entry("interconnect enabled; pod is in local zone and egressnode is in remote zone", true, true, false), // snat won't be visible - ginkgotable.Entry("interconnect enabled; pod is in remote zone and egressnode is in local zone", true, false, true), + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, true, true), + ginkgo.Entry("interconnect enabled; pod and egressnode are in local zone", true, true, true), + ginkgo.Entry("interconnect enabled; pod is in local zone and egressnode is in remote zone", true, true, false), // snat won't be visible + ginkgo.Entry("interconnect enabled; pod is in remote zone and egressnode is in local zone", true, false, true), ) - ginkgotable.DescribeTable("egressIP pod retry should remove OVN pod egress setup when EgressIP stops matching pod label", + ginkgo.DescribeTable("egressIP pod retry should remove OVN pod egress setup when EgressIP stops matching pod label", func(interconnect bool, podZone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -3811,9 +3832,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global"), - ginkgotable.Entry("interconnect enabled; pod is in global zone", true, "global"), - ginkgotable.Entry("interconnect enabled; pod is in remote zone", true, "remote"), // static re-route is visible but reroute policy won't be + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global"), + ginkgo.Entry("interconnect enabled; pod is in global zone", true, "global"), + ginkgo.Entry("interconnect enabled; pod is in remote zone", true, "remote"), // static re-route is visible but reroute policy won't be ) ginkgo.It("should not treat pod update if pod already had assigned IP when it got the ADD", func() { @@ -4173,7 +4194,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" ginkgo.Context("On node DELETE", func() { - ginkgotable.DescribeTable("should treat pod update if pod did not have an assigned IP when it got the ADD", + ginkgo.DescribeTable("should treat pod update if pod did not have an assigned IP when it got the ADD", func(interconnect bool, podZone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -4348,9 +4369,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global"), - ginkgotable.Entry("interconnect enabled; pod is in global zone", true, "global"), - ginkgotable.Entry("interconnect enabled; pod is in remote zone", true, "remote"), // static re-route is visible but reroute policy won't be + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global"), + ginkgo.Entry("interconnect enabled; pod is in global zone", true, "global"), + ginkgo.Entry("interconnect enabled; pod is in remote zone", true, "remote"), // static re-route is visible but reroute policy won't be ) ginkgo.It("should not treat pod DELETE if pod did not have an assigned IP when it got the ADD and we receive a DELETE before the IP UPDATE", func() { @@ -4418,7 +4439,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" ginkgo.Context("IPv6 on namespace UPDATE", func() { - ginkgotable.DescribeTable("should remove OVN pod egress setup when EgressIP is deleted", + ginkgo.DescribeTable("should remove OVN pod egress setup when EgressIP is deleted", func(interconnect bool, podZone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -4637,12 +4658,12 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global"), - ginkgotable.Entry("interconnect enabled; pod is in global zone", true, "global"), - ginkgotable.Entry("interconnect enabled; pod is in remote zone", true, "remote"), + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global"), + ginkgo.Entry("interconnect enabled; pod is in global zone", true, "global"), + ginkgo.Entry("interconnect enabled; pod is in remote zone", true, "remote"), ) - ginkgotable.DescribeTable("egressIP retry should remove OVN pod egress setup when EgressIP is deleted", + ginkgo.DescribeTable("egressIP retry should remove OVN pod egress setup when EgressIP is deleted", func(interconnect bool, podZone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -4880,12 +4901,12 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global"), - ginkgotable.Entry("interconnect enabled; pod is in global zone", true, "global"), - ginkgotable.Entry("interconnect enabled; pod is in remote zone", true, "remote"), + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global"), + ginkgo.Entry("interconnect enabled; pod is in global zone", true, "global"), + ginkgo.Entry("interconnect enabled; pod is in remote zone", true, "remote"), ) - ginkgotable.DescribeTable("should remove OVN pod egress setup when EgressIP stops matching", + ginkgo.DescribeTable("should remove OVN pod egress setup when EgressIP stops matching", func(interconnect bool, podZone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -5120,9 +5141,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global"), - ginkgotable.Entry("interconnect enabled; pod is in global zone", true, "global"), - ginkgotable.Entry("interconnect enabled; pod is in remote zone", true, "remote"), + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global"), + ginkgo.Entry("interconnect enabled; pod is in global zone", true, "global"), + ginkgo.Entry("interconnect enabled; pod is in remote zone", true, "remote"), ) ginkgo.It("should not remove OVN pod egress setup when EgressIP stops matching, but pod never had any IP to begin with", func() { @@ -5190,7 +5211,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" }) ginkgo.Context("on EgressIP UPDATE", func() { - ginkgotable.DescribeTable("should update OVN on EgressIP .spec.egressips change", + ginkgo.DescribeTable("should update OVN on EgressIP .spec.egressips change", func(interconnect bool, node1Zone, node2Zone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -5406,7 +5427,11 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" } egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) - egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) + servedPodsIPs := []string{podV4IP} + if node1Zone == "remote" { + servedPodsIPs = nil + } + egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets(servedPodsIPs, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) expectedDatabaseState := []libovsdbtest.TestData{ @@ -5709,14 +5734,14 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), - ginkgotable.Entry("interconnect enabled; node1 and node2 in single zone", true, "global", "global"), + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), + ginkgo.Entry("interconnect enabled; node1 and node2 in single zone", true, "global", "global"), // will showcase localzone setup - master is in pod's zone where pod's reroute policy towards egressNode will be done. // NOTE: SNAT won't be visible because its in remote zone - ginkgotable.Entry("interconnect enabled; node1 in local and node2 in remote zones", true, "local", "remote"), + ginkgo.Entry("interconnect enabled; node1 in local and node2 in remote zones", true, "local", "remote"), // will showcase localzone setup - master is in egress node's zone where pod's SNAT policy and static route will be done. // NOTE: reroute policy won't be visible because its in remote zone (pod is in remote zone) - ginkgotable.Entry("interconnect enabled; node1 in remote and node2 in local zones", true, "remote", "local"), + ginkgo.Entry("interconnect enabled; node1 in remote and node2 in local zones", true, "remote", "local"), ) ginkgo.It("should delete and re-create and delete", func() { @@ -7044,7 +7069,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) - ginkgotable.DescribeTable("egressIP pod managed by multiple objects, verify standby works wells, verify syncPodAssignmentCache on restarts", + ginkgo.DescribeTable("egressIP pod managed by multiple objects, verify standby works wells, verify syncPodAssignmentCache on restarts", func(interconnect bool, node1Zone, node2Zone string) { config.OVNKubernetesFeature.EnableInterconnect = interconnect app.Action = func(ctx *cli.Context) error { @@ -7274,6 +7299,10 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" egressIPs2, nodes2 := getEgressIPStatus(egressIP2Name) gomega.Expect(nodes2[0]).To(gomega.Equal(node1.Name)) gomega.Expect(egressIPs2[0]).To(gomega.Equal(egressIP3)) + // egress node is node 1 and pod is located on node 1. If node 2 is local, there is not config + if !isNode1Local && isNode2Local { + return nil + } recordedEvent := <-fakeOvn.fakeRecorder.Events gomega.Expect(recordedEvent).To(gomega.ContainSubstring("EgressIP object egressip-2 will not be configured for pod egressip-namespace_egress-pod since another egressIP object egressip is serving it, this is undefined")) @@ -7688,14 +7717,14 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }, - ginkgotable.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), - ginkgotable.Entry("interconnect enabled; node1 and node2 in global zones", true, "global", "global"), + ginkgo.Entry("interconnect disabled; non-ic - single zone setup", false, "global", "global"), + ginkgo.Entry("interconnect enabled; node1 and node2 in global zones", true, "global", "global"), // will showcase localzone setup - master is in pod's zone where pod's reroute policy towards egressNode will be done. // NOTE: SNAT won't be visible because its in remote zone - ginkgotable.Entry("interconnect enabled; node1 in global and node2 in remote zones", true, "global", "remote"), + ginkgo.Entry("interconnect enabled; node1 in global and node2 in remote zones", true, "global", "remote"), // will showcase localzone setup - master is in egress node's zone where pod's SNAT policy and static route will be done. // NOTE: reroute policy won't be visible because its in remote zone (pod is in remote zone) - ginkgotable.Entry("interconnect enabled; node1 in remote and node2 in global zones", true, "remote", "global"), + ginkgo.Entry("interconnect enabled; node1 in remote and node2 in global zones", true, "remote", "global"), ) }) diff --git a/go-controller/pkg/ovn/egressip_udn_l2_test.go b/go-controller/pkg/ovn/egressip_udn_l2_test.go new file mode 100644 index 0000000000..a2c58654bf --- /dev/null +++ b/go-controller/pkg/ovn/egressip_udn_l2_test.go @@ -0,0 +1,2543 @@ +package ovn + +import ( + "context" + "fmt" + "net" + + cnitypes "github.com/containernetworking/cni/pkg/types" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + egressipv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/udnenabledsvc" + libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "github.com/urfave/cli/v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = ginkgo.Describe("EgressIP Operations for user defined network with topology L2", func() { + var ( + app *cli.App + fakeOvn *FakeOVN + ) + + const ( + nadName1 = "nad1" + networkName1 = "network1" + networkName1_ = networkName1 + "_" + node1Name = "node1" + v4Net1 = "20.128.0.0/14" + v4Node1Net1 = "20.128.0.0/16" + v4Pod1IPNode1Net1 = "20.128.0.5" + node1DefaultRtoJIP = "100.64.0.1" + node1DefaultRtoJIPCIDR = node1DefaultRtoJIP + "/16" + node1Network1RtoSIP = "100.65.0.2" + node1Network1RtoSIPCIDR = node1Network1RtoSIP + "/16" + podName3 = "egress-pod3" + v4Pod2IPNode1Net1 = "20.128.0.6" + v4Node1Tsp = "100.88.0.2" + node2Name = "node2" + v4Node2Net1 = "20.129.0.0/16" + v4Node2Tsp = "100.88.0.3" + podName4 = "egress-pod4" + v4Pod1IPNode2Net1 = "20.129.0.2" + v4Pod2IPNode2Net1 = "20.129.0.3" + node2DefaultRtoJIP = "100.66.0.1" + node2DefaultRtoJIPCIDR = node2DefaultRtoJIP + "/16" + node2Network1RtoSIP = "100.67.0.2" + node2Network1RtoSIPCIDR = node2Network1RtoSIP + "/16" + eIP1Mark = 50000 + eIP2Mark = 50001 + layer2SwitchName = "ovn_layer2_switch" + gwIP = "192.168.126.1" + gwIP2 = "192.168.127.1" + ) + + getEgressIPStatusLen := func(egressIPName string) func() int { + return func() int { + tmp, err := fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Get(context.TODO(), egressIPName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return len(tmp.Status.Items) + } + } + + getIPNetWithIP := func(cidr string) *net.IPNet { + ip, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + panic(err.Error()) + } + ipNet.IP = ip + return ipNet + } + + setPrimaryNetworkAnnot := func(pod *corev1.Pod, nadName, cidr string) { + var err error + hwAddr, _ := net.ParseMAC("00:00:5e:00:53:01") + pod.Annotations, err = util.MarshalPodAnnotation(pod.Annotations, + &util.PodAnnotation{ + IPs: []*net.IPNet{getIPNetWithIP(cidr)}, + MAC: hwAddr, + Role: "primary", + }, + nadName) + if err != nil { + panic(err.Error()) + } + } + + ginkgo.BeforeEach(func() { + // Restore global default values before each testcase + gomega.Expect(config.PrepareTestConfig()).Should(gomega.Succeed()) + config.OVNKubernetesFeature.EnableEgressIP = true + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableInterconnect = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + config.Gateway.Mode = config.GatewayModeShared + config.OVNKubernetesFeature.EgressIPNodeHealthCheckPort = 1234 + + app = cli.NewApp() + app.Name = "test" + app.Flags = config.Flags + + fakeOvn = NewFakeOVN(false) + }) + + ginkgo.AfterEach(func() { + fakeOvn.shutdown() + // Restore global default values + gomega.Expect(config.PrepareTestConfig()).Should(gomega.Succeed()) + }) + + ginkgo.Context("sync", func() { + ginkgo.It("should remove stale LRPs and configures missing LRPs", func() { + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + nadName := util.GetNADName(eipNamespace2, nadName1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, egressPodLabel) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, egressPodLabel) + egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) + egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNRemote, nadName, fmt.Sprintf("%s%s", v4Pod2IPNode2Net1, util.GetIPFullMaskString(v4Pod2IPNode2Net1))) + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop":"192.168.126.1", "next-hops": ["192.168.126.1"]}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + }, + }, + } + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP2, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), // stale gateway + getReRoutePolicyForController(egressIPName, eipNamespace2, podName, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP2, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), // stale pod + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName, IPFamilyValueV4, netInfo.GetNetworkName())}, // stale policies + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDNLocal, egressPodUDNLocal, egressPodCDNRemote, egressPodUDNRemote}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + &egressipv1.EgressIPList{ + Items: []egressipv1.EgressIP{eIP}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + fakeOvn.controller.logicalPortCache.add(&egressPodUDNLocal, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + fakeOvn.controller.eIPC.zone = node1.Name + fakeOvn.controller.zone = node1.Name + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(fakeOvn.networkManager.Start()).Should(gomega.Succeed()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID", "egressip-nat2-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP2, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNRemote.Namespace, egressPodCDNRemote.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.NAT{ + UUID: "egressip-nat2-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getReRoutePolicyForController(egressIPName, eipNamespace2, podName4, v4Pod2IPNode2Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName4, IPFamilyValueV4, netInfo.GetNetworkName()), + "udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", "udn-no-reroute-service-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), "udn-enabled-svc-no-reroute-UUID", + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("EgressIP update", func() { + ginkgo.It("should update UDN and CDN config", func() { + // Test steps: + // update an EIP selecting a pod on an UDN and another pod on a CDN + // EIP egresses locally and remote + // EIP egresses remote + // EIP egresses locally and remote + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + nadName := util.GetNADName(eipNamespace2, nadName1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, egressPodLabel) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, egressPodLabel) + egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) + egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNRemote, nadName, fmt.Sprintf("%s%s", v4Pod2IPNode2Net1, util.GetIPFullMaskString(v4Pod2IPNode2Net1))) + + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node1Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node2Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + twoNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: twoNodeStatus, + }, + } + ginkgo.By("create EgressIP that selects pods in a CDN and UDN") + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDNLocal, egressPodUDNLocal, egressPodCDNRemote, egressPodUDNRemote}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.zone = node1.Name + fakeOvn.eIPController.zone = node1.Name + fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + secConInfo, ok := fakeOvn.secondaryControllers[networkName1] + gomega.Expect(ok).To(gomega.BeTrue()) + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + err = fakeOvn.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // simulate Start() of secondary network controller + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(secConInfo.bnc.GetNetInfo()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(secConInfo.bnc.GetNetInfo(), node1Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + secConInfo.bnc.logicalPortCache.add(&egressPodUDNLocal, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + _, err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Create(context.TODO(), &eIP, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID", "egressip-nat2-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP2, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNRemote.Namespace, egressPodCDNRemote.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.NAT{ + UUID: "egressip-nat2-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getReRoutePolicyForController(egressIPName, eipNamespace2, podName4, v4Pod2IPNode2Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{ + "udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName4, IPFamilyValueV4, netInfo.GetNetworkName())}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + ginkgo.By("patch EgressIP status to ensure remote node is egressable only") + oneNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + err = patchEgressIP(fakeOvn.controller.kube.PatchEgressIP, eIP.Name, generateEgressIPPatches(eIP1Mark, oneNodeStatus)...) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(getEgressIPStatusLen(eIP.Name)).Should(gomega.Equal(1)) + expectedDatabaseStateOneEgressNode := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + Policies: []string{"udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName())}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateOneEgressNode)) + + ginkgo.By("restore both nodes as egressable") + err = patchEgressIP(fakeOvn.controller.kube.PatchEgressIP, eIP.Name, generateEgressIPPatches(eIP1Mark, twoNodeStatus)...) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(getEgressIPStatusLen(eIP.Name)).Should(gomega.Equal(2)) + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("EgressIP delete", func() { + ginkgo.It("should del UDN and CDN config", func() { + // Test steps: + // One EIP selecting a pod on an UDN and another pod on a CDN + // EIP egresses locally and remote + // Delete EIP + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + nadName := util.GetNADName(eipNamespace2, nadName1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, egressPodLabel) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, egressPodLabel) + egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, egressPodLabel) + egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) + egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNRemote, nadName, fmt.Sprintf("%s%s", v4Pod2IPNode2Net1, util.GetIPFullMaskString(v4Pod2IPNode2Net1))) + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node1Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node2Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + twoNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: twoNodeStatus, + }, + } + ginkgo.By("create EgressIP that selects pods in a CDN and UDN") + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDNLocal, egressPodUDNLocal, egressPodCDNRemote, egressPodUDNRemote}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + fakeOvn.controller.zone = node1.Name + fakeOvn.eIPController.zone = node1.Name + err = fakeOvn.eIPController.SyncLocalNodeZonesCache() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + secConInfo, ok := fakeOvn.secondaryControllers[networkName1] + gomega.Expect(ok).To(gomega.BeTrue()) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + secConInfo.bnc.logicalPortCache.add(&egressPodUDNLocal, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + _, err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Create(context.TODO(), &eIP, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID", "egressip-nat2-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP2, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNRemote.Namespace, egressPodCDNRemote.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.NAT{ + UUID: "egressip-nat2-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getReRoutePolicyForController(egressIPName, eipNamespace2, podName4, v4Pod2IPNode2Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{ + "udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName4, IPFamilyValueV4, netInfo.GetNetworkName()), + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + ginkgo.By("delete EgressIP") + err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Delete(context.TODO(), eIP.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressIPServedPodsASCDNv4.Addresses = nil + egressIPServedPodsASUDNv4.Addresses = nil + expectedDatabaseState := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + Policies: []string{"udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName())}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: secConInfo.bnc.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("Namespace update", func() { + ginkgo.It("should update UDN and CDN config", func() { + // Test steps: + // create an EIP not selecting a pod on an UDN and another pod on a CDN because namespace labels aren't selected + // EIP egresses locally and remote + // Update namespace to match EIP selectors + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, nil) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, nil) + egressPodCDN := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPodUDN := *newPodWithLabels(eipNamespace2, podName2, node1Name, podV4IP2, egressPodLabel) + + nadNsName := util.GetNADName(eipNamespace2, nadName1) + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadNsName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node1Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node2Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + twoNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: twoNodeStatus, + }, + } + ginkgo.By("create EgressIP that doesnt select pods in a CDN and UDN") + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDN, egressPodUDN}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + &egressipv1.EgressIPList{ + Items: []egressipv1.EgressIP{eIP}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.logicalPortCache.add(&egressPodCDN, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + fakeOvn.controller.zone = node1Name + fakeOvn.controller.eIPC.zone = node1Name + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + err = fakeOvn.eIPController.SyncLocalNodeZonesCache() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + secConInfo, ok := fakeOvn.secondaryControllers[networkName1] + gomega.Expect(ok).To(gomega.BeTrue()) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + secConInfo.bnc.logicalPortCache.add(&egressPodUDN, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + ginkgo.By("update namespaces with label so its now selected by EgressIP") + egressCDNNamespace.Labels = egressPodLabel + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.Background(), egressCDNNamespace, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressUDNNamespace.Labels = egressPodLabel + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.Background(), egressUDNNamespace, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDN.Namespace, egressPodCDN.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDN.Namespace, egressPodCDN.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{"udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", + "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName())}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("Namespace delete", func() { + ginkgo.It("should delete UDN and CDN config", func() { + // Test steps: + // create an EIP selecting a pod on an UDN and another pod on a CDN + // EIP egresses locally and remote + // Delete namespace + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, egressPodLabel) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, egressPodLabel) + egressPodCDN := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPodUDN := *newPodWithLabels(eipNamespace2, podName2, node1Name, podV4IP2, egressPodLabel) + + nadNsName := util.GetNADName(eipNamespace2, nadName1) + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadNsName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node1Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node2Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + twoNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: twoNodeStatus, + }, + } + ginkgo.By("create EgressIP that selects pods in a CDN and UDN") + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDN, egressPodUDN}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + &egressipv1.EgressIPList{ + Items: []egressipv1.EgressIP{eIP}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.logicalPortCache.add(&egressPodCDN, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + fakeOvn.controller.zone = node1Name + fakeOvn.eIPController.zone = node1Name + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + fakeOvn.controller.logicalPortCache.add(&egressPodUDN, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + err = fakeOvn.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDN.Namespace, egressPodCDN.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDN.Namespace, egressPodCDN.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{"udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", + "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName())}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("Pod update", func() { + ginkgo.It("should update UDN and CDN config", func() { + // Test steps: + // create an EIP no pods + // Create multiple pods, some selected by EIP selectors and some not + // EIP egresses locally and remote + app.Action = func(ctx *cli.Context) error { + // Node 1 is local, Node 2 is remote + egressIP1 := "192.168.126.101" + egressIP2 := "192.168.126.102" + node1IPv4 := "192.168.126.202" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + _, node1CDNSubnet, _ := net.ParseCIDR(v4Node1Subnet) + _, node1UDNSubnet, _ := net.ParseCIDR(v4Node1Net1) + nadName := util.GetNADName(eipNamespace2, nadName1) + egressCDNNamespace := newNamespaceWithLabels(eipNamespace, egressPodLabel) + egressUDNNamespace := newNamespaceWithLabels(eipNamespace2, egressPodLabel) + egressPodCDNLocal := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, nil) + egressPodUDNLocal := *newPodWithLabels(eipNamespace2, podName2, node1Name, v4Pod1IPNode1Net1, nil) + egressPodCDNRemote := *newPodWithLabels(eipNamespace, podName3, node2Name, podV4IP2, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodCDNRemote, ovntypes.DefaultNetworkName, fmt.Sprintf("%s%s", podV4IP2, util.GetIPFullMaskString(podV4IP2))) + egressPodUDNRemote := *newPodWithLabels(eipNamespace2, podName4, node2Name, v4Pod2IPNode2Net1, egressPodLabel) + setPrimaryNetworkAnnot(&egressPodUDNRemote, nadName, fmt.Sprintf("%s%s", v4Pod2IPNode2Net1, util.GetIPFullMaskString(v4Pod2IPNode2Net1))) + netconf := ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{ + Name: networkName1, + Type: "ovn-k8s-cni-overlay", + }, + Role: ovntypes.NetworkRolePrimary, + Topology: ovntypes.Layer2Topology, + NADName: nadName, + Subnets: v4Net1, + } + nad, err := newNetworkAttachmentDefinition( + eipNamespace2, + nadName1, + netconf, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + netInfo, err := util.NewNetInfo(&netconf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + node1Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node1IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node1Subnet, networkName1, v4Node1Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node1Tsp), + "k8s.ovn.org/zone-name": node1Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node1Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node1DefaultRtoJIPCIDR, networkName1, node1Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + labels := map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1 := getNodeObj(node1Name, node1Annotations, labels) + node2Annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", node2IPv4CIDR, ""), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\",\"%s\":\"%s\"}", v4Node2Subnet, networkName1, v4Node2Net1), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{\"ipv4\":\"%s/16\"}", v4Node2Tsp), + "k8s.ovn.org/zone-name": node2Name, + "k8s.ovn.org/node-chassis-id": "473ca66d-d800-472f-b289-1ab81ae7f21c", + "k8s.ovn.org/remote-zone-migrated": node2Name, + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": fmt.Sprintf("{\"default\":{\"ipv4\":\"%s\"},\"%s\":{\"ipv4\":\"%s\"}}", node2DefaultRtoJIPCIDR, networkName1, node2Network1RtoSIPCIDR), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + util.OvnNodeL3GatewayConfig: fmt.Sprintf(`{"%s":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"%s", "next-hops": ["%s"]}, +"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop": "192.168.126.1", "next-hops": ["192.168.126.1"]}}`, networkName1, v4Net1, gwIP, gwIP), + } + node2 := getNodeObj(node2Name, node2Annotations, labels) + twoNodeStatus := []egressipv1.EgressIPStatusItem{ + { + Node: node1Name, + EgressIP: egressIP1, + }, + { + Node: node2Name, + EgressIP: egressIP2, + }, + } + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMetaWithMark(egressIPName, eIP1Mark), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1, egressIP2}, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: twoNodeStatus, + }, + } + ginkgo.By("create EgressIP that doesnt select pods in a CDN and UDN") + initialDB := []libovsdbtest.TestData{ + //CDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + }, + // UDN start + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: netInfo.GetNetworkScopedSwitchName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedSwitchName(node1.Name), + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: networkName1, ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + }, + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: initialDB, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressCDNNamespace, *egressUDNNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPodCDNLocal, egressPodUDNLocal, egressPodCDNRemote, egressPodUDNRemote}, + }, + &nadv1.NetworkAttachmentDefinitionList{ + Items: []nadv1.NetworkAttachmentDefinition{*nad}, + }, + &egressipv1.EgressIPList{ + Items: []egressipv1.EgressIP{eIP}, + }, + ) + asf := addressset.NewOvnAddressSetFactory(fakeOvn.nbClient, true, false) + // watch EgressIP depends on UDN enabled svcs address set being available + c := udnenabledsvc.NewController(fakeOvn.nbClient, asf, fakeOvn.controller.watchFactory.ServiceCoreInformer(), []string{}) + go func() { + gomega.Expect(c.Run(ctx.Done())).Should(gomega.Succeed()) + }() + // Add pod IPs to CDN cache + iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") + nCDN.IP = iCDN + fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) + fakeOvn.controller.zone = node1Name + fakeOvn.eIPController.zone = node1Name + err = fakeOvn.eIPController.SyncLocalNodeZonesCache() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.networkManager.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + secConInfo, ok := fakeOvn.secondaryControllers[networkName1] + gomega.Expect(ok).To(gomega.BeTrue()) + // Add pod IPs to UDN cache + iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") + nUDN.IP = iUDN + secConInfo.bnc.logicalPortCache.add(&egressPodUDNLocal, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) + ginkgo.By("update pod with label so its now selected by EgressIP") + egressPodCDNLocal.Labels = egressPodLabel + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(eipNamespace).Update(context.Background(), &egressPodCDNLocal, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressPodUDNLocal.Labels = egressPodLabel + _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(eipNamespace2).Update(context.Background(), &egressPodUDNLocal, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASCDNv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, ovntypes.DefaultNetworkName, DefaultNetworkControllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4}) + egressIPServedPodsASUDNv4, _ := buildEgressIPServedPodsAddressSetsForController([]string{v4Pod1IPNode1Net1}, netInfo.GetNetworkName(), DefaultNetworkControllerName) + gomega.Eventually(c.IsAddressSetAvailable).Should(gomega.BeTrue()) + dbIDs := udnenabledsvc.GetAddressSetDBIDs() + udnEnabledSvcV4, _ := addressset.GetTestDbAddrSets(dbIDs, []string{}) + node1LRP := "k8s-node1" + expectedDatabaseStateTwoEgressNodes := []libovsdbtest.TestData{ + // CDN + getReRouteStaticRoute(v4ClusterSubnet, node1DefaultRtoJIP), + getReRoutePolicy(podV4IP, "4", "reroute-UUID", []string{node1DefaultRtoJIP, v4Node2Tsp}, + getEgressIPLRPReRouteDbIDs(eIP.Name, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, v4ClusterSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4ClusterSubnet, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: ovntypes.GWRouterPrefix + node1.Name, + UUID: ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"egressip-nat-UUID", "egressip-nat2-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"default-no-reroute-UUID", "no-reroute-service-UUID", + "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID"}, + StaticRoutes: []string{"reroute-static-route-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + node1.Name, + Networks: []string{node1DefaultRtoJIPCIDR}, + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASCDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, ovntypes.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1Name + "-UUID", + Name: "k8s-" + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1CDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1Name + "-UUID", + Name: node1Name, + Ports: []string{"k8s-" + node1Name + "-UUID"}, + QOSRules: []string{"default-QoS-UUID"}, + }, + &nbdb.NAT{ + UUID: "egressip-nat-UUID", + LogicalIP: podV4IP2, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNRemote.Namespace, egressPodCDNRemote.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.NAT{ + UUID: "egressip-nat2-UUID", + LogicalIP: podV4IP, + ExternalIP: egressIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPodCDNLocal.Namespace, egressPodCDNLocal.Name, IPFamilyValueV4, DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: &node1LRP, + Options: map[string]string{ + "stateless": "false", + }, + }, + getNoReRouteReplyTrafficPolicy(ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + getDefaultQoSRule(false, ovntypes.DefaultNetworkName, DefaultNetworkControllerName), + egressSVCServedPodsASv4, + egressIPServedPodsASCDNv4, + egressNodeIPsASv4, + + // UDN + getReRoutePolicyForController(egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP, node2Network1RtoSIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getReRoutePolicyForController(egressIPName, eipNamespace2, podName4, v4Pod2IPNode2Net1, eIP1Mark, IPFamilyValueV4, []string{gwIP}, netInfo.GetNetworkName(), DefaultNetworkControllerName), + getNoReRoutePolicyForUDNEnabledSvc(false, netInfo.GetNetworkName(), DefaultNetworkControllerName, egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, udnEnabledSvcV4.Name), + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, v4Net1), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4Net1, config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: ovntypes.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASUDNv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "udn-default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": ovntypes.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, netInfo.GetNetworkName(), DefaultNetworkControllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID", + Name: ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName, + Networks: []string{node1Network1RtoSIPCIDR}, + }, + &nbdb.LogicalRouter{ + UUID: netInfo.GetNetworkScopedGWRouterName(node1.Name) + "-UUID", + Name: netInfo.GetNetworkScopedGWRouterName(node1.Name), + Ports: []string{ovntypes.RouterToSwitchPrefix + networkName1_ + layer2SwitchName + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + Policies: []string{"udn-default-no-reroute-node-UUID", "udn-default-no-reroute-UUID", + "udn-no-reroute-service-UUID", "udn-enabled-svc-no-reroute-UUID", + fmt.Sprintf("%s-no-reroute-reply-traffic", netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName2, IPFamilyValueV4, netInfo.GetNetworkName()), + getReRoutePolicyUUID(eipNamespace2, podName4, IPFamilyValueV4, netInfo.GetNetworkName()), + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + networkName1_ + node1Name + "-UUID", + Name: "k8s-" + networkName1_ + node1Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr(node1UDNSubnet).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: networkName1_ + layer2SwitchName + "-UUID", + Name: networkName1_ + layer2SwitchName, + Ports: []string{"k8s-" + networkName1_ + node1Name + "-UUID"}, + ExternalIDs: map[string]string{ovntypes.NetworkExternalID: netInfo.GetNetworkName(), ovntypes.TopologyExternalID: ovntypes.Layer2Topology}, + QOSRules: []string{fmt.Sprintf("%s-QoS-UUID", netInfo.GetNetworkName())}, + }, + getNoReRouteReplyTrafficPolicyForController(netInfo.GetNetworkName(), DefaultNetworkControllerName), + getDefaultQoSRule(false, netInfo.GetNetworkName(), DefaultNetworkControllerName), + egressIPServedPodsASUDNv4, + udnEnabledSvcV4, + } + ginkgo.By("ensure expected equals actual") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseStateTwoEgressNodes)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) +}) diff --git a/go-controller/pkg/ovn/egressip_udn_l3_test.go b/go-controller/pkg/ovn/egressip_udn_l3_test.go index b0e4f6b7dc..e4336b0a10 100644 --- a/go-controller/pkg/ovn/egressip_udn_l3_test.go +++ b/go-controller/pkg/ovn/egressip_udn_l3_test.go @@ -220,9 +220,9 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol Ports: []string{"k8s-" + node1Name + "-UUID"}, }, // UDN start - //getGWPktMarkLRPForController(eIP1Mark, egressIPName, eipNamespace2, podName3, v4Pod2IPNode1Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), - //getGWPktMarkLRPForController(eIP2Mark, egressIPName, eipNamespace2, podName4, v4Pod1IPNode2Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), //stale EIP mark - //getGWPktMarkLRPForController(eIP2Mark, egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), //stale EIP mark + getGWPktMarkLRPForController(eIP1Mark, egressIPName, eipNamespace2, podName3, v4Pod2IPNode1Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), + getGWPktMarkLRPForController(eIP2Mark, egressIPName, eipNamespace2, podName4, v4Pod1IPNode2Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), //stale EIP mark + getGWPktMarkLRPForController(eIP2Mark, egressIPName, eipNamespace2, podName2, v4Pod1IPNode1Net1, IPFamilyValueV4, networkName1, DefaultNetworkControllerName), //stale EIP mark &nbdb.LogicalRouterPort{ UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + networkName1_ + node1.Name + "-UUID", Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + networkName1_ + node1.Name, @@ -291,11 +291,11 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) fakeOvn.controller.eIPC.zone = node1.Name fakeOvn.controller.zone = node1.Name - err = fakeOvn.eIPController.ensureL3ClusterRouterPoliciesForNetwork(netInfo) + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3SwitchPoliciesForNode(netInfo, node1Name) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(fakeOvn.controller.eIPC.nadController.Start()).Should(gomega.Succeed()) + gomega.Expect(fakeOvn.networkManager.Start()).Should(gomega.Succeed()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() @@ -655,12 +655,14 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) secConInfo, ok := fakeOvn.secondaryControllers[networkName1] gomega.Expect(ok).To(gomega.BeTrue()) - err = fakeOvn.nadController.Start() + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) // simulate Start() of secondary network controller - err = fakeOvn.eIPController.ensureL3ClusterRouterPoliciesForNetwork(secConInfo.bnc.NetInfo) + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(secConInfo.bnc.GetNetInfo()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3SwitchPoliciesForNode(secConInfo.bnc.NetInfo, node1Name) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(secConInfo.bnc.GetNetInfo(), node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1167,7 +1169,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) fakeOvn.controller.zone = node1.Name fakeOvn.eIPController.zone = node1.Name - err = fakeOvn.nadController.Start() + err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1660,11 +1662,13 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol fakeOvn.controller.logicalPortCache.add(&egressPodCDN, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) fakeOvn.controller.zone = node1Name fakeOvn.controller.eIPC.zone = node1Name - err = fakeOvn.nadController.Start() + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3ClusterRouterPoliciesForNetwork(netInfo) + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3SwitchPoliciesForNode(netInfo, node1Name) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2025,11 +2029,11 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol iUDN, nUDN, _ := net.ParseCIDR(v4Pod1IPNode1Net1 + "/23") nUDN.IP = iUDN fakeOvn.controller.logicalPortCache.add(&egressPodUDN, "", util.GetNADName(nad.Namespace, nad.Name), "", nil, []*net.IPNet{nUDN}) - err = fakeOvn.nadController.Start() + err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3ClusterRouterPoliciesForNetwork(netInfo) + err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.eIPController.ensureL3SwitchPoliciesForNode(netInfo, node1Name) + err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2373,7 +2377,7 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol iCDN, nCDN, _ := net.ParseCIDR(podV4IP + "/23") nCDN.IP = iCDN fakeOvn.controller.logicalPortCache.add(&egressPodCDNLocal, "", ovntypes.DefaultNetworkName, "", nil, []*net.IPNet{nCDN}) - err = fakeOvn.nadController.Start() + err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) fakeOvn.controller.zone = node1Name fakeOvn.eIPController.zone = node1Name diff --git a/go-controller/pkg/ovn/egressqos.go b/go-controller/pkg/ovn/egressqos.go index 5a993ecbbf..d717a7d995 100644 --- a/go-controller/pkg/ovn/egressqos.go +++ b/go-controller/pkg/ovn/egressqos.go @@ -168,7 +168,7 @@ func (oc *DefaultNetworkController) createASForEgressQoSRule(podSelector metav1. for _, pod := range pods { // we don't handle HostNetworked or completed pods or not-scheduled pods or remote-zone pods if !util.PodWantsHostNetwork(pod) && !util.PodCompleted(pod) && util.PodScheduled(pod) && oc.isPodScheduledinLocalZone(pod) { - podIPs, err := util.GetPodIPsOfNetwork(pod, oc.NetInfo) + podIPs, err := util.GetPodIPsOfNetwork(pod, oc.GetNetInfo()) if err != nil && !errors.Is(err, util.ErrNoPodIPFound) { return nil, nil, err } @@ -754,7 +754,7 @@ func (oc *DefaultNetworkController) syncEgressQoSPod(key string) error { return nil } - podIPs, err := util.GetPodIPsOfNetwork(pod, oc.NetInfo) + podIPs, err := util.GetPodIPsOfNetwork(pod, oc.GetNetInfo()) if errors.Is(err, util.ErrNoPodIPFound) { return nil // reprocess it when it is updated with an IP } @@ -845,8 +845,8 @@ func (oc *DefaultNetworkController) onEgressQoSPodUpdate(oldObj, newObj interfac oldPodLabels := labels.Set(oldPod.Labels) newPodLabels := labels.Set(newPod.Labels) - oldPodIPs, _ := util.GetPodIPsOfNetwork(oldPod, oc.NetInfo) - newPodIPs, _ := util.GetPodIPsOfNetwork(newPod, oc.NetInfo) + oldPodIPs, _ := util.GetPodIPsOfNetwork(oldPod, oc.GetNetInfo()) + newPodIPs, _ := util.GetPodIPsOfNetwork(newPod, oc.GetNetInfo()) isOldPodLocal := oc.isPodScheduledinLocalZone(oldPod) isNewPodLocal := oc.isPodScheduledinLocalZone(newPod) oldPodCompleted := util.PodCompleted(oldPod) diff --git a/go-controller/pkg/ovn/egressservices_test.go b/go-controller/pkg/ovn/egressservices_test.go index f11bf28308..78091a310b 100644 --- a/go-controller/pkg/ovn/egressservices_test.go +++ b/go-controller/pkg/ovn/egressservices_test.go @@ -5,7 +5,6 @@ import ( "fmt" "net" - ginkgotable "github.com/onsi/ginkgo/extensions/table" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -569,7 +568,7 @@ var _ = ginkgo.Describe("OVN Egress Service Operations", func() { ginkgo.Context("on egress service changes", func() { - ginkgotable.DescribeTable("should create/update/delete OVN configuration", func(interconnectEnabled bool) { + ginkgo.DescribeTable("should create/update/delete OVN configuration", func(interconnectEnabled bool) { app.Action = func(ctx *cli.Context) error { namespaceT := *newNamespace("testns") config.IPv6Mode = true @@ -781,11 +780,11 @@ var _ = ginkgo.Describe("OVN Egress Service Operations", func() { err := app.Run([]string{app.Name}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) }, - ginkgotable.Entry("IC Disabled, all nodes are in a single zone", false), - ginkgotable.Entry("IC Enabled, node1 is in the local zone, node2 in remote", true), + ginkgo.Entry("IC Disabled, all nodes are in a single zone", false), + ginkgo.Entry("IC Enabled, node1 is in the local zone, node2 in remote", true), ) - ginkgotable.DescribeTable("should delete resources when host changes to ALL", func(interconnectEnabled bool) { + ginkgo.DescribeTable("should delete resources when host changes to ALL", func(interconnectEnabled bool) { app.Action = func(ctx *cli.Context) error { namespaceT := *newNamespace("testns") config.IPv6Mode = true @@ -948,12 +947,12 @@ var _ = ginkgo.Describe("OVN Egress Service Operations", func() { err := app.Run([]string{app.Name}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) }, - ginkgotable.Entry("IC Disabled, all nodes are in a single zone", false), - ginkgotable.Entry("IC Enabled, node1 is in the local zone, node2 in remote", true)) + ginkgo.Entry("IC Disabled, all nodes are in a single zone", false), + ginkgo.Entry("IC Enabled, node1 is in the local zone, node2 in remote", true)) }) ginkgo.Context("on endpointslices changes", func() { - ginkgotable.DescribeTable("should create/update/delete OVN configuration", func(interconnectEnabled bool) { + ginkgo.DescribeTable("should create/update/delete OVN configuration", func(interconnectEnabled bool) { app.Action = func(ctx *cli.Context) error { namespaceT := *newNamespace("testns") config.IPv6Mode = true @@ -1265,12 +1264,12 @@ var _ = ginkgo.Describe("OVN Egress Service Operations", func() { err := app.Run([]string{app.Name}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) }, - ginkgotable.Entry("IC Disabled, all nodes are in a single zone", false), - ginkgotable.Entry("IC Enabled, node1 is in the local zone, node2 in remote", true)) + ginkgo.Entry("IC Disabled, all nodes are in a single zone", false), + ginkgo.Entry("IC Enabled, node1 is in the local zone, node2 in remote", true)) }) ginkgo.Context("on nodes changes", func() { - ginkgotable.DescribeTable("should create/update/delete logical router policies and address sets", func(interconnectEnabled bool) { + ginkgo.DescribeTable("should create/update/delete logical router policies and address sets", func(interconnectEnabled bool) { app.Action = func(ctx *cli.Context) error { namespaceT := *newNamespace("testns") config.IPv6Mode = true @@ -1570,8 +1569,8 @@ var _ = ginkgo.Describe("OVN Egress Service Operations", func() { err := app.Run([]string{app.Name}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) }, - ginkgotable.Entry("IC Disabled, all nodes are in a single zone", false), - ginkgotable.Entry("IC Enabled, node1 is in the local zone, node2 in remote", true)) + ginkgo.Entry("IC Disabled, all nodes are in a single zone", false), + ginkgo.Entry("IC Enabled, node1 is in the local zone, node2 in remote", true)) }) }) diff --git a/go-controller/pkg/ovn/external_ids_syncer/logical_router_policy/logical_router_policy_sync_test.go b/go-controller/pkg/ovn/external_ids_syncer/logical_router_policy/logical_router_policy_sync_test.go index 42396a7563..e041b21adb 100644 --- a/go-controller/pkg/ovn/external_ids_syncer/logical_router_policy/logical_router_policy_sync_test.go +++ b/go-controller/pkg/ovn/external_ids_syncer/logical_router_policy/logical_router_policy_sync_test.go @@ -12,7 +12,6 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - ginkgotable "github.com/onsi/ginkgo/extensions/table" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" ) @@ -66,12 +65,12 @@ var _ = ginkgo.Describe("OVN Logical Router Syncer", func() { ) ginkgo.Context("EgressIP", func() { - ginkgotable.DescribeTable("reroutes", func(sync lrpSync) { + ginkgo.DescribeTable("reroutes", func(sync lrpSync) { // pod reroutes may not have any owner references besides 'name' which equals EgressIP name performTest(defaultNetworkControllerName, sync.initialLRPs, sync.finalLRPs, sync.v4ClusterSubnets, sync.v6ClusterSubnets, sync.v4JoinSubnet, sync.v6JoinSubnet, sync.pods) }, - ginkgotable.Entry("add reference to IPv4 LRP with no reference", lrpSync{ + ginkgo.Entry("add reference to IPv4 LRP with no reference", lrpSync{ initialLRPs: []*nbdb.LogicalRouterPolicy{getReRouteLRP(podNamespace, podName, v4PodIPStr, 0, v4IPFamilyValue, v4PodNextHops, map[string]string{"name": egressIPName}, defaultNetworkControllerName)}, finalLRPs: []*nbdb.LogicalRouterPolicy{getReRouteLRP(podNamespace, podName, v4PodIPStr, 0, v4IPFamilyValue, v4PodNextHops, @@ -82,7 +81,7 @@ var _ = ginkgo.Describe("OVN Logical Router Syncer", func() { v4JoinSubnet: v4JoinSubnet, pods: podsNetInfo{v4PodNetInfo, v6PodNetInfo, v4Pod2NetInfo}, }), - ginkgotable.Entry("add reference to IPv6 LRP with no reference", lrpSync{ + ginkgo.Entry("add reference to IPv6 LRP with no reference", lrpSync{ initialLRPs: []*nbdb.LogicalRouterPolicy{getReRouteLRP(podNamespace, podName, v6PodIPStr, 0, v6IPFamilyValue, v6PodNextHops, map[string]string{"name": egressIPName}, defaultNetworkControllerName)}, finalLRPs: []*nbdb.LogicalRouterPolicy{getReRouteLRP(podNamespace, podName, v6PodIPStr, 0, v6IPFamilyValue, v6PodNextHops, @@ -92,7 +91,7 @@ var _ = ginkgo.Describe("OVN Logical Router Syncer", func() { v6JoinSubnet: v6JoinSubnet, pods: podsNetInfo{v4PodNetInfo, v6PodNetInfo, v4Pod2NetInfo}, }), - ginkgotable.Entry("add references to IPv4 & IPv6 (dual) LRP with no references", lrpSync{ + ginkgo.Entry("add references to IPv4 & IPv6 (dual) LRP with no references", lrpSync{ initialLRPs: []*nbdb.LogicalRouterPolicy{ getReRouteLRP(podNamespace, podName, v4PodIPStr, 0, v4IPFamilyValue, v4PodNextHops, map[string]string{"name": egressIPName}, defaultNetworkControllerName), getReRouteLRP(podNamespace, podName, v6PodIPStr, 0, v6IPFamilyValue, v6PodNextHops, map[string]string{"name": egressIPName}, defaultNetworkControllerName)}, @@ -107,7 +106,7 @@ var _ = ginkgo.Describe("OVN Logical Router Syncer", func() { v6JoinSubnet: v6JoinSubnet, pods: podsNetInfo{v4PodNetInfo, v6PodNetInfo, v4Pod2NetInfo}, }), - ginkgotable.Entry("does not modify IPv4 LRP which contains a reference", lrpSync{ + ginkgo.Entry("does not modify IPv4 LRP which contains a reference", lrpSync{ initialLRPs: []*nbdb.LogicalRouterPolicy{getReRouteLRP(podNamespace, podName, v4PodIPStr, 0, v4IPFamilyValue, v4PodNextHops, getEgressIPLRPReRouteDbIDs(egressIPName, podNamespace, podName, v4IPFamilyValue, defaultNetworkName, defaultNetworkControllerName).GetExternalIDs(), defaultNetworkControllerName)}, @@ -118,7 +117,7 @@ var _ = ginkgo.Describe("OVN Logical Router Syncer", func() { v4JoinSubnet: v4JoinSubnet, pods: podsNetInfo{v4PodNetInfo}, }), - ginkgotable.Entry("does not return error when unable to build a reference because of failed pod IP lookup", lrpSync{ + ginkgo.Entry("does not return error when unable to build a reference because of failed pod IP lookup", lrpSync{ initialLRPs: []*nbdb.LogicalRouterPolicy{getReRouteLRP(podNamespace, podName, v4PodIPStr, 0, v4IPFamilyValue, v4PodNextHops, map[string]string{"name": egressIPName}, defaultNetworkControllerName)}, @@ -131,12 +130,12 @@ var _ = ginkgo.Describe("OVN Logical Router Syncer", func() { }), ) - ginkgotable.DescribeTable("pod to join, pod to pod, pod to node aka 'no reroutes'", func(sync lrpSync) { + ginkgo.DescribeTable("pod to join, pod to pod, pod to node aka 'no reroutes'", func(sync lrpSync) { // pod to join LRPs may not have any owner references specified performTest(defaultNetworkControllerName, sync.initialLRPs, sync.finalLRPs, sync.v4ClusterSubnets, sync.v6ClusterSubnets, sync.v4JoinSubnet, sync.v6JoinSubnet, nil) }, - ginkgotable.Entry("does not modify LRP with owner references", lrpSync{ + ginkgo.Entry("does not modify LRP with owner references", lrpSync{ initialLRPs: []*nbdb.LogicalRouterPolicy{getNoReRouteLRP(fmt.Sprintf("ip4.src == %s && ip4.dst == %s", v4PodClusterSubnetStr, v4JoinSubnetStr), getEgressIPLRPNoReRoutePodToJoinDbIDs(v4IPFamilyValue, defaultNetworkName, "controller").GetExternalIDs(), nil), }, @@ -147,7 +146,7 @@ var _ = ginkgo.Describe("OVN Logical Router Syncer", func() { v4ClusterSubnets: []*net.IPNet{v4PodClusterSubnet}, v4JoinSubnet: v4JoinSubnet, }), - ginkgotable.Entry("updates IPv4 pod to pod, pod to join, pod to node with no references and does not modify LRP with references", lrpSync{ + ginkgo.Entry("updates IPv4 pod to pod, pod to join, pod to node with no references and does not modify LRP with references", lrpSync{ initialLRPs: []*nbdb.LogicalRouterPolicy{ { Priority: types.DefaultNoRereoutePriority, @@ -212,7 +211,7 @@ var _ = ginkgo.Describe("OVN Logical Router Syncer", func() { v6ClusterSubnets: []*net.IPNet{v6PodClusterSubnet}, v6JoinSubnet: v6JoinSubnet, }), - ginkgotable.Entry("updates IPv6 pod to pod, pod to join with no references and does not modify LRP with references", lrpSync{ + ginkgo.Entry("updates IPv6 pod to pod, pod to join with no references and does not modify LRP with references", lrpSync{ initialLRPs: []*nbdb.LogicalRouterPolicy{ { Priority: types.DefaultNoRereoutePriority, @@ -275,7 +274,7 @@ var _ = ginkgo.Describe("OVN Logical Router Syncer", func() { v6ClusterSubnets: []*net.IPNet{v6PodClusterSubnet}, v6JoinSubnet: v6JoinSubnet, }), - ginkgotable.Entry("updates IPv4 & IPv6 pod to pod, pod to join with no references and does not modify LRP with references", lrpSync{ + ginkgo.Entry("updates IPv4 & IPv6 pod to pod, pod to join with no references and does not modify LRP with references", lrpSync{ initialLRPs: []*nbdb.LogicalRouterPolicy{ { Priority: types.DefaultNoRereoutePriority, diff --git a/go-controller/pkg/ovn/external_ids_syncer/nat/nat_sync_test.go b/go-controller/pkg/ovn/external_ids_syncer/nat/nat_sync_test.go index c3b04d4c22..9e09ac9d2f 100644 --- a/go-controller/pkg/ovn/external_ids_syncer/nat/nat_sync_test.go +++ b/go-controller/pkg/ovn/external_ids_syncer/nat/nat_sync_test.go @@ -11,7 +11,6 @@ import ( libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - ginkgotable "github.com/onsi/ginkgo/extensions/table" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" ) @@ -52,9 +51,9 @@ var _ = ginkgo.Describe("NAT Syncer", func() { ginkgo.Context("EgressIP", func() { - ginkgotable.DescribeTable("egress NATs", func(sync natSync) { + ginkgo.DescribeTable("egress NATs", func(sync natSync) { performTest(defaultNetworkControllerName, sync.initialNATs, sync.finalNATs, sync.pods) - }, ginkgotable.Entry("converts legacy IPv4 NATs", natSync{ + }, ginkgo.Entry("converts legacy IPv4 NATs", natSync{ initialNATs: []*nbdb.NAT{getSNAT(nat1UUID, pod1V4CIDRStr, egressIP, legacyExtIDs)}, finalNATs: []*nbdb.NAT{getSNAT(nat1UUID, pod1V4CIDRStr, egressIP, pod1V4ExtIDs)}, pods: podsNetInfo{ @@ -70,7 +69,7 @@ var _ = ginkgo.Describe("NAT Syncer", func() { }, }, }), - ginkgotable.Entry("converts legacy IPv6 NATs", natSync{ + ginkgo.Entry("converts legacy IPv6 NATs", natSync{ initialNATs: []*nbdb.NAT{getSNAT(nat1UUID, pod1V6CIDRStr, egressIP, legacyExtIDs)}, finalNATs: []*nbdb.NAT{getSNAT(nat1UUID, pod1V6CIDRStr, egressIP, pod1V6ExtIDs)}, pods: podsNetInfo{ @@ -86,7 +85,7 @@ var _ = ginkgo.Describe("NAT Syncer", func() { }, }, }), - ginkgotable.Entry("converts legacy dual stack NATs", natSync{ + ginkgo.Entry("converts legacy dual stack NATs", natSync{ initialNATs: []*nbdb.NAT{getSNAT(nat1UUID, pod1V4CIDRStr, egressIP, legacyExtIDs), getSNAT(nat2UUID, pod1V6CIDRStr, egressIP, legacyExtIDs)}, finalNATs: []*nbdb.NAT{getSNAT(nat1UUID, pod1V4CIDRStr, egressIP, pod1V4ExtIDs), getSNAT(nat2UUID, pod1V6CIDRStr, egressIP, pod1V6ExtIDs)}, pods: podsNetInfo{ @@ -102,7 +101,7 @@ var _ = ginkgo.Describe("NAT Syncer", func() { }, }, }), - ginkgotable.Entry("doesn't alter NAT with correct external IDs", natSync{ + ginkgo.Entry("doesn't alter NAT with correct external IDs", natSync{ initialNATs: []*nbdb.NAT{getSNAT(nat1UUID, pod1V6CIDRStr, egressIP, pod1V6ExtIDs)}, finalNATs: []*nbdb.NAT{getSNAT(nat1UUID, pod1V6CIDRStr, egressIP, pod1V6ExtIDs)}, pods: podsNetInfo{ diff --git a/go-controller/pkg/ovn/gateway.go b/go-controller/pkg/ovn/gateway.go index 9e732f1c02..43a2fe0181 100644 --- a/go-controller/pkg/ovn/gateway.go +++ b/go-controller/pkg/ovn/gateway.go @@ -143,67 +143,66 @@ func WithLoadBalancerGroups(routerLBGroup, clusterLBGroup, switchLBGroup string) } } -// cleanupStalePodSNATs removes SNATs against nodeIP for the given node if the SNAT.logicalIP isn't an active podIP on this node -// We don't have to worry about missing SNATs that should be added because addLogicalPort takes care of this for all pods -// when RequestRetryObjs is called for each node add. -// This is executed only when disableSNATMultipleGateways = true -// SNATs configured for the join subnet are ignored. -// -// NOTE: On startup libovsdb adds back all the pods and this should normally update all existing SNATs -// accordingly. Due to a stale egressIP cache bug https://issues.redhat.com/browse/OCPBUGS-1520 we ended up adding -// wrong pod->nodeSNATs which won't get cleared up unless explicitly deleted. +// cleanupStalePodSNATs removes pod SNATs against nodeIP for the given node if +// the SNAT.logicalIP isn't an active podIP, the pod network is being advertised +// on this node or disableSNATMultipleGWs=false. We don't have to worry about +// missing SNATs that should be added because addLogicalPort takes care of this +// for all pods when RequestRetryObjs is called for each node add. +// Other non-pod SNATs like join subnet SNATs are ignored. +// NOTE: On startup libovsdb adds back all the pods and this should normally +// update all existing SNATs accordingly. Due to a stale egressIP cache bug +// https://issues.redhat.com/browse/OCPBUGS-1520 we ended up adding wrong +// pod->nodeSNATs which won't get cleared up unless explicitly deleted. // NOTE2: egressIP SNATs are synced in EIP controller. -// TODO (tssurya): Add support cleaning up even if disableSNATMultipleGWs=false, we'd need to remove the perPod -// SNATs in case someone switches between these modes. See https://github.com/ovn-org/ovn-kubernetes/issues/3232 func (gw *GatewayManager) cleanupStalePodSNATs(nodeName string, nodeIPs []*net.IPNet, gwLRPIPs []net.IP) error { - if !config.Gateway.DisableSNATMultipleGWs { - return nil - } - - pods, err := gw.kube.GetPods(metav1.NamespaceAll, metav1.ListOptions{ - FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String(), - }) - if err != nil { - return fmt.Errorf("unable to list existing pods on node: %s, %w", - nodeName, err) - } - - podIPsOnNode := sets.NewString() // collects all podIPs on node - for _, pod := range pods { - pod := *pod - if !util.PodScheduled(&pod) { //if the pod is not scheduled we should not remove the nat - continue + // collect all the pod IPs for which we should be doing the SNAT; if the pod + // network is advertised or DisableSNATMultipleGWs==false we consider all + // the SNATs stale + podIPsWithSNAT := sets.New[string]() + if !gw.isRoutingAdvertised(nodeName) && config.Gateway.DisableSNATMultipleGWs { + pods, err := gw.kube.GetPods(metav1.NamespaceAll, metav1.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String(), + }) + if err != nil { + return fmt.Errorf("unable to list existing pods on node: %s, %w", + nodeName, err) } - if util.PodCompleted(&pod) { - collidingPod, err := findPodWithIPAddresses(gw.watchFactory, gw.netInfo, []net.IP{utilnet.ParseIPSloppy(pod.Status.PodIP)}, "") //even if a pod is completed we should still delete the nat if the ip is not in use anymore - if err != nil { - return fmt.Errorf("lookup for pods with same ip as %s %s failed: %w", pod.Namespace, pod.Name, err) - } - if collidingPod != nil { //if the ip is in use we should not remove the nat + for _, pod := range pods { + pod := *pod + if !util.PodScheduled(&pod) { //if the pod is not scheduled we should not remove the nat continue } - } - podIPs, err := util.GetPodIPsOfNetwork(&pod, gw.netInfo) - if err != nil && errors.Is(err, util.ErrNoPodIPFound) { - // It is possible that the pod is scheduled during this time, but the LSP add or - // IP Allocation has not happened and it is waiting for the WatchPods to start - // after WatchNodes completes (This function is called during syncNodes). So since - // the pod doesn't have any IPs, there is no SNAT here to keep for this pod so we skip - // this pod from processing and move onto the next one. - klog.Warningf("Unable to fetch podIPs for pod %s/%s: %v", pod.Namespace, pod.Name, err) - continue // no-op - } else if err != nil { - return fmt.Errorf("unable to fetch podIPs for pod %s/%s: %w", pod.Namespace, pod.Name, err) - } - for _, podIP := range podIPs { - podIPsOnNode.Insert(podIP.String()) + if util.PodCompleted(&pod) { + collidingPod, err := findPodWithIPAddresses(gw.watchFactory, gw.netInfo, []net.IP{utilnet.ParseIPSloppy(pod.Status.PodIP)}, "") //even if a pod is completed we should still delete the nat if the ip is not in use anymore + if err != nil { + return fmt.Errorf("lookup for pods with same ip as %s %s failed: %w", pod.Namespace, pod.Name, err) + } + if collidingPod != nil { //if the ip is in use we should not remove the nat + continue + } + } + podIPs, err := util.GetPodIPsOfNetwork(&pod, gw.netInfo) + if err != nil && errors.Is(err, util.ErrNoPodIPFound) { + // It is possible that the pod is scheduled during this time, but the LSP add or + // IP Allocation has not happened and it is waiting for the WatchPods to start + // after WatchNodes completes (This function is called during syncNodes). So since + // the pod doesn't have any IPs, there is no SNAT here to keep for this pod so we skip + // this pod from processing and move onto the next one. + klog.Warningf("Unable to fetch podIPs for pod %s/%s: %v", pod.Namespace, pod.Name, err) + continue // no-op + } else if err != nil { + return fmt.Errorf("unable to fetch podIPs for pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + for _, podIP := range podIPs { + podIPsWithSNAT.Insert(podIP.String()) + } } } - gatewayRouter := nbdb.LogicalRouter{ + gatewayRouter := &nbdb.LogicalRouter{ Name: gw.gwRouterName, } - routerNats, err := libovsdbops.GetRouterNATs(gw.nbClient, &gatewayRouter) + routerNATs, err := libovsdbops.GetRouterNATs(gw.nbClient, gatewayRouter) if err != nil && errors.Is(err, libovsdbclient.ErrNotFound) { return fmt.Errorf("unable to get NAT entries for router %s on node %s: %w", gatewayRouter.Name, nodeName, err) } @@ -211,7 +210,7 @@ func (gw *GatewayManager) cleanupStalePodSNATs(nodeName string, nodeIPs []*net.I nodeIPset := sets.New(util.IPNetsIPToStringSlice(nodeIPs)...) gwLRPIPset := sets.New(util.StringSlice(gwLRPIPs)...) natsToDelete := []*nbdb.NAT{} - for _, routerNat := range routerNats { + for _, routerNat := range routerNATs { routerNat := routerNat if routerNat.Type != nbdb.NATTypeSNAT { continue @@ -219,7 +218,7 @@ func (gw *GatewayManager) cleanupStalePodSNATs(nodeName string, nodeIPs []*net.I if !nodeIPset.Has(routerNat.ExternalIP) { continue } - if podIPsOnNode.Has(routerNat.LogicalIP) { + if podIPsWithSNAT.Has(routerNat.LogicalIP) { continue } if gwLRPIPset.Has(routerNat.LogicalIP) { @@ -232,12 +231,14 @@ func (gw *GatewayManager) cleanupStalePodSNATs(nodeName string, nodeIPs []*net.I } natsToDelete = append(natsToDelete, routerNat) } + if len(natsToDelete) > 0 { - err := libovsdbops.DeleteNATs(gw.nbClient, &gatewayRouter, natsToDelete...) + err := libovsdbops.DeleteNATs(gw.nbClient, gatewayRouter, natsToDelete...) if err != nil { return fmt.Errorf("unable to delete NATs %+v from node %s: %w", natsToDelete, nodeName, err) } } + return nil } @@ -786,8 +787,7 @@ func (gw *GatewayManager) GatewayInit( nats := make([]*nbdb.NAT, 0, len(clusterIPSubnet)) var nat *nbdb.NAT - - if !config.Gateway.DisableSNATMultipleGWs { + if !config.Gateway.DisableSNATMultipleGWs && !gw.isRoutingAdvertised(nodeName) { // Default SNAT rules. DisableSNATMultipleGWs=false in LGW (traffic egresses via mp0) always. // We are not checking for gateway mode to be shared explicitly to reduce topology differences. for _, entry := range clusterIPSubnet { @@ -1264,6 +1264,13 @@ func (gw *GatewayManager) containsJoinIP(ip net.IP) bool { return util.IsContainedInAnyCIDR(ipNet, gw.netInfo.JoinSubnets()...) } +func (gw *GatewayManager) isRoutingAdvertised(node string) bool { + if gw.netInfo.IsSecondary() { + return false + } + return util.IsPodNetworkAdvertisedAtNode(gw.netInfo, node) +} + func (gw *GatewayManager) syncGatewayLogicalNetwork( node *kapi.Node, l3GatewayConfig *util.L3GatewayConfig, diff --git a/go-controller/pkg/ovn/gateway_test.go b/go-controller/pkg/ovn/gateway_test.go index ebc1453b1a..6bcd3ea52e 100644 --- a/go-controller/pkg/ovn/gateway_test.go +++ b/go-controller/pkg/ovn/gateway_test.go @@ -2006,7 +2006,7 @@ func newGatewayManager(ovn *FakeOVN, nodeName string) *GatewayManager { controller.defaultCOPPUUID, controller.kube, controller.nbClient, - controller.NetInfo, + controller.GetNetInfo(), ovn.watcher, WithLoadBalancerGroups( controller.routerLoadBalancerGroupUUID, diff --git a/go-controller/pkg/ovn/gress_policy.go b/go-controller/pkg/ovn/gress_policy.go index d4a8f3df6d..ee93ba8b72 100644 --- a/go-controller/pkg/ovn/gress_policy.go +++ b/go-controller/pkg/ovn/gress_policy.go @@ -56,7 +56,7 @@ type gressPolicy struct { ipv6Mode bool } -func newGressPolicy(policyType knet.PolicyType, idx int, namespace, name, controllerName string, isNetPolStateless bool, netInfo util.BasicNetInfo) *gressPolicy { +func newGressPolicy(policyType knet.PolicyType, idx int, namespace, name, controllerName string, isNetPolStateless bool, netInfo util.NetInfo) *gressPolicy { ipv4Mode, ipv6Mode := netInfo.IPMode() return &gressPolicy{ controllerName: controllerName, diff --git a/go-controller/pkg/ovn/hybrid_test.go b/go-controller/pkg/ovn/hybrid_test.go index 55f781637e..08828e4d6d 100644 --- a/go-controller/pkg/ovn/hybrid_test.go +++ b/go-controller/pkg/ovn/hybrid_test.go @@ -26,6 +26,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" @@ -211,13 +212,24 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { libovsdbOvnNBClient, libovsdbOvnSBClient, libovsdbCleanup, err = libovsdbtest.NewNBSBTestHarness(dbSetup) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - clusterController, err := NewOvnController(fakeClient.GetMasterClientset(), f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, - record.NewFakeRecorder(10), wg, nil, NewPortCache(stopChan)) + clusterController, err := NewOvnController( + fakeClient.GetMasterClientset(), + f, + stopChan, + nil, + networkmanager.Default().Interface(), + libovsdbOvnNBClient, + libovsdbOvnSBClient, + record.NewFakeRecorder(10), + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) c, cancel := context.WithCancel(ctx.Context) defer cancel() clusterManager, err := cm.NewClusterManager(fakeClient.GetClusterManagerClientset(), f, "identity", wg, nil) - gomega.Expect(clusterManager).NotTo(gomega.BeNil()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = clusterManager.Start(c) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer clusterManager.Stop() @@ -369,8 +381,19 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { expectedDatabaseState := []libovsdbtest.TestData{ovnClusterRouterLRP} expectedDatabaseState = addNodeLogicalFlows(expectedDatabaseState, expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterRouterPortGroup, expectedClusterPortGroup, &node1) - clusterController, err := NewOvnController(fakeClient.GetMasterClientset(), f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, - record.NewFakeRecorder(10), wg, nil, NewPortCache(stopChan)) + clusterController, err := NewOvnController( + fakeClient.GetMasterClientset(), + f, + stopChan, + nil, + networkmanager.Default().Interface(), + libovsdbOvnNBClient, + libovsdbOvnSBClient, + record.NewFakeRecorder(10), + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) setupCOPP := true @@ -666,8 +689,19 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { libovsdbOvnNBClient, libovsdbOvnSBClient, libovsdbCleanup, err = libovsdbtest.NewNBSBTestHarness(dbSetup) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - clusterController, err := NewOvnController(fakeClient.GetMasterClientset(), f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, - record.NewFakeRecorder(10), wg, nil, NewPortCache(stopChan)) + clusterController, err := NewOvnController( + fakeClient.GetMasterClientset(), + f, + stopChan, + nil, + networkmanager.Default().Interface(), + libovsdbOvnNBClient, + libovsdbOvnSBClient, + record.NewFakeRecorder(10), + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) setupCOPP := true @@ -838,8 +872,19 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { expectedDatabaseState := []libovsdbtest.TestData{ovnClusterRouterLRP} expectedDatabaseState = addNodeLogicalFlows(expectedDatabaseState, expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterRouterPortGroup, expectedClusterPortGroup, &node1) - clusterController, err := NewOvnController(fakeClient.GetMasterClientset(), f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, - record.NewFakeRecorder(10), wg, nil, NewPortCache(stopChan)) + clusterController, err := NewOvnController( + fakeClient.GetMasterClientset(), + f, + stopChan, + nil, + networkmanager.Default().Interface(), + libovsdbOvnNBClient, + libovsdbOvnSBClient, + record.NewFakeRecorder(10), + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) setupCOPP := true @@ -1122,8 +1167,19 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { libovsdbOvnNBClient, libovsdbOvnSBClient, libovsdbCleanup, err = libovsdbtest.NewNBSBTestHarness(dbSetup) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - clusterController, err := NewOvnController(fakeClient.GetMasterClientset(), f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, - record.NewFakeRecorder(10), wg, nil, NewPortCache(stopChan)) + clusterController, err := NewOvnController( + fakeClient.GetMasterClientset(), + f, + stopChan, + nil, + networkmanager.Default().Interface(), + libovsdbOvnNBClient, + libovsdbOvnSBClient, + record.NewFakeRecorder(10), + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) setupCOPP := true @@ -1324,8 +1380,19 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { libovsdbOvnNBClient, libovsdbOvnSBClient, libovsdbCleanup, err = libovsdbtest.NewNBSBTestHarness(dbSetup) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - clusterController, err := NewOvnController(fakeClient.GetMasterClientset(), f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, - record.NewFakeRecorder(10), wg, nil, NewPortCache(stopChan)) + clusterController, err := NewOvnController( + fakeClient.GetMasterClientset(), + f, + stopChan, + nil, + networkmanager.Default().Interface(), + libovsdbOvnNBClient, + libovsdbOvnSBClient, + record.NewFakeRecorder(10), + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) setupCOPP := true @@ -1518,8 +1585,19 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { expectedDatabaseState := []libovsdbtest.TestData{ovnClusterRouterLRP} expectedDatabaseState = addNodeLogicalFlows(expectedDatabaseState, expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterRouterPortGroup, expectedClusterPortGroup, &node1) - clusterController, err := NewOvnController(fakeClient, f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, - record.NewFakeRecorder(10), wg, nil, NewPortCache(stopChan)) + clusterController, err := NewOvnController( + fakeClient, + f, + stopChan, + nil, + networkmanager.Default().Interface(), + libovsdbOvnNBClient, + libovsdbOvnSBClient, + record.NewFakeRecorder(10), + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) setupCOPP := true @@ -1740,8 +1818,19 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { libovsdbOvnNBClient, libovsdbOvnSBClient, libovsdbCleanup, err = libovsdbtest.NewNBSBTestHarness(dbSetup) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - clusterController, err := NewOvnController(fakeClient, f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, - record.NewFakeRecorder(10), wg, nil, NewPortCache(stopChan)) + clusterController, err := NewOvnController( + fakeClient, + f, + stopChan, + nil, + networkmanager.Default().Interface(), + libovsdbOvnNBClient, + libovsdbOvnSBClient, + record.NewFakeRecorder(10), + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) setupCOPP := true diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index e346bee98d..9d5f81d8c0 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -56,20 +56,20 @@ func (oc *DefaultNetworkController) SetupMaster(existingNodeNames []string) erro } // Create OVNJoinSwitch that will be used to connect gateway routers to the distributed router. - return oc.gatewayTopologyFactory.NewJoinSwitch(logicalRouter, oc.NetInfo, oc.ovnClusterLRPToJoinIfAddrs) + return oc.gatewayTopologyFactory.NewJoinSwitch(logicalRouter, oc.GetNetInfo(), oc.ovnClusterLRPToJoinIfAddrs) } func (oc *DefaultNetworkController) newClusterRouter() (*nbdb.LogicalRouter, error) { if oc.multicastSupport { return oc.gatewayTopologyFactory.NewClusterRouterWithMulticastSupport( oc.GetNetworkScopedClusterRouterName(), - oc.NetInfo, + oc.GetNetInfo(), oc.defaultCOPPUUID, ) } return oc.gatewayTopologyFactory.NewClusterRouter( oc.GetNetworkScopedClusterRouterName(), - oc.NetInfo, + oc.GetNetInfo(), oc.defaultCOPPUUID, ) } @@ -849,7 +849,7 @@ func (oc *DefaultNetworkController) newGatewayManager(nodeName string) *GatewayM oc.defaultCOPPUUID, oc.kube, oc.nbClient, - oc.NetInfo, + oc.GetNetInfo(), oc.watchFactory, oc.gatewayOptions()..., ) diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go index c87d4173c5..2d36a42beb 100644 --- a/go-controller/pkg/ovn/master_test.go +++ b/go-controller/pkg/ovn/master_test.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net" - "reflect" "strconv" "sync" "sync/atomic" @@ -26,6 +25,7 @@ import ( libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/sbdb" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" @@ -941,10 +941,11 @@ var _ = ginkgo.Describe("Default network controller operations", func() { ) const ( - clusterIPNet string = "10.1.0.0" - clusterCIDR string = clusterIPNet + "/16" - clusterv6CIDR string = "aef0::/48" - vlanID = 1024 + clusterIPNet string = "10.1.0.0" + clusterCIDR string = clusterIPNet + "/16" + clusterv6CIDR string = "aef0::/48" + vlanID = 1024 + Node1GatewayRouterIP = "172.16.16.2" ) ginkgo.BeforeEach(func() { @@ -972,8 +973,8 @@ var _ = ginkgo.Describe("Default network controller operations", func() { SystemID: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", NodeSubnet: "10.1.1.0/24", GWRouter: types.GWRouterPrefix + "node1", - GatewayRouterIPMask: "172.16.16.2/24", - GatewayRouterIP: "172.16.16.2", + GatewayRouterIPMask: Node1GatewayRouterIP + "/24", + GatewayRouterIP: Node1GatewayRouterIP, GatewayRouterNextHop: "172.16.16.1", PhysicalBridgeName: "br-eth0", NodeGWIP: "10.1.1.1/24", @@ -1051,7 +1052,19 @@ var _ = ginkgo.Describe("Default network controller operations", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) recorder = record.NewFakeRecorder(10) - oc, err = NewOvnController(fakeClient, f, stopChan, nil, nbClient, sbClient, recorder, wg, nil, NewPortCache(stopChan)) + oc, err = NewOvnController( + fakeClient, + f, + stopChan, + nil, + networkmanager.Default().Interface(), + nbClient, + sbClient, + recorder, + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(oc).NotTo(gomega.BeNil()) @@ -1205,81 +1218,114 @@ var _ = ginkgo.Describe("Default network controller operations", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) - ginkgo.It("clear stale pod SNATs from syncGateway", func() { - - app.Action = func(ctx *cli.Context) error { - - _, err := config.InitConfig(ctx, nil, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - config.Gateway.DisableSNATMultipleGWs = true - - // create a pod on this node - ns := newNamespace("namespace-1") - pod := *newPodWithLabels(ns.Name, podName, node1.Name, "10.0.0.3", egressPodLabel) - _, err = fakeClient.KubeClient.CoreV1().Pods(ns.Name).Create(context.TODO(), &pod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Let the real code run and ensure OVN database sync - gomega.Expect(oc.WatchNodes()).To(gomega.Succeed()) - - var clusterSubnets []*net.IPNet - for _, clusterSubnet := range config.Default.ClusterSubnets { - clusterSubnets = append(clusterSubnets, clusterSubnet.CIDR) - } - externalIP, _, err := net.ParseCIDR(l3GatewayConfig.IPAddresses[0].String()) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - skipSnat := config.Gateway.DisableSNATMultipleGWs - subnet := ovntest.MustParseIPNet(node1.NodeSubnet) - - expectedNBDatabaseState = generateGatewayInitExpectedNB(expectedNBDatabaseState, expectedOVNClusterRouter, - expectedNodeSwitch, node1.Name, clusterSubnets, []*net.IPNet{subnet}, l3GatewayConfig, - []*net.IPNet{classBIPAddress(node1.LrpIP)}, []*net.IPNet{classBIPAddress(node1.DrLrpIP)}, - skipSnat, node1.NodeMgmtPortIP, "1400") + extraNats := []*nbdb.NAT{ + newNodeSNAT("stale-nodeNAT-UUID-1", "10.1.0.3", Node1GatewayRouterIP), + newNodeSNAT("stale-nodeNAT-UUID-2", "10.2.0.3", Node1GatewayRouterIP), + newNodeSNAT("stale-nodeNAT-UUID-3", "10.0.0.3", Node1GatewayRouterIP), + newNodeSNAT("stale-nodeNAT-UUID-4", "10.0.0.3", "172.16.16.3"), + } + ginkgo.DescribeTable( + "reconciles pod network SNATs from syncGateway", + func(condition func(*DefaultNetworkController), expectedExtraNATs ...*nbdb.NAT) { + app.Action = func(ctx *cli.Context) error { + _, err := config.InitConfig(ctx, nil, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // create a pod on this node + ns := newNamespace("namespace-1") + pod := *newPodWithLabels(ns.Name, podName, node1.Name, "10.0.0.3", egressPodLabel) + _, err = fakeClient.KubeClient.CoreV1().Pods(ns.Name).Create(context.TODO(), &pod, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Let the real code run and ensure OVN database sync + gomega.Expect(oc.WatchNodes()).To(gomega.Succeed()) + + // add stale SNATs from pods to nodes on wrong node + GR := &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node1.Name, + } + err = libovsdbops.CreateOrUpdateNATs(nbClient, GR, extraNats...) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // generate specific test conditions + condition(oc) + + // ensure the stale SNAT's are cleaned up + gomega.Expect(oc.StartServiceController(wg, false)).To(gomega.Succeed()) + subnet := ovntest.MustParseIPNet(node1.NodeSubnet) + err = oc.syncDefaultGatewayLogicalNetwork(&testNode, l3GatewayConfig, []*net.IPNet{subnet}, []string{node1.NodeIP}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + skipSnat := config.Gateway.DisableSNATMultipleGWs || oc.isPodNetworkAdvertisedAtNode(node1.Name) + var clusterSubnets []*net.IPNet + for _, clusterSubnet := range config.Default.ClusterSubnets { + clusterSubnets = append(clusterSubnets, clusterSubnet.CIDR) + } + expectedNBDatabaseState = generateGatewayInitExpectedNB(expectedNBDatabaseState, expectedOVNClusterRouter, + expectedNodeSwitch, node1.Name, clusterSubnets, []*net.IPNet{subnet}, l3GatewayConfig, + []*net.IPNet{classBIPAddress(node1.LrpIP)}, []*net.IPNet{classBIPAddress(node1.DrLrpIP)}, + skipSnat, node1.NodeMgmtPortIP, "1400") + + GR = nil + for _, testObj := range expectedNBDatabaseState { + if router, ok := testObj.(*nbdb.LogicalRouter); ok && router.UUID == types.GWRouterPrefix+node1.Name+"-UUID" { + GR = router + break + } + } + for _, expectedNat := range expectedExtraNATs { + expectedNBDatabaseState = append(expectedNBDatabaseState, expectedNat) + GR.Nat = append(GR.Nat, expectedNat.UUID) + } + gomega.Eventually(oc.nbClient).Should(libovsdbtest.HaveData(expectedNBDatabaseState)) - // add stale SNATs from pods to nodes on wrong node - staleNats := []*nbdb.NAT{ - newNodeSNAT("stale-nodeNAT-UUID-1", "10.1.0.3", externalIP.String()), - newNodeSNAT("stale-nodeNAT-UUID-2", "10.2.0.3", externalIP.String()), - newNodeSNAT("stale-nodeNAT-UUID-3", "10.0.0.3", externalIP.String()), - newNodeSNAT("stale-nodeNAT-UUID-4", "10.0.0.3", "172.16.16.3"), + return nil } - GR := &nbdb.LogicalRouter{ - Name: types.GWRouterPrefix + node1.Name, - } - err = libovsdbops.CreateOrUpdateNATs(nbClient, GR, staleNats...) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // ensure the stale SNAT's are cleaned up - gomega.Expect(oc.StartServiceController(wg, false)).To(gomega.Succeed()) - err = oc.syncDefaultGatewayLogicalNetwork(&testNode, l3GatewayConfig, []*net.IPNet{subnet}, []string{node1.NodeIP}) + err := app.Run([]string{ + app.Name, + "-cluster-subnets=" + clusterCIDR, + "--init-gateways", + "--nodeport", + }) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - expectedNBDatabaseState = append(expectedNBDatabaseState, - newNodeSNAT("stale-nodeNAT-UUID-3", "10.0.0.3", externalIP.String()), // won't be deleted since pod exists on this node - newNodeSNAT("stale-nodeNAT-UUID-4", "10.0.0.3", "172.16.16.3")) // won't be deleted on this node but will be deleted on the node whose IP is 172.16.16.3 since this pod belongs to this node - newNodeSNAT("nat-join-0-UUID", node1.LrpIP, externalIP.String()) // join subnet SNAT won't be affected by sync - for _, testObj := range expectedNBDatabaseState { - uuid := reflect.ValueOf(testObj).Elem().FieldByName("UUID").Interface().(string) - if uuid == types.GWRouterPrefix+node1.Name+"-UUID" { - GR := testObj.(*nbdb.LogicalRouter) - GR.Nat = []string{"stale-nodeNAT-UUID-3", "stale-nodeNAT-UUID-4", "nat-join-0-UUID"} - *testObj.(*nbdb.LogicalRouter) = *GR - break - } - } - gomega.Eventually(oc.nbClient).Should(libovsdbtest.HaveData(expectedNBDatabaseState)) - - return nil - } - - err := app.Run([]string{ - app.Name, - "-cluster-subnets=" + clusterCIDR, - "--init-gateways", - "--nodeport", - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) + }, + ginkgo.Entry( + "When DisableSNATMultipleGWs is true", + func(*DefaultNetworkController) { + config.Gateway.DisableSNATMultipleGWs = true + }, + newNodeSNAT("stale-nodeNAT-UUID-3", "10.0.0.3", Node1GatewayRouterIP), // won't be deleted since pod exists on this node + newNodeSNAT("stale-nodeNAT-UUID-4", "10.0.0.3", "172.16.16.3"), // won't be deleted on this node but will be deleted on the node whose IP is 172.16.16.3 since this pod belongs to this node + ), + ginkgo.Entry( + "When DisableSNATMultipleGWs is false", + func(*DefaultNetworkController) { + config.Gateway.DisableSNATMultipleGWs = false + }, + newNodeSNAT("stale-nodeNAT-UUID-4", "10.0.0.3", "172.16.16.3"), // won't be deleted on this node but will be deleted on the node whose IP is 172.16.16.3 since this pod belongs to this node + ), + ginkgo.Entry( + "When pod network is advertised and DisableSNATMultipleGWs is true", + func(oc *DefaultNetworkController) { + config.Gateway.DisableSNATMultipleGWs = true + mutableNetInfo := util.NewMutableNetInfo(oc.GetNetInfo()) + mutableNetInfo.SetPodNetworkAdvertisedVRFs(map[string][]string{"node1": {"vrf"}}) + oc.Reconcile(mutableNetInfo) + }, + newNodeSNAT("stale-nodeNAT-UUID-4", "10.0.0.3", "172.16.16.3"), // won't be deleted on this node but will be deleted on the node whose IP is 172.16.16.3 since this pod belongs to this node + ), + ginkgo.Entry( + "When pod network is advertised and DisableSNATMultipleGWs is false", + func(oc *DefaultNetworkController) { + config.Gateway.DisableSNATMultipleGWs = false + mutableNetInfo := util.NewMutableNetInfo(oc.GetNetInfo()) + mutableNetInfo.SetPodNetworkAdvertisedVRFs(map[string][]string{"node1": {"vrf"}}) + oc.Reconcile(mutableNetInfo) + }, + newNodeSNAT("stale-nodeNAT-UUID-4", "10.0.0.3", "172.16.16.3"), // won't be deleted on this node but will be deleted on the node whose IP is 172.16.16.3 since this pod belongs to this node + ), + ) ginkgo.It("does not list node's pods when updating node after successfully adding the node", func() { app.Action = func(ctx *cli.Context) error { @@ -2058,10 +2104,14 @@ func TestController_syncNodes(t *testing.T) { f, stopChan, nil, + networkmanager.Default().Interface(), nbClient, sbClient, record.NewFakeRecorder(0), - wg, nil, NewPortCache(stopChan)) + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).ToNot(gomega.HaveOccurred()) err = controller.syncNodes([]interface{}{&testNode}) if err != nil { @@ -2159,10 +2209,14 @@ func TestController_deleteStaleNodeChassis(t *testing.T) { f, stopChan, nil, + networkmanager.Default().Interface(), nbClient, sbClient, record.NewFakeRecorder(0), - wg, nil, NewPortCache(stopChan)) + wg, + nil, + NewPortCache(stopChan), + ) gomega.Expect(err).ToNot(gomega.HaveOccurred()) err = controller.deleteStaleNodeChassis(&tt.node) diff --git a/go-controller/pkg/ovn/multicast_test.go b/go-controller/pkg/ovn/multicast_test.go index 5ba004d581..16b667ef78 100644 --- a/go-controller/pkg/ovn/multicast_test.go +++ b/go-controller/pkg/ovn/multicast_test.go @@ -698,7 +698,8 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { tPod.populateControllerLogicalSwitchCache(bnc) } if nad != nil { - Expect(fakeOvn.controller.nadController.Start()).To(Succeed()) + Expect(fakeOvn.networkManager.Start()).To(Succeed()) + defer fakeOvn.networkManager.Stop() } Expect(bnc.WatchNamespaces()).To(Succeed()) @@ -715,7 +716,7 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { ports = append(ports, tPod.portUUID) } expectedData := getMulticastPolicyExpectedData(netInfo, namespace1.Name, ports) - expectedData = append(expectedData, getExpectedPodsAndSwitches(bnc.NetInfo, tPods, []string{nodeName})...) + expectedData = append(expectedData, getExpectedPodsAndSwitches(bnc.GetNetInfo(), tPods, []string{nodeName})...) Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) asf.ExpectAddressSetWithAddresses(namespace1.Name, tPodIPs) return nil @@ -765,7 +766,8 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { bnc, _ := startBaseNetworkController(fakeOvn, nad) if nad != nil { - Expect(fakeOvn.controller.nadController.Start()).To(Succeed()) + Expect(fakeOvn.networkManager.Start()).To(Succeed()) + defer fakeOvn.networkManager.Stop() } Expect(bnc.WatchNamespaces()).To(Succeed()) @@ -789,7 +791,7 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { acl = expectedData[3].(*nbdb.ACL) Expect(acl.Name).To(BeNil()) Expect(acl.ExternalIDs[libovsdbops.ObjectNameKey.String()]).To(Equal(longNameSpace2Name)) - expectedData = append(expectedData, getExpectedPodsAndSwitches(bnc.NetInfo, []testPod{}, []string{node.Name})...) + expectedData = append(expectedData, getExpectedPodsAndSwitches(bnc.GetNetInfo(), []testPod{}, []string{node.Name})...) // Enable multicast in the namespace. updateMulticast(fakeOvn, ns1, true) updateMulticast(fakeOvn, ns2, true) @@ -847,7 +849,8 @@ var _ = Describe("OVN Multicast with IP Address Family", func() { tPod.populateControllerLogicalSwitchCache(bnc) } if nad != nil { - Expect(fakeOvn.controller.nadController.Start()).To(Succeed()) + Expect(fakeOvn.networkManager.Start()).To(Succeed()) + defer fakeOvn.networkManager.Stop() } Expect(bnc.WatchNamespaces()).To(Succeed()) diff --git a/go-controller/pkg/ovn/namespace.go b/go-controller/pkg/ovn/namespace.go index 54a4408e91..64b34dbce3 100644 --- a/go-controller/pkg/ovn/namespace.go +++ b/go-controller/pkg/ovn/namespace.go @@ -164,7 +164,7 @@ func (oc *DefaultNetworkController) updateNamespace(old, newer *kapi.Namespace) if util.PodWantsHostNetwork(pod) { continue } - podIPs, err := util.GetPodIPsOfNetwork(pod, oc.NetInfo) + podIPs, err := util.GetPodIPsOfNetwork(pod, oc.GetNetInfo()) if err != nil { errors = append(errors, fmt.Errorf("unable to get pod %q IPs for SNAT rule removal err (%v)", logicalPort, err)) } diff --git a/go-controller/pkg/ovn/namespace_test.go b/go-controller/pkg/ovn/namespace_test.go index 31e380159b..ed64dca633 100644 --- a/go-controller/pkg/ovn/namespace_test.go +++ b/go-controller/pkg/ovn/namespace_test.go @@ -79,6 +79,10 @@ func buildNamespaceAddressSets(namespace string, ips []string) (*nbdb.AddressSet return addressset.GetTestDbAddrSets(getNamespaceAddrSetDbIDs(namespace, "default-network-controller"), ips) } +func buildUDNNamespaceAddressSets(networkName, namespace string, ips []string) (*nbdb.AddressSet, *nbdb.AddressSet) { + return addressset.GetTestDbAddrSets(getNamespaceAddrSetDbIDs(namespace, networkName+"-network-controller"), ips) +} + var _ = ginkgo.Describe("OVN Namespace Operations", func() { const ( namespaceName = "namespace1" diff --git a/go-controller/pkg/ovn/ovn.go b/go-controller/pkg/ovn/ovn.go index 04fbbebcda..c2920507c7 100644 --- a/go-controller/pkg/ovn/ovn.go +++ b/go-controller/pkg/ovn/ovn.go @@ -204,7 +204,7 @@ func (oc *DefaultNetworkController) ensureLocalZonePod(oldPod, pod *kapi.Pod, ad func (oc *DefaultNetworkController) ensureRemotePodIP(oldPod, pod *kapi.Pod, addPort bool) error { if (addPort || (oldPod != nil && len(pod.Status.PodIPs) != len(oldPod.Status.PodIPs))) && !util.PodWantsHostNetwork(pod) { - podIfAddrs, err := util.GetPodCIDRsWithFullMask(pod, oc.NetInfo) + podIfAddrs, err := util.GetPodCIDRsWithFullMask(pod, oc.GetNetInfo()) if err != nil { // not finding pod IPs on a remote pod is common until the other node wires the pod, suppress it return fmt.Errorf("failed to obtain IPs to add remote pod %s/%s: %w", @@ -324,7 +324,7 @@ func (oc *DefaultNetworkController) removeRemoteZonePod(pod *kapi.Pod) error { } if kubevirt.IsPodLiveMigratable(pod) { - ips, err := util.GetPodCIDRsWithFullMask(pod, oc.NetInfo) + ips, err := util.GetPodCIDRsWithFullMask(pod, oc.GetNetInfo()) if err != nil && !errors.Is(err, util.ErrNoPodIPFound) { return fmt.Errorf("failed to get pod ips for the pod %s/%s: %w", pod.Namespace, pod.Name, err) } @@ -487,14 +487,13 @@ func (oc *DefaultNetworkController) InitEgressServiceZoneController() (*egresssv // If the EgressIP controller is enabled it will take care of creating the // "no reroute" policies - we can pass "noop" functions to the egress service controller. initClusterEgressPolicies := func(nbClient libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, ni util.NetInfo, - clusterSubnets []*net.IPNet, controllerName string) error { + clusterSubnets []*net.IPNet, controllerName, routerName string) error { return nil } ensureNodeNoReroutePolicies := func(nbClient libovsdbclient.Client, addressSetFactory addressset.AddressSetFactory, network, router, controller string, nodeLister listers.NodeLister, v4, v6 bool) error { return nil } - deleteLegacyDefaultNoRerouteNodePolicies := func(libovsdbclient.Client, string, string) error { return nil } // used only when IC=true createDefaultNodeRouteToExternal := func(nbClient libovsdbclient.Client, clusterRouter, gwRouterName string, clusterSubnets []config.CIDRNetworkEntry) error { return nil @@ -503,12 +502,11 @@ func (oc *DefaultNetworkController) InitEgressServiceZoneController() (*egresssv if !config.OVNKubernetesFeature.EnableEgressIP { initClusterEgressPolicies = InitClusterEgressPolicies ensureNodeNoReroutePolicies = ensureDefaultNoRerouteNodePolicies - deleteLegacyDefaultNoRerouteNodePolicies = DeleteLegacyDefaultNoRerouteNodePolicies createDefaultNodeRouteToExternal = libovsdbutil.CreateDefaultRouteToExternal } - return egresssvc_zone.NewController(oc.NetInfo, DefaultNetworkControllerName, oc.client, oc.nbClient, oc.addressSetFactory, - initClusterEgressPolicies, ensureNodeNoReroutePolicies, deleteLegacyDefaultNoRerouteNodePolicies, + return egresssvc_zone.NewController(oc.GetNetInfo(), DefaultNetworkControllerName, oc.client, oc.nbClient, oc.addressSetFactory, + initClusterEgressPolicies, ensureNodeNoReroutePolicies, createDefaultNodeRouteToExternal, oc.stopChan, oc.watchFactory.EgressServiceInformer(), oc.watchFactory.ServiceCoreInformer(), oc.watchFactory.EndpointSliceCoreInformer(), @@ -531,6 +529,7 @@ func (oc *DefaultNetworkController) newANPController() error { oc.zone, oc.recorder, oc.observManager, + oc.networkManager, ) return err } diff --git a/go-controller/pkg/ovn/ovn_test.go b/go-controller/pkg/ovn/ovn_test.go index 1ad919d950..017a8ecfef 100644 --- a/go-controller/pkg/ovn/ovn_test.go +++ b/go-controller/pkg/ovn/ovn_test.go @@ -25,15 +25,14 @@ import ( egressfirewall "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1" egressfirewallfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/clientset/versioned/fake" egressip "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1" - egressipv1 "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1" egressipfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/clientset/versioned/fake" egressqos "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1" egressqosfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/clientset/versioned/fake" egressservice "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1" egressservicefake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/clientset/versioned/fake" udnclientfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/clientset/versioned/fake" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" - fakenad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + testnm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/networkmanager" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -74,23 +73,24 @@ type secondaryControllerInfo struct { } type FakeOVN struct { - fakeClient *util.OVNMasterClientset - watcher *factory.WatchFactory - controller *DefaultNetworkController - stopChan chan struct{} - wg *sync.WaitGroup - asf *addressset.FakeAddressSetFactory - fakeRecorder *record.FakeRecorder - nbClient libovsdbclient.Client - sbClient libovsdbclient.Client - dbSetup libovsdbtest.TestSetup - nbsbCleanup *libovsdbtest.Context - egressQoSWg *sync.WaitGroup - egressSVCWg *sync.WaitGroup - anpWg *sync.WaitGroup - nadController *nad.NetAttachDefinitionController - eIPController *EgressIPController - portCache *PortCache + fakeClient *util.OVNMasterClientset + watcher *factory.WatchFactory + controller *DefaultNetworkController + stopChan chan struct{} + wg *sync.WaitGroup + asf *addressset.FakeAddressSetFactory + fakeRecorder *record.FakeRecorder + nbClient libovsdbclient.Client + sbClient libovsdbclient.Client + dbSetup libovsdbtest.TestSetup + nbsbCleanup *libovsdbtest.Context + egressQoSWg *sync.WaitGroup + egressSVCWg *sync.WaitGroup + anpWg *sync.WaitGroup + networkManager networkmanager.Controller + eIPController *EgressIPController + portCache *PortCache + // information map of all secondary network controllers secondaryControllers map[string]secondaryControllerInfo } @@ -206,28 +206,53 @@ func (o *FakeOVN) init(nadList []nettypes.NetworkAttachmentDefinition) { o.nbClient, o.sbClient, o.nbsbCleanup, err = libovsdbtest.NewNBSBTestHarness(o.dbSetup) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + o.stopChan = make(chan struct{}) + o.wg = &sync.WaitGroup{} + + o.networkManager = networkmanager.Default() + if config.OVNKubernetesFeature.EnableMultiNetwork { + o.networkManager, err = networkmanager.NewForZone("test", &testnm.FakeControllerManager{}, o.watcher) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + o.portCache = NewPortCache(o.stopChan) kubeOVN := &kube.KubeOVN{ Kube: kube.Kube{KClient: o.fakeClient.KubeClient}, EIPClient: o.fakeClient.EgressIPClient, } - o.eIPController = NewEIPController(o.nbClient, kubeOVN, o.watcher, - o.fakeRecorder, o.portCache, o.nadController, o.asf, config.IPv4Mode, config.IPv6Mode, "", DefaultNetworkControllerName) + o.eIPController = NewEIPController( + o.nbClient, + kubeOVN, + o.watcher, + o.fakeRecorder, + o.portCache, + o.networkManager.Interface(), + o.asf, + config.IPv4Mode, + config.IPv6Mode, + "", + DefaultNetworkControllerName, + ) if o.asf == nil { o.eIPController.addressSetFactory = addressset.NewOvnAddressSetFactory(o.nbClient, config.IPv4Mode, config.IPv6Mode) } - o.stopChan = make(chan struct{}) - o.wg = &sync.WaitGroup{} - o.controller, err = NewOvnController(o.fakeClient, o.watcher, - o.stopChan, o.asf, - o.nbClient, o.sbClient, - o.fakeRecorder, o.wg, - o.eIPController, o.portCache) + + o.controller, err = NewOvnController(o.fakeClient, + o.watcher, + o.stopChan, + o.asf, + o.networkManager.Interface(), + o.nbClient, + o.sbClient, + o.fakeRecorder, + o.wg, + o.eIPController, + o.portCache, + ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) o.controller.multicastSupport = config.EnableMulticast - o.nadController = o.controller.nadController.(*nad.NetAttachDefinitionController) - o.eIPController.nadController = o.controller.nadController.(*nad.NetAttachDefinitionController) o.eIPController.zone = o.controller.zone + setupCOPP := false setupClusterController(o.controller, setupCOPP) for _, n := range nadList { @@ -238,6 +263,9 @@ func (o *FakeOVN) init(nadList []nettypes.NetworkAttachmentDefinition) { err = o.watcher.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = o.eIPController.SyncLocalNodeZonesCache() + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "syncing Nodes OVN zones status must succeed to support EgressIP") + existingNodes, err := o.controller.kube.GetNodes() if err == nil { for _, node := range existingNodes { @@ -295,10 +323,19 @@ func resetNBClient(ctx context.Context, nbClient libovsdbclient.Client) { // NewOvnController creates a new OVN controller for creating logical network // infrastructure and policy -func NewOvnController(ovnClient *util.OVNMasterClientset, wf *factory.WatchFactory, stopChan chan struct{}, - addressSetFactory addressset.AddressSetFactory, libovsdbOvnNBClient libovsdbclient.Client, - libovsdbOvnSBClient libovsdbclient.Client, recorder record.EventRecorder, wg *sync.WaitGroup, - eIPController *EgressIPController, portCache *PortCache) (*DefaultNetworkController, error) { +func NewOvnController( + ovnClient *util.OVNMasterClientset, + wf *factory.WatchFactory, + stopChan chan struct{}, + addressSetFactory addressset.AddressSetFactory, + networkManager networkmanager.Interface, + libovsdbOvnNBClient libovsdbclient.Client, + libovsdbOvnSBClient libovsdbclient.Client, + recorder record.EventRecorder, + wg *sync.WaitGroup, + eIPController *EgressIPController, + portCache *PortCache, +) (*DefaultNetworkController, error) { fakeAddr, ok := addressSetFactory.(*addressset.FakeAddressSetFactory) if addressSetFactory == nil || (ok && fakeAddr == nil) { @@ -341,20 +378,13 @@ func NewOvnController(ovnClient *util.OVNMasterClientset, wf *factory.WatchFacto return nil, err } - var nadController *nad.NetAttachDefinitionController - if config.OVNKubernetesFeature.EnableMultiNetwork { - nadController, err = nad.NewNetAttachDefinitionController("test", &fakenad.FakeNetworkControllerManager{}, wf, nil) - if err != nil { - return nil, err - } - } - dnc, err := newDefaultNetworkControllerCommon(cnci, stopChan, wg, addressSetFactory, nadController, nil, portCache, eIPController) + dnc, err := newDefaultNetworkControllerCommon(cnci, stopChan, wg, addressSetFactory, networkManager, nil, eIPController, portCache) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if nbZoneFailed { // Delete the NBGlobal row as this function created it. Otherwise many tests would fail while // checking the expectedData in the NBDB. - err = deleteTestNBGlobal(libovsdbOvnNBClient, "global") + err = deleteTestNBGlobal(libovsdbOvnNBClient) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -386,7 +416,7 @@ func createTestNBGlobal(nbClient libovsdbclient.Client, zone string) error { return nil } -func deleteTestNBGlobal(nbClient libovsdbclient.Client, zone string) error { +func deleteTestNBGlobal(nbClient libovsdbclient.Client) error { p := func(nbGlobal *nbdb.NBGlobal) bool { return true } @@ -467,21 +497,21 @@ func (o *FakeOVN) NewSecondaryNetworkController(netattachdef *nettypes.NetworkAt switch topoType { case types.Layer3Topology: - l3Controller, err := NewSecondaryLayer3NetworkController(cnci, nInfo, o.nadController, o.eIPController, o.portCache) + l3Controller, err := NewSecondaryLayer3NetworkController(cnci, nInfo, o.networkManager.Interface(), o.eIPController, o.portCache) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if o.asf != nil { // use fake asf only when enabled l3Controller.addressSetFactory = asf } secondaryController = &l3Controller.BaseSecondaryNetworkController case types.Layer2Topology: - l2Controller, err := NewSecondaryLayer2NetworkController(cnci, nInfo, o.nadController) + l2Controller, err := NewSecondaryLayer2NetworkController(cnci, nInfo, o.networkManager.Interface(), o.eIPController, o.portCache) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if o.asf != nil { // use fake asf only when enabled l2Controller.addressSetFactory = asf } secondaryController = &l2Controller.BaseSecondaryNetworkController case types.LocalnetTopology: - localnetController := NewSecondaryLocalnetNetworkController(cnci, nInfo, o.nadController) + localnetController := NewSecondaryLocalnetNetworkController(cnci, nInfo, o.networkManager.Interface()) if o.asf != nil { // use fake asf only when enabled localnetController.addressSetFactory = asf } @@ -495,7 +525,7 @@ func (o *FakeOVN) NewSecondaryNetworkController(netattachdef *nettypes.NetworkAt if nbZoneFailed { // Delete the NBGlobal row as this function created it. Otherwise many tests would fail while // checking the expectedData in the NBDB. - err = deleteTestNBGlobal(o.nbClient, "global") + err = deleteTestNBGlobal(o.nbClient) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } } else { @@ -503,7 +533,9 @@ func (o *FakeOVN) NewSecondaryNetworkController(netattachdef *nettypes.NetworkAt } ginkgo.By(fmt.Sprintf("OVN test init: add NAD %s to secondary network controller of %s network %s", nadName, topoType, netName)) - secondaryController.AddNADs(nadName) + mutableNetInfo := util.NewMutableNetInfo(secondaryController.GetNetInfo()) + mutableNetInfo.AddNADs(nadName) + _ = util.ReconcileNetInfo(secondaryController.ReconcilableNetInfo, mutableNetInfo) return nil } @@ -511,7 +543,7 @@ func (o *FakeOVN) patchEgressIPObj(nodeName, egressIPName, egressIP, network str // NOTE: Cluster manager is the one who patches the egressIP object. // For the sake of unit testing egressip zone controller we need to patch egressIP object manually // There are tests in cluster-manager package covering the patch logic. - status := []egressipv1.EgressIPStatusItem{ + status := []egressip.EgressIPStatusItem{ { Node: nodeName, EgressIP: egressIP, diff --git a/go-controller/pkg/ovn/pod_selector_address_set.go b/go-controller/pkg/ovn/pod_selector_address_set.go index 802526b2b6..7ea2b2e5ab 100644 --- a/go-controller/pkg/ovn/pod_selector_address_set.go +++ b/go-controller/pkg/ovn/pod_selector_address_set.go @@ -177,7 +177,7 @@ func (psas *PodSelectorAddressSet) init(bnc *BaseNetworkController) error { podSelector: psas.podSelector, namespaceSelector: psas.namespaceSelector, namespace: psas.namespace, - netInfo: bnc.NetInfo, + netInfo: bnc.GetNetInfo(), ipv4Mode: ipv4Mode, ipv6Mode: ipv6Mode, stopChan: psas.cancelableContext.Done(), @@ -467,7 +467,7 @@ func (bnc *BaseNetworkController) podSelectorPodNeedsDelete(pod *kapi.Pod, podHa if !util.PodCompleted(pod) { return "", nil } - ips, err := util.GetPodIPsOfNetwork(pod, bnc.NetInfo) + ips, err := util.GetPodIPsOfNetwork(pod, bnc.GetNetInfo()) if err != nil { // if pod has no IP, nothing to do klog.Warningf("Failed to get IPs of pod %s/%s during address_set pod selector removal: %v", @@ -482,7 +482,7 @@ func (bnc *BaseNetworkController) podSelectorPodNeedsDelete(pod *kapi.Pod, podHa } // completed pod be deleted a long time ago, check if there is a new pod with that same ip - collidingPod, err := findPodWithIPAddresses(bnc.watchFactory, bnc.NetInfo, ips, nodeName) + collidingPod, err := findPodWithIPAddresses(bnc.watchFactory, bnc.GetNetInfo(), ips, nodeName) if err != nil { return "", fmt.Errorf("lookup for pods with the same IPs [%s] failed: %w", util.JoinIPs(ips, " "), err) } diff --git a/go-controller/pkg/ovn/pods.go b/go-controller/pkg/ovn/pods.go index 3f9da4bd8a..4802feb71c 100644 --- a/go-controller/pkg/ovn/pods.go +++ b/go-controller/pkg/ovn/pods.go @@ -226,7 +226,7 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *kapi.Pod) (err error) { return nil } - _, networkMap, err := util.GetPodNADToNetworkMapping(pod, oc.NetInfo) + _, networkMap, err := util.GetPodNADToNetworkMapping(pod, oc.GetNetInfo()) if err != nil { // multus won't add this Pod if this fails, should never happen return fmt.Errorf("error getting default-network's network-attachment for pod %s/%s: %v", pod.Namespace, pod.Name, err) @@ -235,7 +235,6 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *kapi.Pod) (err error) { if len(networkMap) > 1 { return fmt.Errorf("more than one NAD requested on default network for pod %s/%s", pod.Namespace, pod.Name) } - var network *nadapi.NetworkSelectionElement for _, network = range networkMap { break @@ -267,7 +266,8 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *kapi.Pod) (err error) { if networkRole != ovntypes.NetworkRolePrimary && util.IsNetworkSegmentationSupportEnabled() { pgName := libovsdbutil.GetPortGroupName(oc.getSecondaryPodsPortGroupDbIDs()) if ops, err = libovsdbops.AddPortsToPortGroupOps(oc.nbClient, ops, pgName, lsp.UUID); err != nil { - return err + return fmt.Errorf("unable to add ports to port group %s for %s/%s part of NAD %s: %w", + pgName, pod.Namespace, pod.Name, nadName, err) } // set open ports for UDN pods, use function without transact, since lsp is not created yet. var parseErr error @@ -309,7 +309,7 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *kapi.Pod) (err error) { if err != nil { return err } - } else if config.Gateway.DisableSNATMultipleGWs { + } else if config.Gateway.DisableSNATMultipleGWs && !oc.isPodNetworkAdvertisedAtNode(pod.Spec.NodeName) { // Add NAT rules to pods if disable SNAT is set and does not have // namespace annotations to go through external egress router if extIPs, err := getExternalIPsGR(oc.watchFactory, pod.Spec.NodeName); err != nil { @@ -332,7 +332,7 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *kapi.Pod) (err error) { return fmt.Errorf("error transacting operations %+v: %v", ops, err) } txOkCallBack() - oc.podRecorder.AddLSP(pod.UID, oc.NetInfo) + oc.podRecorder.AddLSP(pod.UID, oc.GetNetInfo()) // check if this pod is serving as an external GW err = oc.addPodExternalGW(pod) @@ -356,7 +356,7 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *kapi.Pod) (err error) { //observe the pod creation latency metric for newly created pods only if newlyCreatedPort { - metrics.RecordPodCreated(pod, oc.NetInfo) + metrics.RecordPodCreated(pod, oc.GetNetInfo()) } return nil } diff --git a/go-controller/pkg/ovn/policy_test.go b/go-controller/pkg/ovn/policy_test.go index 4bb6363d07..5edb74f7f5 100644 --- a/go-controller/pkg/ovn/policy_test.go +++ b/go-controller/pkg/ovn/policy_test.go @@ -38,8 +38,8 @@ import ( func getFakeController(controllerName string) *DefaultNetworkController { controller := &DefaultNetworkController{ BaseNetworkController: BaseNetworkController{ - controllerName: controllerName, - NetInfo: &util.DefaultNetInfo{}, + controllerName: controllerName, + ReconcilableNetInfo: &util.DefaultNetInfo{}, }, } return controller @@ -76,8 +76,8 @@ func newNetworkPolicy(name, namespace string, podSelector metav1.LabelSelector, func getFakeBaseController(netInfo util.NetInfo) *BaseNetworkController { return &BaseNetworkController{ - controllerName: getNetworkControllerName(netInfo.GetNetworkName()), - NetInfo: netInfo, + controllerName: getNetworkControllerName(netInfo.GetNetworkName()), + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), } } diff --git a/go-controller/pkg/ovn/secondary_layer2_network_controller.go b/go-controller/pkg/ovn/secondary_layer2_network_controller.go index c69cafbbab..938b9c4135 100644 --- a/go-controller/pkg/ovn/secondary_layer2_network_controller.go +++ b/go-controller/pkg/ovn/secondary_layer2_network_controller.go @@ -17,7 +17,7 @@ import ( libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" svccontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/services" lsm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager" @@ -30,7 +30,6 @@ import ( utilerrors "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" corev1 "k8s.io/api/core/v1" - kapi "k8s.io/api/core/v1" "k8s.io/klog/v2" ) @@ -148,19 +147,19 @@ func (h *secondaryLayer2NetworkControllerEventHandler) UpdateResource(oldObj, ne if !ok { return fmt.Errorf("could not cast %T object to Node", newObj) } - oldNode, ok := oldObj.(*kapi.Node) + oldNode, ok := oldObj.(*corev1.Node) if !ok { return fmt.Errorf("could not cast oldObj of type %T to *kapi.Node", oldObj) } newNodeIsLocalZoneNode := h.oc.isLocalZoneNode(newNode) - nodeSubnetChanged := nodeSubnetChanged(oldNode, newNode, h.oc.NetInfo.GetNetworkName()) + nodeSubnetChanged := nodeSubnetChanged(oldNode, newNode, h.oc.GetNetworkName()) if newNodeIsLocalZoneNode { var nodeSyncsParam *nodeSyncs if h.oc.isLocalZoneNode(oldNode) { // determine what actually changed in this update and combine that with what failed previously _, mgmtUpdateFailed := h.oc.mgmtPortFailed.Load(newNode.Name) shouldSyncMgmtPort := mgmtUpdateFailed || - macAddressChanged(oldNode, newNode, h.oc.NetInfo.GetNetworkName()) || + macAddressChanged(oldNode, newNode, h.oc.GetNetworkName()) || nodeSubnetChanged _, gwUpdateFailed := h.oc.gatewaysFailed.Load(newNode.Name) shouldSyncGW := gwUpdateFailed || @@ -257,10 +256,19 @@ type SecondaryLayer2NetworkController struct { // Controller in charge of services svcController *svccontroller.Controller + + // EgressIP controller utilized only to initialize a network with OVN polices to support EgressIP functionality. + eIPController *EgressIPController } // NewSecondaryLayer2NetworkController create a new OVN controller for the given secondary layer2 nad -func NewSecondaryLayer2NetworkController(cnci *CommonNetworkControllerInfo, netInfo util.NetInfo, nadController nad.NADController) (*SecondaryLayer2NetworkController, error) { +func NewSecondaryLayer2NetworkController( + cnci *CommonNetworkControllerInfo, + netInfo util.NetInfo, + networkManager networkmanager.Interface, + eIPController *EgressIPController, + portCache *PortCache, +) (*SecondaryLayer2NetworkController, error) { stopChan := make(chan struct{}) @@ -280,7 +288,7 @@ func NewSecondaryLayer2NetworkController(cnci *CommonNetworkControllerInfo, netI cnci.watchFactory.ServiceCoreInformer(), cnci.watchFactory.EndpointSliceCoreInformer(), cnci.watchFactory.NodeCoreInformer(), - nadController, + networkManager, cnci.recorder, netInfo, ) @@ -296,9 +304,9 @@ func NewSecondaryLayer2NetworkController(cnci *CommonNetworkControllerInfo, netI BaseNetworkController: BaseNetworkController{ CommonNetworkControllerInfo: *cnci, controllerName: getNetworkControllerName(netInfo.GetNetworkName()), - NetInfo: netInfo, + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), lsManager: lsManagerFactoryFn(), - logicalPortCache: NewPortCache(stopChan), + logicalPortCache: portCache, namespaces: make(map[string]*namespaceInfo), namespacesMutex: sync.Mutex{}, addressSetFactory: addressSetFactory, @@ -309,7 +317,7 @@ func NewSecondaryLayer2NetworkController(cnci *CommonNetworkControllerInfo, netI wg: &sync.WaitGroup{}, localZoneNodes: &sync.Map{}, cancelableCtx: util.NewCancelableContext(), - nadController: nadController, + networkManager: networkManager, }, }, }, @@ -317,10 +325,11 @@ func NewSecondaryLayer2NetworkController(cnci *CommonNetworkControllerInfo, netI syncZoneICFailed: sync.Map{}, gatewayManagers: sync.Map{}, svcController: svcController, + eIPController: eIPController, } if config.OVNKubernetesFeature.EnableInterconnect { - oc.zoneICHandler = zoneinterconnect.NewZoneInterconnectHandler(oc.NetInfo, oc.nbClient, oc.sbClient, oc.watchFactory) + oc.zoneICHandler = zoneinterconnect.NewZoneInterconnectHandler(oc.GetNetInfo(), oc.nbClient, oc.sbClient, oc.watchFactory) } if oc.allocatesPodAnnotation() { @@ -328,7 +337,7 @@ func NewSecondaryLayer2NetworkController(cnci *CommonNetworkControllerInfo, netI if oc.allowPersistentIPs() { ipamClaimsReconciler := persistentips.NewIPAMClaimReconciler( oc.kube, - oc.NetInfo, + oc.GetNetInfo(), oc.watchFactory.IPAMClaimsInformer().Lister(), ) oc.ipamClaimsReconciler = ipamClaimsReconciler @@ -418,7 +427,7 @@ func (oc *SecondaryLayer2NetworkController) Init() error { } oc.defaultCOPPUUID = defaultCOPPUUID - clusterLBGroupUUID, switchLBGroupUUID, routerLBGroupUUID, err := initLoadBalancerGroups(oc.nbClient, oc.NetInfo) + clusterLBGroupUUID, switchLBGroupUUID, routerLBGroupUUID, err := initLoadBalancerGroups(oc.nbClient, oc.GetNetInfo()) if err != nil { return err } @@ -459,7 +468,7 @@ func (oc *SecondaryLayer2NetworkController) Stop() { func (oc *SecondaryLayer2NetworkController) initRetryFramework() { oc.retryNodes = oc.newRetryFramework(factory.NodeType) oc.retryPods = oc.newRetryFramework(factory.PodType) - if oc.allocatesPodAnnotation() && oc.NetInfo.AllowsPersistentIPs() { + if oc.allocatesPodAnnotation() && oc.AllowsPersistentIPs() { oc.retryIPAMClaims = oc.newRetryFramework(factory.IPAMClaimsType) } @@ -539,22 +548,31 @@ func (oc *SecondaryLayer2NetworkController) addUpdateLocalNodeEvent(node *corev1 } } } - if util.IsNetworkSegmentationSupportEnabled() && oc.IsPrimaryNetwork() { - if nSyncs.syncMgmtPort { - // Layer 2 networks have a single, large subnet, that's the one - // associated to the controller. Take the management port IP from - // there. - subnets := oc.Subnets() - hostSubnets := make([]*net.IPNet, 0, len(subnets)) - for _, subnet := range oc.Subnets() { - hostSubnets = append(hostSubnets, subnet.CIDR) - } - if _, err := oc.syncNodeManagementPort(node, oc.GetNetworkScopedSwitchName(types.OVNLayer2Switch), oc.GetNetworkScopedGWRouterName(node.Name), hostSubnets); err != nil { - errs = append(errs, err) - oc.mgmtPortFailed.Store(node.Name, true) - } else { - oc.mgmtPortFailed.Delete(node.Name) - } + + if nSyncs.syncMgmtPort { + // Layer 2 networks have a single, large subnet, that's the one + // associated to the controller. Take the management port IP from + // there. + subnets := oc.Subnets() + hostSubnets := make([]*net.IPNet, 0, len(subnets)) + for _, subnet := range oc.Subnets() { + hostSubnets = append(hostSubnets, subnet.CIDR) + } + if _, err := oc.syncNodeManagementPort(node, oc.GetNetworkScopedSwitchName(types.OVNLayer2Switch), + oc.GetNetworkScopedGWRouterName(node.Name), hostSubnets); err != nil { + errs = append(errs, err) + oc.mgmtPortFailed.Store(node.Name, true) + } else { + oc.mgmtPortFailed.Delete(node.Name) + } + } + + if config.OVNKubernetesFeature.EnableEgressIP { + if err := oc.eIPController.ensureRouterPoliciesForNetwork(oc.GetNetInfo()); err != nil { + errs = append(errs, fmt.Errorf("failed to ensure EgressIP router policies for network %s: %v", oc.GetNetworkName(), err)) + } + if err := oc.eIPController.ensureSwitchPoliciesForNode(oc.GetNetInfo(), node.Name); err != nil { + errs = append(errs, fmt.Errorf("failed to ensure EgressIP switch policies for network %s: %v", oc.GetNetworkName(), err)) } } } @@ -665,7 +683,7 @@ func (oc *SecondaryLayer2NetworkController) deleteNodeEvent(node *corev1.Node) e // externalIP = "169.254.0.12"; which is the masqueradeIP for this L2 UDN // so all in all we want to condionally SNAT all packets that are coming from pods hosted on this node, // which are leaving via UDN's mpX interface to the UDN's masqueradeIP. -func (oc *SecondaryLayer2NetworkController) addUDNClusterSubnetEgressSNAT(localPodSubnets []*net.IPNet, routerName string, node *kapi.Node) error { +func (oc *SecondaryLayer2NetworkController) addUDNClusterSubnetEgressSNAT(localPodSubnets []*net.IPNet, routerName string, node *corev1.Node) error { outputPort := types.GWRouterToJoinSwitchPrefix + routerName nats, err := oc.buildUDNEgressSNAT(localPodSubnets, outputPort, node) if err != nil { @@ -748,7 +766,7 @@ func (oc *SecondaryLayer2NetworkController) newGatewayManager(nodeName string) * oc.defaultCOPPUUID, oc.kube, oc.nbClient, - oc.NetInfo, + oc.GetNetInfo(), oc.watchFactory, oc.gatewayOptions()..., ) diff --git a/go-controller/pkg/ovn/secondary_layer2_network_controller_test.go b/go-controller/pkg/ovn/secondary_layer2_network_controller_test.go index b24c77961b..b47107b274 100644 --- a/go-controller/pkg/ovn/secondary_layer2_network_controller_test.go +++ b/go-controller/pkg/ovn/secondary_layer2_network_controller_test.go @@ -22,10 +22,10 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" + testnm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/networkmanager" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) @@ -340,10 +340,10 @@ var _ = Describe("OVN Multi-Homed pod operations for layer2 network", func() { networkConfig, err := util.NewNetInfo(netConf) Expect(err).NotTo(HaveOccurred()) - nadController := &nad.FakeNADController{ + fakeNetworkManager := &testnm.FakeNetworkManager{ PrimaryNetworks: map[string]util.NetInfo{}, } - nadController.PrimaryNetworks[ns] = networkConfig + fakeNetworkManager.PrimaryNetworks[ns] = networkConfig nad, err := newNetworkAttachmentDefinition( ns, nadName, @@ -423,7 +423,9 @@ var _ = Describe("OVN Multi-Homed pod operations for layer2 network", func() { &secondaryNetController.bnc.CommonNetworkControllerInfo, networkConfig, nodeName, - nadController, + fakeNetworkManager, + nil, + NewPortCache(ctx.Done()), ).Cleanup()).To(Succeed()) Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData([]libovsdbtest.TestData{nbZone})) @@ -552,7 +554,7 @@ func expectedLayer2EgressEntities(netInfo util.NetInfo, gwConfig util.L3GatewayC staticRouteOutputPort := ovntypes.GWRouterToExtSwitchPrefix + gwRouterName gwRouterToNetworkSwitchPortName := ovntypes.RouterToSwitchPrefix + netInfo.GetNetworkScopedName(ovntypes.OVNLayer2Switch) gwRouterToExtSwitchPortName := fmt.Sprintf("%s%s", ovntypes.GWRouterToExtSwitchPrefix, gwRouterName) - masqSNAT := newMasqueradeManagementNATEntry(masqSNATUUID1, "169.254.169.14", layer2Subnet().String(), netInfo) + masqSNAT := newMasqueradeManagementNATEntry(masqSNATUUID1, netInfo) var nat []string if config.Gateway.DisableSNATMultipleGWs { @@ -668,9 +670,15 @@ func dummyLayer2PrimaryUserDefinedNetwork(subnets string) secondaryNetInfo { return secondaryNet } -func newSecondaryLayer2NetworkController(cnci *CommonNetworkControllerInfo, netInfo util.NetInfo, nodeName string, - nadController networkAttachDefController.NADController) *SecondaryLayer2NetworkController { - layer2NetworkController, _ := NewSecondaryLayer2NetworkController(cnci, netInfo, nadController) +func newSecondaryLayer2NetworkController( + cnci *CommonNetworkControllerInfo, + netInfo util.NetInfo, + nodeName string, + networkManager networkmanager.Interface, + eIPController *EgressIPController, + portCache *PortCache, +) *SecondaryLayer2NetworkController { + layer2NetworkController, _ := NewSecondaryLayer2NetworkController(cnci, netInfo, networkManager, eIPController, portCache) layer2NetworkController.gatewayManagers.Store( nodeName, newDummyGatewayManager(cnci.kube, cnci.nbClient, netInfo, cnci.watchFactory, nodeName), @@ -752,7 +760,7 @@ func setupFakeOvnForLayer2Topology(fakeOvn *FakeOVN, initialDB libovsdbtest.Test return fmt.Errorf("expected pod annotation %q", util.OvnPodAnnotationName) } } - if err = fakeOvn.controller.nadController.Start(); err != nil { + if err = fakeOvn.networkManager.Start(); err != nil { return err } diff --git a/go-controller/pkg/ovn/secondary_layer3_network_controller.go b/go-controller/pkg/ovn/secondary_layer3_network_controller.go index 1c99522826..18d22182f8 100644 --- a/go-controller/pkg/ovn/secondary_layer3_network_controller.go +++ b/go-controller/pkg/ovn/secondary_layer3_network_controller.go @@ -17,7 +17,7 @@ import ( libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - nad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" svccontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/services" lsm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager" @@ -159,8 +159,8 @@ func (h *secondaryLayer3NetworkControllerEventHandler) UpdateResource(oldObj, ne return fmt.Errorf("could not cast oldObj of type %T to *kapi.Node", oldObj) } newNodeIsLocalZoneNode := h.oc.isLocalZoneNode(newNode) - zoneClusterChanged := h.oc.nodeZoneClusterChanged(oldNode, newNode, newNodeIsLocalZoneNode, h.oc.NetInfo.GetNetworkName()) - nodeSubnetChanged := nodeSubnetChanged(oldNode, newNode, h.oc.NetInfo.GetNetworkName()) + zoneClusterChanged := h.oc.nodeZoneClusterChanged(oldNode, newNode, newNodeIsLocalZoneNode, h.oc.GetNetworkName()) + nodeSubnetChanged := nodeSubnetChanged(oldNode, newNode, h.oc.GetNetworkName()) if newNodeIsLocalZoneNode { var nodeSyncsParam *nodeSyncs if h.oc.isLocalZoneNode(oldNode) { @@ -310,8 +310,13 @@ type SecondaryLayer3NetworkController struct { } // NewSecondaryLayer3NetworkController create a new OVN controller for the given secondary layer3 NAD -func NewSecondaryLayer3NetworkController(cnci *CommonNetworkControllerInfo, netInfo util.NetInfo, nadController nad.NADController, - eIPController *EgressIPController, portCache *PortCache) (*SecondaryLayer3NetworkController, error) { +func NewSecondaryLayer3NetworkController( + cnci *CommonNetworkControllerInfo, + netInfo util.NetInfo, + networkManager networkmanager.Interface, + eIPController *EgressIPController, + portCache *PortCache, +) (*SecondaryLayer3NetworkController, error) { stopChan := make(chan struct{}) ipv4Mode, ipv6Mode := netInfo.IPMode() @@ -330,7 +335,7 @@ func NewSecondaryLayer3NetworkController(cnci *CommonNetworkControllerInfo, netI cnci.watchFactory.ServiceCoreInformer(), cnci.watchFactory.EndpointSliceCoreInformer(), cnci.watchFactory.NodeCoreInformer(), - nadController, + networkManager, cnci.recorder, netInfo, ) @@ -344,7 +349,7 @@ func NewSecondaryLayer3NetworkController(cnci *CommonNetworkControllerInfo, netI BaseNetworkController: BaseNetworkController{ CommonNetworkControllerInfo: *cnci, controllerName: getNetworkControllerName(netInfo.GetNetworkName()), - NetInfo: netInfo, + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), lsManager: lsm.NewLogicalSwitchManager(), logicalPortCache: portCache, namespaces: make(map[string]*namespaceInfo), @@ -358,7 +363,7 @@ func NewSecondaryLayer3NetworkController(cnci *CommonNetworkControllerInfo, netI localZoneNodes: &sync.Map{}, zoneICHandler: zoneICHandler, cancelableCtx: util.NewCancelableContext(), - nadController: nadController, + networkManager: networkManager, }, }, mgmtPortFailed: sync.Map{}, @@ -614,7 +619,7 @@ func (oc *SecondaryLayer3NetworkController) Init(ctx context.Context) error { // Only configure join switch, GR, cluster port groups and multicast default policies for user defined primary networks. if util.IsNetworkSegmentationSupportEnabled() && oc.IsPrimaryNetwork() { - if err := oc.gatewayTopologyFactory.NewJoinSwitch(clusterRouter, oc.NetInfo, oc.ovnClusterLRPToJoinIfAddrs); err != nil { + if err := oc.gatewayTopologyFactory.NewJoinSwitch(clusterRouter, oc.GetNetInfo(), oc.ovnClusterLRPToJoinIfAddrs); err != nil { return fmt.Errorf("failed to create join switch for network %q: %v", oc.GetNetworkName(), err) } @@ -632,7 +637,7 @@ func (oc *SecondaryLayer3NetworkController) Init(ctx context.Context) error { if _, _, err := util.RunOVNNbctl("--columns=_uuid", "list", "Load_Balancer_Group"); err != nil { klog.Warningf("Load Balancer Group support enabled, however version of OVN in use does not support Load Balancer Groups.") } else { - clusterLBGroupUUID, switchLBGroupUUID, routerLBGroupUUID, err := initLoadBalancerGroups(oc.nbClient, oc.NetInfo) + clusterLBGroupUUID, switchLBGroupUUID, routerLBGroupUUID, err := initLoadBalancerGroups(oc.nbClient, oc.GetNetInfo()) if err != nil { return err } @@ -759,11 +764,11 @@ func (oc *SecondaryLayer3NetworkController) addUpdateLocalNodeEvent(node *kapi.N } if config.OVNKubernetesFeature.EnableEgressIP && util.IsNetworkSegmentationSupportEnabled() && oc.IsPrimaryNetwork() { - if err = oc.eIPController.ensureL3ClusterRouterPoliciesForNetwork(oc.NetInfo); err != nil { - errs = append(errs, fmt.Errorf("failed to add network %s to EgressIP controller: %v", oc.NetInfo.GetNetworkName(), err)) + if err = oc.eIPController.ensureRouterPoliciesForNetwork(oc.GetNetInfo()); err != nil { + errs = append(errs, fmt.Errorf("failed to ensure EgressIP router polices for network %s: %v", oc.GetNetworkName(), err)) } - if err = oc.eIPController.ensureL3SwitchPoliciesForNode(oc.NetInfo, node.Name); err != nil { - errs = append(errs, fmt.Errorf("failed to ensure EgressIP switch policies: %v", err)) + if err = oc.eIPController.ensureSwitchPoliciesForNode(oc.GetNetInfo(), node.Name); err != nil { + errs = append(errs, fmt.Errorf("failed to ensure EgressIP switch policies for network %s: %v", oc.GetNetworkName(), err)) } } @@ -1016,13 +1021,13 @@ func (oc *SecondaryLayer3NetworkController) newClusterRouter() (*nbdb.LogicalRou if oc.multicastSupport { return oc.gatewayTopologyFactory.NewClusterRouterWithMulticastSupport( oc.GetNetworkScopedClusterRouterName(), - oc.NetInfo, + oc.GetNetInfo(), oc.defaultCOPPUUID, ) } return oc.gatewayTopologyFactory.NewClusterRouter( oc.GetNetworkScopedClusterRouterName(), - oc.NetInfo, + oc.GetNetInfo(), oc.defaultCOPPUUID, ) } @@ -1033,7 +1038,7 @@ func (oc *SecondaryLayer3NetworkController) newGatewayManager(nodeName string) * oc.defaultCOPPUUID, oc.kube, oc.nbClient, - oc.NetInfo, + oc.GetNetInfo(), oc.watchFactory, oc.gatewayOptions()..., ) diff --git a/go-controller/pkg/ovn/secondary_layer3_network_controller_test.go b/go-controller/pkg/ovn/secondary_layer3_network_controller_test.go index efbd660c4b..9a5adeb2b0 100644 --- a/go-controller/pkg/ovn/secondary_layer3_network_controller_test.go +++ b/go-controller/pkg/ovn/secondary_layer3_network_controller_test.go @@ -26,10 +26,11 @@ import ( libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" libovsdbutil "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" - fakenad "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/nad" + testnm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/networkmanager" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" networkingv1 "k8s.io/api/networking/v1" @@ -168,7 +169,8 @@ var _ = Describe("OVN Multi-Homed pod operations", func() { _, ok := pod.Annotations[util.OvnPodAnnotationName] Expect(ok).To(BeFalse()) - Expect(fakeOvn.controller.nadController.Start()).NotTo(HaveOccurred()) + Expect(fakeOvn.networkManager.Start()).NotTo(HaveOccurred()) + defer fakeOvn.networkManager.Stop() Expect(fakeOvn.controller.WatchNamespaces()).NotTo(HaveOccurred()) Expect(fakeOvn.controller.WatchPods()).NotTo(HaveOccurred()) @@ -185,7 +187,7 @@ var _ = Describe("OVN Multi-Homed pod operations", func() { if netInfo.isPrimary { Expect(secondaryNetController.bnc.WatchNetworkPolicy()).To(Succeed()) - ninfo, err := fakeOvn.nadController.GetActiveNetworkForNamespace(ns) + ninfo, err := fakeOvn.networkManager.Interface().GetActiveNetworkForNamespace(ns) Expect(err).NotTo(HaveOccurred()) Expect(ninfo.GetNetworkName()).To(Equal(netInfo.netName)) } @@ -310,11 +312,14 @@ var _ = Describe("OVN Multi-Homed pod operations", func() { ) Expect(err).NotTo(HaveOccurred()) - networkConfig.SetNADs(util.GetNADName(nad.Namespace, nad.Name)) - nadController := &fakenad.FakeNADController{ + mutableNetworkConfig := util.NewMutableNetInfo(networkConfig) + mutableNetworkConfig.SetNADs(util.GetNADName(nad.Namespace, nad.Name)) + networkConfig = mutableNetworkConfig + + fakeNetworkManager := &testnm.FakeNetworkManager{ PrimaryNetworks: make(map[string]util.NetInfo), } - nadController.PrimaryNetworks[ns] = networkConfig + fakeNetworkManager.PrimaryNetworks[ns] = networkConfig const nodeIPv4CIDR = "192.168.126.202/24" testNode, err := newNodeWithSecondaryNets(nodeName, nodeIPv4CIDR, netInfo) @@ -376,7 +381,8 @@ var _ = Describe("OVN Multi-Homed pod operations", func() { _, ok := pod.Annotations[util.OvnPodAnnotationName] Expect(ok).To(BeFalse()) - Expect(fakeOvn.controller.nadController.Start()).NotTo(HaveOccurred()) + Expect(fakeOvn.networkManager.Start()).NotTo(HaveOccurred()) + defer fakeOvn.networkManager.Stop() Expect(fakeOvn.controller.WatchNamespaces()).To(Succeed()) Expect(fakeOvn.controller.WatchPods()).To(Succeed()) @@ -401,8 +407,9 @@ var _ = Describe("OVN Multi-Homed pod operations", func() { &secondaryNetController.bnc.CommonNetworkControllerInfo, networkConfig, nodeName, - nadController, - nil, NewPortCache(ctx.Done()), + fakeNetworkManager, + nil, + NewPortCache(ctx.Done()), ).Cleanup()).To(Succeed()) Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(defaultNetExpectations)) @@ -497,26 +504,26 @@ func (sni *secondaryNetInfo) setupOVNDependencies(dbData *libovsdbtest.TestSetup } externalIDs := map[string]string{ - ovntypes.NetworkExternalID: sni.netName, - ovntypes.NetworkRoleExternalID: sni.getNetworkRole(), + types.NetworkExternalID: sni.netName, + types.NetworkRoleExternalID: sni.getNetworkRole(), } switch sni.topology { - case ovntypes.Layer2Topology: + case types.Layer2Topology: dbData.NBData = append(dbData.NBData, &nbdb.LogicalSwitch{ - Name: netInfo.GetNetworkScopedName(ovntypes.OVNLayer2Switch), - UUID: netInfo.GetNetworkScopedName(ovntypes.OVNLayer2Switch) + "_UUID", + Name: netInfo.GetNetworkScopedName(types.OVNLayer2Switch), + UUID: netInfo.GetNetworkScopedName(types.OVNLayer2Switch) + "_UUID", ExternalIDs: externalIDs, }) - case ovntypes.Layer3Topology: + case types.Layer3Topology: dbData.NBData = append(dbData.NBData, &nbdb.LogicalSwitch{ Name: netInfo.GetNetworkScopedName(nodeName), UUID: netInfo.GetNetworkScopedName(nodeName) + "_UUID", ExternalIDs: externalIDs, }) - case ovntypes.LocalnetTopology: + case types.LocalnetTopology: dbData.NBData = append(dbData.NBData, &nbdb.LogicalSwitch{ - Name: netInfo.GetNetworkScopedName(ovntypes.OVNLocalnetSwitch), - UUID: netInfo.GetNetworkScopedName(ovntypes.OVNLocalnetSwitch) + "_UUID", + Name: netInfo.GetNetworkScopedName(types.OVNLocalnetSwitch), + UUID: netInfo.GetNetworkScopedName(types.OVNLocalnetSwitch) + "_UUID", ExternalIDs: externalIDs, }) default: @@ -528,9 +535,9 @@ func (sni *secondaryNetInfo) setupOVNDependencies(dbData *libovsdbtest.TestSetup func (sni *secondaryNetInfo) netconf() *ovncnitypes.NetConf { const plugin = "ovn-k8s-cni-overlay" - role := ovntypes.NetworkRoleSecondary + role := types.NetworkRoleSecondary if sni.isPrimary { - role = ovntypes.NetworkRolePrimary + role = types.NetworkRolePrimary } return &ovncnitypes.NetConf{ NetConf: cnitypes.NetConf{ @@ -589,7 +596,7 @@ func dummySecondaryLayer3UserDefinedNetwork(clustersubnets, hostsubnets string) return secondaryNetInfo{ netName: secondaryNetworkName, nadName: namespacedName(ns, nadName), - topology: ovntypes.Layer3Topology, + topology: types.Layer3Topology, clustersubnets: clustersubnets, hostsubnets: hostsubnets, } @@ -690,8 +697,8 @@ func expectedGWRouterPlusNATAndStaticRoutes( netInfo util.NetInfo, gwConfig util.L3GatewayConfig, ) []libovsdbtest.TestData { - gwRouterToExtLRPUUID := fmt.Sprintf("%s%s-UUID", ovntypes.GWRouterToExtSwitchPrefix, gwRouterName) - gwRouterToJoinLRPUUID := fmt.Sprintf("%s%s-UUID", ovntypes.GWRouterToJoinSwitchPrefix, gwRouterName) + gwRouterToExtLRPUUID := fmt.Sprintf("%s%s-UUID", types.GWRouterToExtSwitchPrefix, gwRouterName) + gwRouterToJoinLRPUUID := fmt.Sprintf("%s%s-UUID", types.GWRouterToJoinSwitchPrefix, gwRouterName) const ( nat1 = "abc-UUID" @@ -703,7 +710,7 @@ func expectedGWRouterPlusNATAndStaticRoutes( ipv4DefaultRoute = "0.0.0.0/0" ) - staticRouteOutputPort := ovntypes.GWRouterToExtSwitchPrefix + netInfo.GetNetworkScopedGWRouterName(nodeName) + staticRouteOutputPort := types.GWRouterToExtSwitchPrefix + netInfo.GetNetworkScopedGWRouterName(nodeName) nextHopIP := gwConfig.NextHops[0].String() nextHopMasqIP := nextHopMasqueradeIP().String() masqSubnet := config.Gateway.V4MasqueradeSubnet @@ -753,18 +760,18 @@ func expectedStaticMACBindings(gwRouterName string, ips []net.IP) []libovsdbtest } func expectedGatewayChassis(nodeName string, netInfo util.NetInfo, gwConfig util.L3GatewayConfig) *nbdb.GatewayChassis { - gwChassisName := fmt.Sprintf("%s%s_%s-%s", ovntypes.RouterToSwitchPrefix, netInfo.GetNetworkName(), nodeName, gwConfig.ChassisID) + gwChassisName := fmt.Sprintf("%s%s_%s-%s", types.RouterToSwitchPrefix, netInfo.GetNetworkName(), nodeName, gwConfig.ChassisID) return &nbdb.GatewayChassis{UUID: gwChassisName + "-UUID", Name: gwChassisName, Priority: 1, ChassisName: gwConfig.ChassisID} } func expectedGRToJoinSwitchLRP(gatewayRouterName string, gwRouterLRPIP *net.IPNet, netInfo util.NetInfo) *nbdb.LogicalRouterPort { - lrpName := fmt.Sprintf("%s%s", ovntypes.GWRouterToJoinSwitchPrefix, gatewayRouterName) + lrpName := fmt.Sprintf("%s%s", types.GWRouterToJoinSwitchPrefix, gatewayRouterName) options := map[string]string{"gateway_mtu": fmt.Sprintf("%d", 1400)} return expectedLogicalRouterPort(lrpName, netInfo, options, gwRouterLRPIP) } func expectedGRToExternalSwitchLRP(gatewayRouterName string, netInfo util.NetInfo, joinSwitchIPs ...*net.IPNet) *nbdb.LogicalRouterPort { - lrpName := fmt.Sprintf("%s%s", ovntypes.GWRouterToExtSwitchPrefix, gatewayRouterName) + lrpName := fmt.Sprintf("%s%s", types.GWRouterToExtSwitchPrefix, gatewayRouterName) return expectedLogicalRouterPort(lrpName, netInfo, nil, joinSwitchIPs...) } @@ -785,8 +792,8 @@ func expectedLogicalRouterPort(lrpName string, netInfo util.NetInfo, options map MAC: mac, Options: options, ExternalIDs: map[string]string{ - ovntypes.TopologyExternalID: netInfo.TopologyType(), - ovntypes.NetworkExternalID: netInfo.GetNetworkName(), + types.TopologyExternalID: netInfo.TopologyType(), + types.NetworkExternalID: netInfo.GetNetworkName(), }, } } @@ -802,7 +809,7 @@ func expectedLayer3EgressEntities(netInfo util.NetInfo, gwConfig util.L3GatewayC ) masqIPAddr := dummyMasqueradeIP().IP.String() clusterRouterName := fmt.Sprintf("%s_ovn_cluster_router", netInfo.GetNetworkName()) - rtosLRPName := fmt.Sprintf("%s%s", ovntypes.RouterToSwitchPrefix, netInfo.GetNetworkScopedName(nodeName)) + rtosLRPName := fmt.Sprintf("%s%s", types.RouterToSwitchPrefix, netInfo.GetNetworkScopedName(nodeName)) rtosLRPUUID := rtosLRPName + "-UUID" nodeIP := gwConfig.IPAddresses[0].IP.String() masqSNAT := newNATEntry(masqSNATUUID1, "169.254.169.14", nodeSubnet.String(), standardNonDefaultNetworkExtIDs(netInfo), "") @@ -843,7 +850,7 @@ func expectedLogicalRouterPolicy(routerPolicyUUID1 string, netInfo util.NetInfo, rerouteAction = "reroute" ) networkScopedSwitchName := netInfo.GetNetworkScopedSwitchName(nodeName) - lrpName := fmt.Sprintf("%s%s", ovntypes.RouterToSwitchPrefix, networkScopedSwitchName) + lrpName := fmt.Sprintf("%s%s", types.RouterToSwitchPrefix, networkScopedSwitchName) return &nbdb.LogicalRouterPolicy{ UUID: routerPolicyUUID1, @@ -863,8 +870,8 @@ func expectedGRStaticRoute(uuid, ipPrefix, nextHop string, policy *nbdb.LogicalR Nexthop: nextHop, Policy: policy, ExternalIDs: map[string]string{ - ovntypes.NetworkExternalID: "isolatednet", - ovntypes.TopologyExternalID: netInfo.TopologyType(), + types.NetworkExternalID: "isolatednet", + types.TopologyExternalID: netInfo.TopologyType(), }, } } @@ -903,9 +910,14 @@ func udnGWSNATAddress() *net.IPNet { } } -func newMasqueradeManagementNATEntry(uuid string, externalIP string, logicalIP string, netInfo util.NetInfo) *nbdb.NAT { - masqSNAT := newNATEntry(uuid, "169.254.169.14", layer2Subnet().String(), standardNonDefaultNetworkExtIDs(netInfo), - getMasqueradeManagementIPSNATMatch(util.IPAddrToHWAddr(managementPortIP(layer2Subnet())).String())) +func newMasqueradeManagementNATEntry(uuid string, netInfo util.NetInfo) *nbdb.NAT { + masqSNAT := newNATEntry( + uuid, + "169.254.169.14", + layer2Subnet().String(), + standardNonDefaultNetworkExtIDs(netInfo), + getMasqueradeManagementIPSNATMatch(util.IPAddrToHWAddr(managementPortIP(layer2Subnet())).String()), + ) masqSNAT.LogicalPort = ptr.To(fmt.Sprintf("rtoj-GR_%s_%s", netInfo.GetNetworkName(), nodeName)) return masqSNAT } @@ -941,11 +953,11 @@ func expectedExternalSwitchAndLSPs(netInfo util.NetInfo, gwConfig util.L3Gateway Addresses: []string{"unknown"}, ExternalIDs: standardNonDefaultNetworkExtIDs(netInfo), Options: map[string]string{"network_name": "physnet"}, - Type: ovntypes.LocalnetTopology, + Type: types.LocalnetTopology, }, &nbdb.LogicalSwitchPort{ UUID: port2UUID, - Name: ovntypes.EXTSwitchToGWRouterPrefix + gwRouterName, + Name: types.EXTSwitchToGWRouterPrefix + gwRouterName, Addresses: []string{gwConfig.MACAddress.String()}, ExternalIDs: standardNonDefaultNetworkExtIDs(netInfo), Options: externalSwitchRouterPortOptions(gwRouterName), @@ -958,7 +970,7 @@ func externalSwitchRouterPortOptions(gatewayRouterName string) map[string]string return map[string]string{ "nat-addresses": "router", "exclude-lb-vips-from-garp": "true", - "router-port": ovntypes.GWRouterToExtSwitchPrefix + gatewayRouterName, + "router-port": types.GWRouterToExtSwitchPrefix + gatewayRouterName, } } @@ -974,10 +986,10 @@ func expectedJoinSwitchAndLSPs(netInfo util.NetInfo, nodeName string) []libovsdb }, &nbdb.LogicalSwitchPort{ UUID: joinToGRLSPUUID, - Name: ovntypes.JoinSwitchToGWRouterPrefix + gwRouterName, + Name: types.JoinSwitchToGWRouterPrefix + gwRouterName, Addresses: []string{"router"}, ExternalIDs: standardNonDefaultNetworkExtIDs(netInfo), - Options: map[string]string{"router-port": ovntypes.GWRouterToJoinSwitchPrefix + gwRouterName}, + Options: map[string]string{"router-port": types.GWRouterToJoinSwitchPrefix + gwRouterName}, Type: "router", }, } @@ -1011,20 +1023,26 @@ func gwRouterOptions(gwConfig util.L3GatewayConfig) map[string]string { func standardNonDefaultNetworkExtIDs(netInfo util.NetInfo) map[string]string { return map[string]string{ - ovntypes.TopologyExternalID: netInfo.TopologyType(), - ovntypes.NetworkExternalID: netInfo.GetNetworkName(), + types.TopologyExternalID: netInfo.TopologyType(), + types.NetworkExternalID: netInfo.GetNetworkName(), } } func standardNonDefaultNetworkExtIDsForLogicalSwitch(netInfo util.NetInfo) map[string]string { externalIDs := standardNonDefaultNetworkExtIDs(netInfo) - externalIDs[ovntypes.NetworkRoleExternalID] = getNetworkRole(netInfo) + externalIDs[types.NetworkRoleExternalID] = getNetworkRole(netInfo) return externalIDs } -func newSecondaryLayer3NetworkController(cnci *CommonNetworkControllerInfo, netInfo util.NetInfo, nodeName string, - nadController networkAttachDefController.NADController, eIPController *EgressIPController, portCache *PortCache) *SecondaryLayer3NetworkController { - layer3NetworkController, err := NewSecondaryLayer3NetworkController(cnci, netInfo, nadController, eIPController, portCache) +func newSecondaryLayer3NetworkController( + cnci *CommonNetworkControllerInfo, + netInfo util.NetInfo, + nodeName string, + networkManager networkmanager.Interface, + eIPController *EgressIPController, + portCache *PortCache, +) *SecondaryLayer3NetworkController { + layer3NetworkController, err := NewSecondaryLayer3NetworkController(cnci, netInfo, networkManager, eIPController, portCache) Expect(err).NotTo(HaveOccurred()) layer3NetworkController.gatewayManagers.Store( nodeName, diff --git a/go-controller/pkg/ovn/secondary_localnet_network_controller.go b/go-controller/pkg/ovn/secondary_localnet_network_controller.go index 4071a317cb..ac750bc89b 100644 --- a/go-controller/pkg/ovn/secondary_localnet_network_controller.go +++ b/go-controller/pkg/ovn/secondary_localnet_network_controller.go @@ -14,7 +14,7 @@ import ( libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" lsm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/persistentips" @@ -185,8 +185,11 @@ type SecondaryLocalnetNetworkController struct { } // NewSecondaryLocalnetNetworkController create a new OVN controller for the given secondary localnet NAD -func NewSecondaryLocalnetNetworkController(cnci *CommonNetworkControllerInfo, netInfo util.NetInfo, - nadController networkAttachDefController.NADController) *SecondaryLocalnetNetworkController { +func NewSecondaryLocalnetNetworkController( + cnci *CommonNetworkControllerInfo, + netInfo util.NetInfo, + networkManager networkmanager.Interface, +) *SecondaryLocalnetNetworkController { stopChan := make(chan struct{}) @@ -198,7 +201,7 @@ func NewSecondaryLocalnetNetworkController(cnci *CommonNetworkControllerInfo, ne BaseNetworkController: BaseNetworkController{ CommonNetworkControllerInfo: *cnci, controllerName: getNetworkControllerName(netInfo.GetNetworkName()), - NetInfo: netInfo, + ReconcilableNetInfo: util.NewReconcilableNetInfo(netInfo), lsManager: lsm.NewL2SwitchManager(), logicalPortCache: NewPortCache(stopChan), namespaces: make(map[string]*namespaceInfo), @@ -211,7 +214,7 @@ func NewSecondaryLocalnetNetworkController(cnci *CommonNetworkControllerInfo, ne wg: &sync.WaitGroup{}, cancelableCtx: util.NewCancelableContext(), localZoneNodes: &sync.Map{}, - nadController: nadController, + networkManager: networkManager, }, }, }, @@ -222,7 +225,7 @@ func NewSecondaryLocalnetNetworkController(cnci *CommonNetworkControllerInfo, ne if oc.allowPersistentIPs() { ipamClaimsReconciler := persistentips.NewIPAMClaimReconciler( oc.kube, - oc.NetInfo, + oc.GetNetInfo(), oc.watchFactory.IPAMClaimsInformer().Lister(), ) oc.ipamClaimsReconciler = ipamClaimsReconciler @@ -256,10 +259,10 @@ func (oc *SecondaryLocalnetNetworkController) Start(ctx context.Context) error { return err } - return oc.run(ctx) + return oc.run() } -func (oc *SecondaryLocalnetNetworkController) run(ctx context.Context) error { +func (oc *SecondaryLocalnetNetworkController) run() error { return oc.BaseSecondaryLayer2NetworkController.run() } @@ -308,7 +311,7 @@ func (oc *SecondaryLocalnetNetworkController) Stop() { func (oc *SecondaryLocalnetNetworkController) initRetryFramework() { oc.retryNodes = oc.newRetryFramework(factory.NodeType) oc.retryPods = oc.newRetryFramework(factory.PodType) - if oc.allocatesPodAnnotation() && oc.NetInfo.AllowsPersistentIPs() { + if oc.allocatesPodAnnotation() && oc.AllowsPersistentIPs() { oc.retryIPAMClaims = oc.newRetryFramework(factory.IPAMClaimsType) } diff --git a/go-controller/pkg/testing/mocks/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1/NetworkAttachmentDefinitionInformer.go b/go-controller/pkg/testing/mocks/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1/NetworkAttachmentDefinitionInformer.go new file mode 100644 index 0000000000..17da81f3f7 --- /dev/null +++ b/go-controller/pkg/testing/mocks/github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/k8s.cni.cncf.io/v1/NetworkAttachmentDefinitionInformer.go @@ -0,0 +1,68 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mocks + +import ( + k8s_cni_cncf_iov1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1" + mock "github.com/stretchr/testify/mock" + cache "k8s.io/client-go/tools/cache" +) + +// NetworkAttachmentDefinitionInformer is an autogenerated mock type for the NetworkAttachmentDefinitionInformer type +type NetworkAttachmentDefinitionInformer struct { + mock.Mock +} + +// Informer provides a mock function with given fields: +func (_m *NetworkAttachmentDefinitionInformer) Informer() cache.SharedIndexInformer { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Informer") + } + + var r0 cache.SharedIndexInformer + if rf, ok := ret.Get(0).(func() cache.SharedIndexInformer); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cache.SharedIndexInformer) + } + } + + return r0 +} + +// Lister provides a mock function with given fields: +func (_m *NetworkAttachmentDefinitionInformer) Lister() k8s_cni_cncf_iov1.NetworkAttachmentDefinitionLister { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Lister") + } + + var r0 k8s_cni_cncf_iov1.NetworkAttachmentDefinitionLister + if rf, ok := ret.Get(0).(func() k8s_cni_cncf_iov1.NetworkAttachmentDefinitionLister); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(k8s_cni_cncf_iov1.NetworkAttachmentDefinitionLister) + } + } + + return r0 +} + +// NewNetworkAttachmentDefinitionInformer creates a new instance of NetworkAttachmentDefinitionInformer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewNetworkAttachmentDefinitionInformer(t interface { + mock.TestingT + Cleanup(func()) +}) *NetworkAttachmentDefinitionInformer { + mock := &NetworkAttachmentDefinitionInformer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/go-controller/pkg/testing/nad/netattach.go b/go-controller/pkg/testing/nad/netattach.go deleted file mode 100644 index 1e60fd8784..0000000000 --- a/go-controller/pkg/testing/nad/netattach.go +++ /dev/null @@ -1,79 +0,0 @@ -package nad - -import ( - "context" - "errors" - - networkAttachDefController "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/network-attach-def-controller" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" -) - -type FakeNetworkController struct { - util.NetInfo -} - -func (nc *FakeNetworkController) Start(ctx context.Context) error { - return nil -} - -func (nc *FakeNetworkController) Stop() {} - -func (nc *FakeNetworkController) Cleanup() error { - return nil -} - -type FakeNetworkControllerManager struct{} - -func (ncm *FakeNetworkControllerManager) NewNetworkController(netInfo util.NetInfo) (networkAttachDefController.NetworkController, error) { - return &FakeNetworkController{netInfo}, nil -} - -func (ncm *FakeNetworkControllerManager) CleanupDeletedNetworks(validNetworks ...util.BasicNetInfo) error { - return nil -} - -type FakeNADController struct { - // namespace -> netInfo - PrimaryNetworks map[string]util.NetInfo -} - -func (nc *FakeNADController) Start() error { return nil } -func (nc *FakeNADController) Stop() {} -func (nc *FakeNADController) GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) { - if primaryNetworks, ok := nc.PrimaryNetworks[namespace]; ok && primaryNetworks != nil { - return primaryNetworks, nil - } - return &util.DefaultNetInfo{}, nil -} -func (nc *FakeNADController) GetNetwork(networkName string) (util.NetInfo, error) { - for _, ni := range nc.PrimaryNetworks { - if ni.GetNetworkName() == networkName { - return ni, nil - } - } - return &util.DefaultNetInfo{}, nil -} -func (nc *FakeNADController) GetActiveNetworkNamespaces(networkName string) ([]string, error) { - namespaces := make([]string, 0) - for namespaceName, primaryNAD := range nc.PrimaryNetworks { - nadNetworkName := primaryNAD.GetNADs()[0] - if nadNetworkName != networkName { - continue - } - namespaces = append(namespaces, namespaceName) - } - return namespaces, nil -} - -func (nc *FakeNADController) DoWithLock(f func(network util.NetInfo) error) error { - var errs []error - for _, ni := range nc.PrimaryNetworks { - if err := f(ni); err != nil { - errs = append(errs, err) - } - } - if len(errs) > 0 { - return errors.Join(errs...) - } - return nil -} diff --git a/go-controller/pkg/testing/networkmanager/fake.go b/go-controller/pkg/testing/networkmanager/fake.go new file mode 100644 index 0000000000..bd9774a6b9 --- /dev/null +++ b/go-controller/pkg/testing/networkmanager/fake.go @@ -0,0 +1,91 @@ +package networkmanager + +import ( + "context" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/networkmanager" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/errors" +) + +type FakeNetworkController struct { + util.NetInfo +} + +func (fnc *FakeNetworkController) Start(ctx context.Context) error { + return nil +} + +func (fnc *FakeNetworkController) Stop() {} + +func (fnc *FakeNetworkController) Cleanup() error { + return nil +} + +func (nc *FakeNetworkController) Reconcile(util.NetInfo) error { + return nil +} + +type FakeControllerManager struct{} + +func (fcm *FakeControllerManager) NewNetworkController(netInfo util.NetInfo) (networkmanager.NetworkController, error) { + return &FakeNetworkController{netInfo}, nil +} + +func (fcm *FakeControllerManager) CleanupStaleNetworks(validNetworks ...util.NetInfo) error { + return nil +} + +func (fcm *FakeControllerManager) GetDefaultNetworkController() networkmanager.ReconcilableNetworkController { + return nil +} + +type FakeNetworkManager struct { + // namespace -> netInfo + PrimaryNetworks map[string]util.NetInfo +} + +func (fnm *FakeNetworkManager) Start() error { return nil } + +func (fnm *FakeNetworkManager) Stop() {} + +func (fnm *FakeNetworkManager) GetActiveNetworkForNamespace(namespace string) (util.NetInfo, error) { + if primaryNetworks, ok := fnm.PrimaryNetworks[namespace]; ok && primaryNetworks != nil { + return primaryNetworks, nil + } + return &util.DefaultNetInfo{}, nil +} + +func (nc *FakeNetworkManager) GetNetwork(networkName string) util.NetInfo { + for _, ni := range nc.PrimaryNetworks { + if ni.GetNetworkName() == networkName { + return ni + } + } + return &util.DefaultNetInfo{} +} + +func (nc *FakeNetworkManager) GetActiveNetworkNamespaces(networkName string) ([]string, error) { + namespaces := make([]string, 0) + for namespaceName, primaryNAD := range nc.PrimaryNetworks { + nadNetworkName := primaryNAD.GetNADs()[0] + if nadNetworkName != networkName { + continue + } + namespaces = append(namespaces, namespaceName) + } + return namespaces, nil +} + +func (nc *FakeNetworkManager) DoWithLock(f func(network util.NetInfo) error) error { + var errs []error + for _, ni := range nc.PrimaryNetworks { + if err := f(ni); err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil +} diff --git a/go-controller/pkg/types/const.go b/go-controller/pkg/types/const.go index 1be9adcb45..22d92cbf24 100644 --- a/go-controller/pkg/types/const.go +++ b/go-controller/pkg/types/const.go @@ -161,8 +161,9 @@ const ( DefaultNetworkLabelSelector = OvnK8sPrefix + "/default-network" // Deprecated: we used to set topology version as an annotation on the node. We don't do this anymore. - OvnK8sTopoAnno = OvnK8sPrefix + "/" + "topology-version" - OvnK8sSmallMTUTaintKey = OvnK8sPrefix + "/" + "mtu-too-small" + OvnK8sTopoAnno = OvnK8sPrefix + "/" + "topology-version" + OvnK8sSmallMTUTaintKey = OvnK8sPrefix + "/" + "mtu-too-small" + OvnRouteAdvertisementsKey = OvnK8sPrefix + "/route-advertisements" // name of the configmap used to synchronize status (e.g. watch for topology changes) OvnK8sStatusCMName = "control-plane-status" diff --git a/go-controller/pkg/util/kube.go b/go-controller/pkg/util/kube.go index 60dc77b412..6822a68d83 100644 --- a/go-controller/pkg/util/kube.go +++ b/go-controller/pkg/util/kube.go @@ -87,7 +87,7 @@ type OVNMasterClientset struct { RouteAdvertisementsClient routeadvertisementsclientset.Interface } -// OVNNetworkControllerManagerClientset +// OVNKubeControllerClientset type OVNKubeControllerClientset struct { KubeClient kubernetes.Interface ANPClient anpclientset.Interface diff --git a/go-controller/pkg/util/multi_network.go b/go-controller/pkg/util/multi_network.go index 2af60e6019..2d2a4bb7dc 100644 --- a/go-controller/pkg/util/multi_network.go +++ b/go-controller/pkg/util/multi_network.go @@ -4,11 +4,13 @@ import ( "errors" "fmt" "net" + "reflect" "strings" "sync" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "golang.org/x/exp/maps" kapi "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -26,9 +28,9 @@ var ( ErrorUnsupportedIPAMKey = errors.New("IPAM key is not supported. Use OVN-K provided IPAM via the `subnets` attribute") ) -// BasicNetInfo is interface which holds basic network information -type BasicNetInfo interface { - // basic network information +// NetInfo exposes read-only information about a network. +type NetInfo interface { + // static information, not expected to change. GetNetworkName() string IsDefault() bool IsPrimaryNetwork() bool @@ -45,8 +47,27 @@ type BasicNetInfo interface { AllowsPersistentIPs() bool PhysicalNetworkName() string - // utility methods - Equals(BasicNetInfo) bool + // dynamic information, can change over time + GetNADs() []string + HasNAD(nadName string) bool + // GetPodNetworkAdvertisedVRFs returns the target VRFs where the pod network + // is advertised per node, through a map of node names to slice of VRFs. + GetPodNetworkAdvertisedVRFs() map[string][]string + // GetPodNetworkAdvertisedOnNodeVRFs returns the target VRFs where the pod + // network is advertised on the specified node. + GetPodNetworkAdvertisedOnNodeVRFs(node string) []string + // GetEgressIPAdvertisedVRFs returns the target VRFs where egress IPs are + // advertised per node, through a map of node names to slice of VRFs. + GetEgressIPAdvertisedVRFs() map[string][]string + // GetEgressIPAdvertisedOnNodeVRFs returns the target VRFs where egress IPs + // are advertised on the specified node. + GetEgressIPAdvertisedOnNodeVRFs(node string) []string + // GetEgressIPAdvertisedNodes return the nodes where egress IP are + // advertised. + GetEgressIPAdvertisedNodes() []string + + // derived information. + GetNamespaces() []string GetNetworkScopedName(name string) string RemoveNetworkScopeFromName(name string) string GetNetworkScopedK8sMgmtIntfName(nodeName string) string @@ -60,20 +81,350 @@ type BasicNetInfo interface { GetNetworkScopedLoadBalancerName(lbName string) string GetNetworkScopedLoadBalancerGroupName(lbGroupName string) string GetNetworkScopedClusterSubnetSNATMatch(nodeName string) string + + // GetNetInfo is an identity method used to get the specific NetInfo + // implementation + GetNetInfo() NetInfo } -// NetInfo correlates which NADs refer to a network in addition to the basic -// network information -type NetInfo interface { - BasicNetInfo - GetNADs() []string - HasNAD(nadName string) bool +// DefaultNetInfo is the default network information +type DefaultNetInfo struct { + mutableNetInfo +} + +// MutableNetInfo is a NetInfo where selected information can be changed. +// Intended to be used by network managers that aggregate network information +// from multiple sources that can change over time. +type MutableNetInfo interface { + NetInfo + + // NADs referencing a network SetNADs(nadName ...string) AddNADs(nadName ...string) DeleteNADs(nadName ...string) + + // VRFs a pod network is being advertised on, also per node + SetPodNetworkAdvertisedVRFs(podAdvertisements map[string][]string) + + // Nodes advertising Egress IP + SetEgressIPAdvertisedVRFs(eipAdvertisements map[string][]string) +} + +// NewMutableNetInfo builds a copy of netInfo as a MutableNetInfo +func NewMutableNetInfo(netInfo NetInfo) MutableNetInfo { + if netInfo == nil { + return nil + } + return copyNetInfo(netInfo).(MutableNetInfo) +} + +// ReconcilableNetInfo is a NetInfo that can be reconciled +type ReconcilableNetInfo interface { + NetInfo + + // canReconcile checks if both networks are compatible and thus can be + // reconciled. Networks are compatible if they are defined by the same + // static network configuration. + canReconcile(NetInfo) bool + + // needsReconcile checks if both networks hold differences in their dynamic + // network configuration that could potentially be reconciled. Note this + // method does not check for compatibility. + needsReconcile(NetInfo) bool + + // reconcile copies dynamic network configuration information from the + // provided network + reconcile(NetInfo) +} + +// NewReconcilableNetInfo builds a copy of netInfo as a ReconcilableNetInfo +func NewReconcilableNetInfo(netInfo NetInfo) ReconcilableNetInfo { + if netInfo == nil { + return nil + } + return copyNetInfo(netInfo).(ReconcilableNetInfo) +} + +// AreNetworksCompatible checks if both networks are compatible and thus can be +// reconciled. Networks are compatible if they are defined by the same +// static network configuration. +func AreNetworksCompatible(l, r NetInfo) bool { + if l == nil && r == nil { + return true + } + if l == nil || r == nil { + return false + } + return reconcilable(l).canReconcile(r) +} + +// DoesNetworkNeedReconciliation checks if both networks hold differences in their dynamic +// network configuration that could potentially be reconciled. Note this +// method does not check for compatibility. +func DoesNetworkNeedReconciliation(l, r NetInfo) bool { + if l == nil && r == nil { + return false + } + if l == nil || r == nil { + return true + } + return reconcilable(l).needsReconcile(r) +} + +// ReconcileNetInfo reconciles the dynamic network configuration +func ReconcileNetInfo(to ReconcilableNetInfo, from NetInfo) error { + if from == nil || to == nil { + return fmt.Errorf("can't reconcile a nil network") + } + if !AreNetworksCompatible(to, from) { + return fmt.Errorf("can't reconcile from incompatible network") + } + reconcilable(to).reconcile(from) + return nil +} + +func copyNetInfo(netInfo NetInfo) any { + switch t := netInfo.GetNetInfo().(type) { + case *DefaultNetInfo: + return t.copy() + case *secondaryNetInfo: + return t.copy() + default: + panic(fmt.Errorf("unrecognized type %T", t)) + } +} + +func reconcilable(netInfo NetInfo) ReconcilableNetInfo { + switch t := netInfo.GetNetInfo().(type) { + case *DefaultNetInfo: + return t + case *secondaryNetInfo: + return t + default: + panic(fmt.Errorf("unrecognized type %T", t)) + } +} + +// mutableNetInfo contains network information that can be changed +type mutableNetInfo struct { + sync.RWMutex + + nads sets.Set[string] + + // Each network can be selected by multiple routeadvertisements each with a + // different target VRF and different selected node. Represent this through + // a map of node names to VRFs. + podNetworkAdvertisements map[string][]string + eipAdvertisements map[string][]string + + // information generated from previous fields, not used in comparisons + + // namespaces from nads + namespaces sets.Set[string] +} + +func mutable(netInfo NetInfo) *mutableNetInfo { + switch t := netInfo.GetNetInfo().(type) { + case *DefaultNetInfo: + return &t.mutableNetInfo + case *secondaryNetInfo: + return &t.mutableNetInfo + default: + panic(fmt.Errorf("unrecognized type %T", t)) + } +} + +func (l *mutableNetInfo) needsReconcile(r NetInfo) bool { + return !mutable(r).equals(l) +} + +func (l *mutableNetInfo) reconcile(r NetInfo) { + l.copyFrom(mutable(r)) +} + +func (l *mutableNetInfo) equals(r *mutableNetInfo) bool { + l.RLock() + defer l.RUnlock() + r.RLock() + defer r.RUnlock() + return reflect.DeepEqual(l.nads, r.nads) && + reflect.DeepEqual(l.podNetworkAdvertisements, r.podNetworkAdvertisements) && + reflect.DeepEqual(l.eipAdvertisements, r.eipAdvertisements) +} + +func (l *mutableNetInfo) copyFrom(r *mutableNetInfo) { + aux := mutableNetInfo{} + r.RLock() + aux.nads = r.nads.Clone() + aux.setPodNetworkAdvertisedOnVRFs(r.podNetworkAdvertisements) + aux.setEgressIPAdvertisedAtNodes(r.eipAdvertisements) + aux.namespaces = r.namespaces.Clone() + r.RUnlock() + l.Lock() + defer l.Unlock() + l.nads = aux.nads + l.podNetworkAdvertisements = aux.podNetworkAdvertisements + l.eipAdvertisements = aux.eipAdvertisements + l.namespaces = aux.namespaces +} + +func (nInfo *mutableNetInfo) SetPodNetworkAdvertisedVRFs(podAdvertisements map[string][]string) { + nInfo.Lock() + defer nInfo.Unlock() + nInfo.setPodNetworkAdvertisedOnVRFs(podAdvertisements) +} + +func (nInfo *mutableNetInfo) setPodNetworkAdvertisedOnVRFs(podAdvertisements map[string][]string) { + nInfo.podNetworkAdvertisements = make(map[string][]string, len(podAdvertisements)) + for node, vrfs := range podAdvertisements { + nInfo.podNetworkAdvertisements[node] = sets.List(sets.New(vrfs...)) + } +} + +func (nInfo *mutableNetInfo) GetPodNetworkAdvertisedVRFs() map[string][]string { + nInfo.RLock() + defer nInfo.RUnlock() + return nInfo.getPodNetworkAdvertisedOnVRFs() +} + +func (nInfo *mutableNetInfo) GetPodNetworkAdvertisedOnNodeVRFs(node string) []string { + nInfo.RLock() + defer nInfo.RUnlock() + return nInfo.getPodNetworkAdvertisedOnVRFs()[node] +} + +func (nInfo *mutableNetInfo) getPodNetworkAdvertisedOnVRFs() map[string][]string { + if nInfo.podNetworkAdvertisements == nil { + return map[string][]string{} + } + return nInfo.podNetworkAdvertisements +} + +func (nInfo *mutableNetInfo) SetEgressIPAdvertisedVRFs(eipAdvertisements map[string][]string) { + nInfo.Lock() + defer nInfo.Unlock() + nInfo.setEgressIPAdvertisedAtNodes(eipAdvertisements) +} + +func (nInfo *mutableNetInfo) setEgressIPAdvertisedAtNodes(eipAdvertisements map[string][]string) { + nInfo.eipAdvertisements = make(map[string][]string, len(eipAdvertisements)) + for node, vrfs := range eipAdvertisements { + nInfo.eipAdvertisements[node] = sets.List(sets.New(vrfs...)) + } +} + +func (nInfo *mutableNetInfo) GetEgressIPAdvertisedVRFs() map[string][]string { + nInfo.RLock() + defer nInfo.RUnlock() + return nInfo.getEgressIPAdvertisedVRFs() +} + +func (nInfo *mutableNetInfo) getEgressIPAdvertisedVRFs() map[string][]string { + if nInfo.eipAdvertisements == nil { + return map[string][]string{} + } + return nInfo.eipAdvertisements } -type DefaultNetInfo struct{} +func (nInfo *mutableNetInfo) GetEgressIPAdvertisedOnNodeVRFs(node string) []string { + nInfo.RLock() + defer nInfo.RUnlock() + return nInfo.getEgressIPAdvertisedVRFs()[node] +} + +func (nInfo *mutableNetInfo) GetEgressIPAdvertisedNodes() []string { + nInfo.RLock() + defer nInfo.RUnlock() + return maps.Keys(nInfo.eipAdvertisements) +} + +// GetNADs returns all the NADs associated with this network +func (nInfo *mutableNetInfo) GetNADs() []string { + nInfo.RLock() + defer nInfo.RUnlock() + return nInfo.getNads().UnsortedList() +} + +// HasNAD returns true if the given NAD exists, used +// to check if the network needs to be plumbed over +func (nInfo *mutableNetInfo) HasNAD(nadName string) bool { + nInfo.RLock() + defer nInfo.RUnlock() + return nInfo.getNads().Has(nadName) +} + +// SetNADs replaces the NADs associated with the network +func (nInfo *mutableNetInfo) SetNADs(nadNames ...string) { + nInfo.Lock() + defer nInfo.Unlock() + nInfo.nads = sets.New[string]() + nInfo.namespaces = sets.New[string]() + nInfo.addNADs(nadNames...) +} + +// AddNADs adds the specified NAD +func (nInfo *mutableNetInfo) AddNADs(nadNames ...string) { + nInfo.Lock() + defer nInfo.Unlock() + nInfo.addNADs(nadNames...) +} + +func (nInfo *mutableNetInfo) addNADs(nadNames ...string) { + for _, name := range nadNames { + nInfo.getNads().Insert(name) + nInfo.getNamespaces().Insert(strings.Split(name, "/")[0]) + } +} + +// DeleteNADs deletes the specified NAD +func (nInfo *mutableNetInfo) DeleteNADs(nadNames ...string) { + nInfo.Lock() + defer nInfo.Unlock() + ns := sets.New[string]() + for _, name := range nadNames { + if !nInfo.getNads().Has(name) { + continue + } + ns.Insert(strings.Split(name, "/")[0]) + nInfo.getNads().Delete(name) + } + if ns.Len() == 0 { + return + } + for existing := range nInfo.getNads() { + ns.Delete(strings.Split(existing, "/")[0]) + } + nInfo.getNamespaces().Delete(ns.UnsortedList()...) +} + +func (nInfo *mutableNetInfo) getNads() sets.Set[string] { + if nInfo.nads == nil { + return sets.New[string]() + } + return nInfo.nads +} + +func (nInfo *mutableNetInfo) getNamespaces() sets.Set[string] { + if nInfo.namespaces == nil { + return sets.New[string]() + } + return nInfo.namespaces +} + +func (nInfo *mutableNetInfo) GetNamespaces() []string { + return nInfo.getNamespaces().UnsortedList() +} + +func (nInfo *DefaultNetInfo) GetNetInfo() NetInfo { + return nInfo +} + +func (nInfo *DefaultNetInfo) copy() *DefaultNetInfo { + c := &DefaultNetInfo{} + c.mutableNetInfo.copyFrom(&nInfo.mutableNetInfo) + + return c +} // GetNetworkName returns the network name func (nInfo *DefaultNetInfo) GetNetworkName() string { @@ -155,36 +506,8 @@ func (nInfo *DefaultNetInfo) GetNetworkScopedClusterSubnetSNATMatch(nodeName str return "" } -// GetNADs returns the NADs associated with the network, no op for default -// network -func (nInfo *DefaultNetInfo) GetNADs() []string { - panic("unexpected call for default network") -} - -// HasNAD returns true if the given NAD exists, already return true for -// default network -func (nInfo *DefaultNetInfo) HasNAD(nadName string) bool { - panic("unexpected call for default network") -} - -// SetNADs replaces the NADs associated with the network, no op for default -// network -func (nInfo *DefaultNetInfo) SetNADs(nadName ...string) { - panic("unexpected call for default network") -} - -// AddNAD adds the specified NAD, no op for default network -func (nInfo *DefaultNetInfo) AddNADs(nadName ...string) { - panic("unexpected call for default network") -} - -// DeleteNAD deletes the specified NAD, no op for default network -func (nInfo *DefaultNetInfo) DeleteNADs(nadName ...string) { - panic("unexpected call for default network") -} - -func (nInfo *DefaultNetInfo) Equals(netBasicInfo BasicNetInfo) bool { - _, ok := netBasicInfo.(*DefaultNetInfo) +func (nInfo *DefaultNetInfo) canReconcile(netInfo NetInfo) bool { + _, ok := netInfo.(*DefaultNetInfo) return ok } @@ -272,6 +595,8 @@ func (nInfo *DefaultNetInfo) PhysicalNetworkName() string { // SecondaryNetInfo holds the network name information for secondary network if non-nil type secondaryNetInfo struct { + mutableNetInfo + netName string // Should this secondary network be used // as the pod's primary network? @@ -286,14 +611,13 @@ type secondaryNetInfo struct { excludeSubnets []*net.IPNet joinSubnets []*net.IPNet - // all net-attach-def NAD names for this network, used to determine if a pod needs - // to be plumbed for this network - sync.Mutex - nadNames sets.Set[string] - physicalNetworkName string } +func (nInfo *secondaryNetInfo) GetNetInfo() NetInfo { + return nInfo +} + // GetNetworkName returns the network name func (nInfo *secondaryNetInfo) GetNetworkName() string { return nInfo.netName @@ -385,42 +709,6 @@ func (nInfo *secondaryNetInfo) getPrefix() string { return GetSecondaryNetworkPrefix(nInfo.netName) } -// GetNADs returns all the NADs associated with this network -func (nInfo *secondaryNetInfo) GetNADs() []string { - nInfo.Lock() - defer nInfo.Unlock() - return nInfo.nadNames.UnsortedList() -} - -// HasNAD returns true if the given NAD exists, used -// to check if the network needs to be plumbed over -func (nInfo *secondaryNetInfo) HasNAD(nadName string) bool { - nInfo.Lock() - defer nInfo.Unlock() - return nInfo.nadNames.Has(nadName) -} - -// SetNADs replaces the NADs associated with the network -func (nInfo *secondaryNetInfo) SetNADs(nadName ...string) { - nInfo.Lock() - defer nInfo.Unlock() - nInfo.nadNames = sets.New(nadName...) -} - -// AddNAD adds the specified NAD -func (nInfo *secondaryNetInfo) AddNADs(nadName ...string) { - nInfo.Lock() - defer nInfo.Unlock() - nInfo.nadNames.Insert(nadName...) -} - -// DeleteNAD deletes the specified NAD -func (nInfo *secondaryNetInfo) DeleteNADs(nadName ...string) { - nInfo.Lock() - defer nInfo.Unlock() - nInfo.nadNames.Delete(nadName...) -} - // TopologyType returns the topology type func (nInfo *secondaryNetInfo) TopologyType() string { return nInfo.topology @@ -486,8 +774,7 @@ func (nInfo *secondaryNetInfo) JoinSubnets() []*net.IPNet { return nInfo.joinSubnets } -// Equals compares for equality this network information with the other -func (nInfo *secondaryNetInfo) Equals(other BasicNetInfo) bool { +func (nInfo *secondaryNetInfo) canReconcile(other NetInfo) bool { if (nInfo == nil) != (other == nil) { return false } @@ -526,10 +813,7 @@ func (nInfo *secondaryNetInfo) Equals(other BasicNetInfo) bool { } func (nInfo *secondaryNetInfo) copy() *secondaryNetInfo { - nInfo.Lock() - defer nInfo.Unlock() - - // everything is immutable except the NADs + // everything here is immutable c := &secondaryNetInfo{ netName: nInfo.netName, primaryNetwork: nInfo.primaryNetwork, @@ -542,14 +826,15 @@ func (nInfo *secondaryNetInfo) copy() *secondaryNetInfo { subnets: nInfo.subnets, excludeSubnets: nInfo.excludeSubnets, joinSubnets: nInfo.joinSubnets, - nadNames: nInfo.nadNames.Clone(), physicalNetworkName: nInfo.physicalNetworkName, } + // copy mutables + c.mutableNetInfo.copyFrom(&nInfo.mutableNetInfo) return c } -func newLayer3NetConfInfo(netconf *ovncnitypes.NetConf) (NetInfo, error) { +func newLayer3NetConfInfo(netconf *ovncnitypes.NetConf) (MutableNetInfo, error) { subnets, _, err := parseSubnets(netconf.Subnets, "", types.Layer3Topology) if err != nil { return nil, err @@ -565,13 +850,15 @@ func newLayer3NetConfInfo(netconf *ovncnitypes.NetConf) (NetInfo, error) { subnets: subnets, joinSubnets: joinSubnets, mtu: netconf.MTU, - nadNames: sets.Set[string]{}, + mutableNetInfo: mutableNetInfo{ + nads: sets.Set[string]{}, + }, } ni.ipv4mode, ni.ipv6mode = getIPMode(subnets) return ni, nil } -func newLayer2NetConfInfo(netconf *ovncnitypes.NetConf) (NetInfo, error) { +func newLayer2NetConfInfo(netconf *ovncnitypes.NetConf) (MutableNetInfo, error) { subnets, excludes, err := parseSubnets(netconf.Subnets, netconf.ExcludeSubnets, types.Layer2Topology) if err != nil { return nil, fmt.Errorf("invalid %s netconf %s: %v", netconf.Topology, netconf.Name, err) @@ -589,13 +876,15 @@ func newLayer2NetConfInfo(netconf *ovncnitypes.NetConf) (NetInfo, error) { excludeSubnets: excludes, mtu: netconf.MTU, allowPersistentIPs: netconf.AllowPersistentIPs, - nadNames: sets.Set[string]{}, + mutableNetInfo: mutableNetInfo{ + nads: sets.Set[string]{}, + }, } ni.ipv4mode, ni.ipv6mode = getIPMode(subnets) return ni, nil } -func newLocalnetNetConfInfo(netconf *ovncnitypes.NetConf) (NetInfo, error) { +func newLocalnetNetConfInfo(netconf *ovncnitypes.NetConf) (MutableNetInfo, error) { subnets, excludes, err := parseSubnets(netconf.Subnets, netconf.ExcludeSubnets, types.LocalnetTopology) if err != nil { return nil, fmt.Errorf("invalid %s netconf %s: %v", netconf.Topology, netconf.Name, err) @@ -609,8 +898,10 @@ func newLocalnetNetConfInfo(netconf *ovncnitypes.NetConf) (NetInfo, error) { mtu: netconf.MTU, vlan: uint(netconf.VLANID), allowPersistentIPs: netconf.AllowPersistentIPs, - nadNames: sets.Set[string]{}, physicalNetworkName: netconf.PhysicalNetworkName, + mutableNetInfo: mutableNetInfo{ + nads: sets.Set[string]{}, + }, } ni.ipv4mode, ni.ipv6mode = getIPMode(subnets) return ni, nil @@ -729,10 +1020,14 @@ func GetSecondaryNetworkPrefix(netName string) string { } func NewNetInfo(netconf *ovncnitypes.NetConf) (NetInfo, error) { + return newNetInfo(netconf) +} + +func newNetInfo(netconf *ovncnitypes.NetConf) (MutableNetInfo, error) { if netconf.Name == types.DefaultNetworkName { return &DefaultNetInfo{}, nil } - var ni NetInfo + var ni MutableNetInfo var err error switch netconf.Topology { case types.Layer3Topology: @@ -884,18 +1179,6 @@ func subnetOverlapCheck(netconf *ovncnitypes.NetConf) error { return nil } -func CopyNetInfo(netInfo NetInfo) NetInfo { - switch t := netInfo.(type) { - case *DefaultNetInfo: - // immutable - return netInfo - case *secondaryNetInfo: - return t.copy() - default: - panic("program error: unrecognized NetInfo") - } -} - // GetPodNADToNetworkMapping sees if the given pod needs to plumb over this given network specified by netconf, // and return the matching NetworkSelectionElement if any exists. // @@ -1032,3 +1315,7 @@ func AllowsPersistentIPs(netInfo NetInfo) bool { return false } } + +func IsPodNetworkAdvertisedAtNode(netInfo NetInfo, node string) bool { + return len(netInfo.GetPodNetworkAdvertisedOnNodeVRFs(node)) > 0 +} diff --git a/go-controller/pkg/util/multi_network_test.go b/go-controller/pkg/util/multi_network_test.go index a098de3e5d..b7d0cbc76f 100644 --- a/go-controller/pkg/util/multi_network_test.go +++ b/go-controller/pkg/util/multi_network_test.go @@ -465,6 +465,8 @@ func TestParseNetconf(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { + config.IPv4Mode = true + config.IPv6Mode = true if test.unsupportedReason != "" { t.Skip(test.unsupportedReason) } @@ -824,7 +826,9 @@ func TestGetPodNADToNetworkMapping(t *testing.T) { netInfo, err := NewNetInfo(test.inputNetConf) g.Expect(err).To(gomega.BeNil()) if test.inputNetConf.NADName != "" { - netInfo.AddNADs(test.inputNetConf.NADName) + mutableNetInfo := NewMutableNetInfo(netInfo) + mutableNetInfo.AddNADs(test.inputNetConf.NADName) + netInfo = mutableNetInfo } pod := &corev1.Pod{ @@ -1017,7 +1021,9 @@ func TestGetPodNADToNetworkMappingWithActiveNetwork(t *testing.T) { netInfo, err := NewNetInfo(test.inputNetConf) g.Expect(err).To(gomega.BeNil()) if test.inputNetConf.NADName != "" { - netInfo.AddNADs(test.inputNetConf.NADName) + mutableNetInfo := NewMutableNetInfo(netInfo) + mutableNetInfo.AddNADs(test.inputNetConf.NADName) + netInfo = mutableNetInfo } var primaryUDNNetInfo NetInfo @@ -1025,7 +1031,9 @@ func TestGetPodNADToNetworkMappingWithActiveNetwork(t *testing.T) { primaryUDNNetInfo, err = NewNetInfo(test.inputPrimaryUDNConfig) g.Expect(err).To(gomega.BeNil()) if test.inputPrimaryUDNConfig.NADName != "" { - primaryUDNNetInfo.AddNADs(test.inputPrimaryUDNConfig.NADName) + mutableNetInfo := NewMutableNetInfo(primaryUDNNetInfo) + mutableNetInfo.AddNADs(test.inputPrimaryUDNConfig.NADName) + primaryUDNNetInfo = mutableNetInfo } } diff --git a/go-controller/pkg/util/node_annotations.go b/go-controller/pkg/util/node_annotations.go index d2581f9977..33a5d36d78 100644 --- a/go-controller/pkg/util/node_annotations.go +++ b/go-controller/pkg/util/node_annotations.go @@ -907,6 +907,25 @@ func ParseNodeGatewayRouterJoinIPv4(node *kapi.Node, netName string) (net.IP, er return ip, nil } +// ParseNodeGatewayRouterJoinIPv6 returns the IPv6 address for the node's gateway router port +// stored in the 'OVNNodeGRLRPAddrs' annotation +func ParseNodeGatewayRouterJoinIPv6(node *kapi.Node, netName string) (net.IP, error) { + primaryIfAddr, err := ParseNodeGatewayRouterJoinNetwork(node, netName) + if err != nil { + return nil, err + } + if primaryIfAddr.IPv6 == "" { + return nil, fmt.Errorf("failed to find an IPv6 address for gateway route interface in node: %s, net: %s, "+ + "annotation values: %+v", node, netName, primaryIfAddr) + } + + ip, _, err := net.ParseCIDR(primaryIfAddr.IPv6) + if err != nil { + return nil, fmt.Errorf("failed to parse gateway router IPv6 address %s, err: %w", primaryIfAddr.IPv6, err) + } + return ip, nil +} + // ParseNodeGatewayRouterJoinAddrs returns the IPv4 and/or IPv6 addresses for the node's gateway router port // stored in the 'OVNNodeGRLRPAddrs' annotation func ParseNodeGatewayRouterJoinAddrs(node *kapi.Node, netName string) ([]*net.IPNet, error) { @@ -1503,7 +1522,7 @@ func filterIPVersion(cidrs []netip.Prefix, v6 bool) []netip.Prefix { // GetNetworkID will retrieve the network id for the specified network from the // first node that contains that network at the network id annotations, it will // return at the first ocurrence, rest of nodes will not be parsed. -func GetNetworkID(nodes []*corev1.Node, nInfo BasicNetInfo) (int, error) { +func GetNetworkID(nodes []*corev1.Node, nInfo NetInfo) (int, error) { for _, node := range nodes { var err error networkID, err := ParseNetworkIDAnnotation(node, nInfo.GetNetworkName()) diff --git a/go-controller/pkg/util/pod_annotation_unit_test.go b/go-controller/pkg/util/pod_annotation_unit_test.go index c2ab82a194..832584c28f 100644 --- a/go-controller/pkg/util/pod_annotation_unit_test.go +++ b/go-controller/pkg/util/pod_annotation_unit_test.go @@ -368,8 +368,9 @@ func newDummyNetInfo(namespace, networkName string) NetInfo { netInfo, _ := newLayer2NetConfInfo(&ovncnitypes.NetConf{ NetConf: cnitypes.NetConf{Name: networkName}, }) - netInfo.AddNADs(GetNADName(namespace, networkName)) - return netInfo + mutableNetInfo := NewMutableNetInfo(netInfo) + mutableNetInfo.AddNADs(GetNADName(namespace, networkName)) + return mutableNetInfo } func TestUnmarshalUDNOpenPortsAnnotation(t *testing.T) { diff --git a/go-controller/vendor/github.com/onsi/ginkgo/LICENSE b/go-controller/vendor/github.com/onsi/ginkgo/LICENSE deleted file mode 100644 index 9415ee72c1..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2013-2014 Onsi Fakhouri - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/go-controller/vendor/github.com/onsi/ginkgo/config/config.go b/go-controller/vendor/github.com/onsi/ginkgo/config/config.go deleted file mode 100644 index 3130c77897..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/config/config.go +++ /dev/null @@ -1,232 +0,0 @@ -/* -Ginkgo accepts a number of configuration options. - -These are documented [here](http://onsi.github.io/ginkgo/#the-ginkgo-cli) - -You can also learn more via - - ginkgo help - -or (I kid you not): - - go test -asdf -*/ -package config - -import ( - "flag" - "time" - - "fmt" -) - -const VERSION = "1.16.5" - -type GinkgoConfigType struct { - RandomSeed int64 - RandomizeAllSpecs bool - RegexScansFilePath bool - FocusStrings []string - SkipStrings []string - SkipMeasurements bool - FailOnPending bool - FailFast bool - FlakeAttempts int - EmitSpecProgress bool - DryRun bool - DebugParallel bool - - ParallelNode int - ParallelTotal int - SyncHost string - StreamHost string -} - -var GinkgoConfig = GinkgoConfigType{} - -type DefaultReporterConfigType struct { - NoColor bool - SlowSpecThreshold float64 - NoisyPendings bool - NoisySkippings bool - Succinct bool - Verbose bool - FullTrace bool - ReportPassed bool - ReportFile string -} - -var DefaultReporterConfig = DefaultReporterConfigType{} - -func processPrefix(prefix string) string { - if prefix != "" { - prefix += "." - } - return prefix -} - -type flagFunc func(string) - -func (f flagFunc) String() string { return "" } -func (f flagFunc) Set(s string) error { f(s); return nil } - -func Flags(flagSet *flag.FlagSet, prefix string, includeParallelFlags bool) { - prefix = processPrefix(prefix) - flagSet.Int64Var(&(GinkgoConfig.RandomSeed), prefix+"seed", time.Now().Unix(), "The seed used to randomize the spec suite.") - flagSet.BoolVar(&(GinkgoConfig.RandomizeAllSpecs), prefix+"randomizeAllSpecs", false, "If set, ginkgo will randomize all specs together. By default, ginkgo only randomizes the top level Describe, Context and When groups.") - flagSet.BoolVar(&(GinkgoConfig.SkipMeasurements), prefix+"skipMeasurements", false, "If set, ginkgo will skip any measurement specs.") - flagSet.BoolVar(&(GinkgoConfig.FailOnPending), prefix+"failOnPending", false, "If set, ginkgo will mark the test suite as failed if any specs are pending.") - flagSet.BoolVar(&(GinkgoConfig.FailFast), prefix+"failFast", false, "If set, ginkgo will stop running a test suite after a failure occurs.") - - flagSet.BoolVar(&(GinkgoConfig.DryRun), prefix+"dryRun", false, "If set, ginkgo will walk the test hierarchy without actually running anything. Best paired with -v.") - - flagSet.Var(flagFunc(flagFocus), prefix+"focus", "If set, ginkgo will only run specs that match this regular expression. Can be specified multiple times, values are ORed.") - flagSet.Var(flagFunc(flagSkip), prefix+"skip", "If set, ginkgo will only run specs that do not match this regular expression. Can be specified multiple times, values are ORed.") - - flagSet.BoolVar(&(GinkgoConfig.RegexScansFilePath), prefix+"regexScansFilePath", false, "If set, ginkgo regex matching also will look at the file path (code location).") - - flagSet.IntVar(&(GinkgoConfig.FlakeAttempts), prefix+"flakeAttempts", 1, "Make up to this many attempts to run each spec. Please note that if any of the attempts succeed, the suite will not be failed. But any failures will still be recorded.") - - flagSet.BoolVar(&(GinkgoConfig.EmitSpecProgress), prefix+"progress", false, "If set, ginkgo will emit progress information as each spec runs to the GinkgoWriter.") - - flagSet.BoolVar(&(GinkgoConfig.DebugParallel), prefix+"debug", false, "If set, ginkgo will emit node output to files when running in parallel.") - - if includeParallelFlags { - flagSet.IntVar(&(GinkgoConfig.ParallelNode), prefix+"parallel.node", 1, "This worker node's (one-indexed) node number. For running specs in parallel.") - flagSet.IntVar(&(GinkgoConfig.ParallelTotal), prefix+"parallel.total", 1, "The total number of worker nodes. For running specs in parallel.") - flagSet.StringVar(&(GinkgoConfig.SyncHost), prefix+"parallel.synchost", "", "The address for the server that will synchronize the running nodes.") - flagSet.StringVar(&(GinkgoConfig.StreamHost), prefix+"parallel.streamhost", "", "The address for the server that the running nodes should stream data to.") - } - - flagSet.BoolVar(&(DefaultReporterConfig.NoColor), prefix+"noColor", false, "If set, suppress color output in default reporter.") - flagSet.Float64Var(&(DefaultReporterConfig.SlowSpecThreshold), prefix+"slowSpecThreshold", 5.0, "(in seconds) Specs that take longer to run than this threshold are flagged as slow by the default reporter.") - flagSet.BoolVar(&(DefaultReporterConfig.NoisyPendings), prefix+"noisyPendings", true, "If set, default reporter will shout about pending tests.") - flagSet.BoolVar(&(DefaultReporterConfig.NoisySkippings), prefix+"noisySkippings", true, "If set, default reporter will shout about skipping tests.") - flagSet.BoolVar(&(DefaultReporterConfig.Verbose), prefix+"v", false, "If set, default reporter print out all specs as they begin.") - flagSet.BoolVar(&(DefaultReporterConfig.Succinct), prefix+"succinct", false, "If set, default reporter prints out a very succinct report") - flagSet.BoolVar(&(DefaultReporterConfig.FullTrace), prefix+"trace", false, "If set, default reporter prints out the full stack trace when a failure occurs") - flagSet.BoolVar(&(DefaultReporterConfig.ReportPassed), prefix+"reportPassed", false, "If set, default reporter prints out captured output of passed tests.") - flagSet.StringVar(&(DefaultReporterConfig.ReportFile), prefix+"reportFile", "", "Override the default reporter output file path.") - -} - -func BuildFlagArgs(prefix string, ginkgo GinkgoConfigType, reporter DefaultReporterConfigType) []string { - prefix = processPrefix(prefix) - result := make([]string, 0) - - if ginkgo.RandomSeed > 0 { - result = append(result, fmt.Sprintf("--%sseed=%d", prefix, ginkgo.RandomSeed)) - } - - if ginkgo.RandomizeAllSpecs { - result = append(result, fmt.Sprintf("--%srandomizeAllSpecs", prefix)) - } - - if ginkgo.SkipMeasurements { - result = append(result, fmt.Sprintf("--%sskipMeasurements", prefix)) - } - - if ginkgo.FailOnPending { - result = append(result, fmt.Sprintf("--%sfailOnPending", prefix)) - } - - if ginkgo.FailFast { - result = append(result, fmt.Sprintf("--%sfailFast", prefix)) - } - - if ginkgo.DryRun { - result = append(result, fmt.Sprintf("--%sdryRun", prefix)) - } - - for _, s := range ginkgo.FocusStrings { - result = append(result, fmt.Sprintf("--%sfocus=%s", prefix, s)) - } - - for _, s := range ginkgo.SkipStrings { - result = append(result, fmt.Sprintf("--%sskip=%s", prefix, s)) - } - - if ginkgo.FlakeAttempts > 1 { - result = append(result, fmt.Sprintf("--%sflakeAttempts=%d", prefix, ginkgo.FlakeAttempts)) - } - - if ginkgo.EmitSpecProgress { - result = append(result, fmt.Sprintf("--%sprogress", prefix)) - } - - if ginkgo.DebugParallel { - result = append(result, fmt.Sprintf("--%sdebug", prefix)) - } - - if ginkgo.ParallelNode != 0 { - result = append(result, fmt.Sprintf("--%sparallel.node=%d", prefix, ginkgo.ParallelNode)) - } - - if ginkgo.ParallelTotal != 0 { - result = append(result, fmt.Sprintf("--%sparallel.total=%d", prefix, ginkgo.ParallelTotal)) - } - - if ginkgo.StreamHost != "" { - result = append(result, fmt.Sprintf("--%sparallel.streamhost=%s", prefix, ginkgo.StreamHost)) - } - - if ginkgo.SyncHost != "" { - result = append(result, fmt.Sprintf("--%sparallel.synchost=%s", prefix, ginkgo.SyncHost)) - } - - if ginkgo.RegexScansFilePath { - result = append(result, fmt.Sprintf("--%sregexScansFilePath", prefix)) - } - - if reporter.NoColor { - result = append(result, fmt.Sprintf("--%snoColor", prefix)) - } - - if reporter.SlowSpecThreshold > 0 { - result = append(result, fmt.Sprintf("--%sslowSpecThreshold=%.5f", prefix, reporter.SlowSpecThreshold)) - } - - if !reporter.NoisyPendings { - result = append(result, fmt.Sprintf("--%snoisyPendings=false", prefix)) - } - - if !reporter.NoisySkippings { - result = append(result, fmt.Sprintf("--%snoisySkippings=false", prefix)) - } - - if reporter.Verbose { - result = append(result, fmt.Sprintf("--%sv", prefix)) - } - - if reporter.Succinct { - result = append(result, fmt.Sprintf("--%ssuccinct", prefix)) - } - - if reporter.FullTrace { - result = append(result, fmt.Sprintf("--%strace", prefix)) - } - - if reporter.ReportPassed { - result = append(result, fmt.Sprintf("--%sreportPassed", prefix)) - } - - if reporter.ReportFile != "" { - result = append(result, fmt.Sprintf("--%sreportFile=%s", prefix, reporter.ReportFile)) - } - - return result -} - -// flagFocus implements the -focus flag. -func flagFocus(arg string) { - if arg != "" { - GinkgoConfig.FocusStrings = append(GinkgoConfig.FocusStrings, arg) - } -} - -// flagSkip implements the -skip flag. -func flagSkip(arg string) { - if arg != "" { - GinkgoConfig.SkipStrings = append(GinkgoConfig.SkipStrings, arg) - } -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/extensions/table/table.go b/go-controller/vendor/github.com/onsi/ginkgo/extensions/table/table.go deleted file mode 100644 index 4b00278073..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/extensions/table/table.go +++ /dev/null @@ -1,110 +0,0 @@ -/* - -Table provides a simple DSL for Ginkgo-native Table-Driven Tests - -The godoc documentation describes Table's API. More comprehensive documentation (with examples!) is available at http://onsi.github.io/ginkgo#table-driven-tests - -*/ - -package table - -import ( - "fmt" - "reflect" - - "github.com/onsi/ginkgo/internal/codelocation" - "github.com/onsi/ginkgo/internal/global" - "github.com/onsi/ginkgo/types" -) - -/* -DescribeTable describes a table-driven test. - -For example: - - DescribeTable("a simple table", - func(x int, y int, expected bool) { - Ω(x > y).Should(Equal(expected)) - }, - Entry("x > y", 1, 0, true), - Entry("x == y", 0, 0, false), - Entry("x < y", 0, 1, false), - ) - -The first argument to `DescribeTable` is a string description. -The second argument is a function that will be run for each table entry. Your assertions go here - the function is equivalent to a Ginkgo It. -The subsequent arguments must be of type `TableEntry`. We recommend using the `Entry` convenience constructors. - -The `Entry` constructor takes a string description followed by an arbitrary set of parameters. These parameters are passed into your function. - -Under the hood, `DescribeTable` simply generates a new Ginkgo `Describe`. Each `Entry` is turned into an `It` within the `Describe`. - -It's important to understand that the `Describe`s and `It`s are generated at evaluation time (i.e. when Ginkgo constructs the tree of tests and before the tests run). - -Individual Entries can be focused (with FEntry) or marked pending (with PEntry or XEntry). In addition, the entire table can be focused or marked pending with FDescribeTable and PDescribeTable/XDescribeTable. - -A description function can be passed to Entry in place of the description. The function is then fed with the entry parameters to generate the description of the It corresponding to that particular Entry. - -For example: - - describe := func(desc string) func(int, int, bool) string { - return func(x, y int, expected bool) string { - return fmt.Sprintf("%s x=%d y=%d expected:%t", desc, x, y, expected) - } - } - - DescribeTable("a simple table", - func(x int, y int, expected bool) { - Ω(x > y).Should(Equal(expected)) - }, - Entry(describe("x > y"), 1, 0, true), - Entry(describe("x == y"), 0, 0, false), - Entry(describe("x < y"), 0, 1, false), - ) -*/ -func DescribeTable(description string, itBody interface{}, entries ...TableEntry) bool { - describeTable(description, itBody, entries, types.FlagTypeNone) - return true -} - -/* -You can focus a table with `FDescribeTable`. This is equivalent to `FDescribe`. -*/ -func FDescribeTable(description string, itBody interface{}, entries ...TableEntry) bool { - describeTable(description, itBody, entries, types.FlagTypeFocused) - return true -} - -/* -You can mark a table as pending with `PDescribeTable`. This is equivalent to `PDescribe`. -*/ -func PDescribeTable(description string, itBody interface{}, entries ...TableEntry) bool { - describeTable(description, itBody, entries, types.FlagTypePending) - return true -} - -/* -You can mark a table as pending with `XDescribeTable`. This is equivalent to `XDescribe`. -*/ -func XDescribeTable(description string, itBody interface{}, entries ...TableEntry) bool { - describeTable(description, itBody, entries, types.FlagTypePending) - return true -} - -func describeTable(description string, itBody interface{}, entries []TableEntry, flag types.FlagType) { - itBodyValue := reflect.ValueOf(itBody) - if itBodyValue.Kind() != reflect.Func { - panic(fmt.Sprintf("DescribeTable expects a function, got %#v", itBody)) - } - - global.Suite.PushContainerNode( - description, - func() { - for _, entry := range entries { - entry.generateIt(itBodyValue) - } - }, - flag, - codelocation.New(2), - ) -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/extensions/table/table_entry.go b/go-controller/vendor/github.com/onsi/ginkgo/extensions/table/table_entry.go deleted file mode 100644 index 4d9c237ad7..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/extensions/table/table_entry.go +++ /dev/null @@ -1,129 +0,0 @@ -package table - -import ( - "fmt" - "reflect" - - "github.com/onsi/ginkgo/internal/codelocation" - "github.com/onsi/ginkgo/internal/global" - "github.com/onsi/ginkgo/types" -) - -/* -TableEntry represents an entry in a table test. You generally use the `Entry` constructor. -*/ -type TableEntry struct { - Description interface{} - Parameters []interface{} - Pending bool - Focused bool - codeLocation types.CodeLocation -} - -func (t TableEntry) generateIt(itBody reflect.Value) { - var description string - descriptionValue := reflect.ValueOf(t.Description) - switch descriptionValue.Kind() { - case reflect.String: - description = descriptionValue.String() - case reflect.Func: - values := castParameters(descriptionValue, t.Parameters) - res := descriptionValue.Call(values) - if len(res) != 1 { - panic(fmt.Sprintf("The describe function should return only a value, returned %d", len(res))) - } - if res[0].Kind() != reflect.String { - panic(fmt.Sprintf("The describe function should return a string, returned %#v", res[0])) - } - description = res[0].String() - default: - panic(fmt.Sprintf("Description can either be a string or a function, got %#v", descriptionValue)) - } - - if t.Pending { - global.Suite.PushItNode(description, func() {}, types.FlagTypePending, t.codeLocation, 0) - return - } - - values := castParameters(itBody, t.Parameters) - body := func() { - itBody.Call(values) - } - - if t.Focused { - global.Suite.PushItNode(description, body, types.FlagTypeFocused, t.codeLocation, global.DefaultTimeout) - } else { - global.Suite.PushItNode(description, body, types.FlagTypeNone, t.codeLocation, global.DefaultTimeout) - } -} - -func castParameters(function reflect.Value, parameters []interface{}) []reflect.Value { - res := make([]reflect.Value, len(parameters)) - funcType := function.Type() - for i, param := range parameters { - if param == nil { - inType := funcType.In(i) - res[i] = reflect.Zero(inType) - } else { - res[i] = reflect.ValueOf(param) - } - } - return res -} - -/* -Entry constructs a TableEntry. - -The first argument is a required description (this becomes the content of the generated Ginkgo `It`). -Subsequent parameters are saved off and sent to the callback passed in to `DescribeTable`. - -Each Entry ends up generating an individual Ginkgo It. -*/ -func Entry(description interface{}, parameters ...interface{}) TableEntry { - return TableEntry{ - Description: description, - Parameters: parameters, - Pending: false, - Focused: false, - codeLocation: codelocation.New(1), - } -} - -/* -You can focus a particular entry with FEntry. This is equivalent to FIt. -*/ -func FEntry(description interface{}, parameters ...interface{}) TableEntry { - return TableEntry{ - Description: description, - Parameters: parameters, - Pending: false, - Focused: true, - codeLocation: codelocation.New(1), - } -} - -/* -You can mark a particular entry as pending with PEntry. This is equivalent to PIt. -*/ -func PEntry(description interface{}, parameters ...interface{}) TableEntry { - return TableEntry{ - Description: description, - Parameters: parameters, - Pending: true, - Focused: false, - codeLocation: codelocation.New(1), - } -} - -/* -You can mark a particular entry as pending with XEntry. This is equivalent to XIt. -*/ -func XEntry(description interface{}, parameters ...interface{}) TableEntry { - return TableEntry{ - Description: description, - Parameters: parameters, - Pending: true, - Focused: false, - codeLocation: codelocation.New(1), - } -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/formatter/formatter.go b/go-controller/vendor/github.com/onsi/ginkgo/formatter/formatter.go deleted file mode 100644 index 30d7cbe129..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/formatter/formatter.go +++ /dev/null @@ -1,190 +0,0 @@ -package formatter - -import ( - "fmt" - "regexp" - "strings" -) - -const COLS = 80 - -type ColorMode uint8 - -const ( - ColorModeNone ColorMode = iota - ColorModeTerminal - ColorModePassthrough -) - -var SingletonFormatter = New(ColorModeTerminal) - -func F(format string, args ...interface{}) string { - return SingletonFormatter.F(format, args...) -} - -func Fi(indentation uint, format string, args ...interface{}) string { - return SingletonFormatter.Fi(indentation, format, args...) -} - -func Fiw(indentation uint, maxWidth uint, format string, args ...interface{}) string { - return SingletonFormatter.Fiw(indentation, maxWidth, format, args...) -} - -type Formatter struct { - ColorMode ColorMode - colors map[string]string - styleRe *regexp.Regexp - preserveColorStylingTags bool -} - -func NewWithNoColorBool(noColor bool) Formatter { - if noColor { - return New(ColorModeNone) - } - return New(ColorModeTerminal) -} - -func New(colorMode ColorMode) Formatter { - f := Formatter{ - ColorMode: colorMode, - colors: map[string]string{ - "/": "\x1b[0m", - "bold": "\x1b[1m", - "underline": "\x1b[4m", - - "red": "\x1b[38;5;9m", - "orange": "\x1b[38;5;214m", - "coral": "\x1b[38;5;204m", - "magenta": "\x1b[38;5;13m", - "green": "\x1b[38;5;10m", - "dark-green": "\x1b[38;5;28m", - "yellow": "\x1b[38;5;11m", - "light-yellow": "\x1b[38;5;228m", - "cyan": "\x1b[38;5;14m", - "gray": "\x1b[38;5;243m", - "light-gray": "\x1b[38;5;246m", - "blue": "\x1b[38;5;12m", - }, - } - colors := []string{} - for color := range f.colors { - colors = append(colors, color) - } - f.styleRe = regexp.MustCompile("{{(" + strings.Join(colors, "|") + ")}}") - return f -} - -func (f Formatter) F(format string, args ...interface{}) string { - return f.Fi(0, format, args...) -} - -func (f Formatter) Fi(indentation uint, format string, args ...interface{}) string { - return f.Fiw(indentation, 0, format, args...) -} - -func (f Formatter) Fiw(indentation uint, maxWidth uint, format string, args ...interface{}) string { - out := fmt.Sprintf(f.style(format), args...) - - if indentation == 0 && maxWidth == 0 { - return out - } - - lines := strings.Split(out, "\n") - - if maxWidth != 0 { - outLines := []string{} - - maxWidth = maxWidth - indentation*2 - for _, line := range lines { - if f.length(line) <= maxWidth { - outLines = append(outLines, line) - continue - } - outWords := []string{} - length := uint(0) - words := strings.Split(line, " ") - for _, word := range words { - wordLength := f.length(word) - if length+wordLength <= maxWidth { - length += wordLength - outWords = append(outWords, word) - continue - } - outLines = append(outLines, strings.Join(outWords, " ")) - outWords = []string{word} - length = wordLength - } - if len(outWords) > 0 { - outLines = append(outLines, strings.Join(outWords, " ")) - } - } - - lines = outLines - } - - if indentation == 0 { - return strings.Join(lines, "\n") - } - - padding := strings.Repeat(" ", int(indentation)) - for i := range lines { - if lines[i] != "" { - lines[i] = padding + lines[i] - } - } - - return strings.Join(lines, "\n") -} - -func (f Formatter) length(styled string) uint { - n := uint(0) - inStyle := false - for _, b := range styled { - if inStyle { - if b == 'm' { - inStyle = false - } - continue - } - if b == '\x1b' { - inStyle = true - continue - } - n += 1 - } - return n -} - -func (f Formatter) CycleJoin(elements []string, joiner string, cycle []string) string { - if len(elements) == 0 { - return "" - } - n := len(cycle) - out := "" - for i, text := range elements { - out += cycle[i%n] + text - if i < len(elements)-1 { - out += joiner - } - } - out += "{{/}}" - return f.style(out) -} - -func (f Formatter) style(s string) string { - switch f.ColorMode { - case ColorModeNone: - return f.styleRe.ReplaceAllString(s, "") - case ColorModePassthrough: - return s - case ColorModeTerminal: - return f.styleRe.ReplaceAllStringFunc(s, func(match string) string { - if out, ok := f.colors[strings.Trim(match, "{}")]; ok { - return out - } - return match - }) - } - - return "" -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/codelocation/code_location.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/codelocation/code_location.go deleted file mode 100644 index aa89d6cba8..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/codelocation/code_location.go +++ /dev/null @@ -1,48 +0,0 @@ -package codelocation - -import ( - "regexp" - "runtime" - "runtime/debug" - "strings" - - "github.com/onsi/ginkgo/types" -) - -func New(skip int) types.CodeLocation { - _, file, line, _ := runtime.Caller(skip + 1) - stackTrace := PruneStack(string(debug.Stack()), skip+1) - return types.CodeLocation{FileName: file, LineNumber: line, FullStackTrace: stackTrace} -} - -// PruneStack removes references to functions that are internal to Ginkgo -// and the Go runtime from a stack string and a certain number of stack entries -// at the beginning of the stack. The stack string has the format -// as returned by runtime/debug.Stack. The leading goroutine information is -// optional and always removed if present. Beware that runtime/debug.Stack -// adds itself as first entry, so typically skip must be >= 1 to remove that -// entry. -func PruneStack(fullStackTrace string, skip int) string { - stack := strings.Split(fullStackTrace, "\n") - // Ensure that the even entries are the method names and the - // the odd entries the source code information. - if len(stack) > 0 && strings.HasPrefix(stack[0], "goroutine ") { - // Ignore "goroutine 29 [running]:" line. - stack = stack[1:] - } - // The "+1" is for skipping over the initial entry, which is - // runtime/debug.Stack() itself. - if len(stack) > 2*(skip+1) { - stack = stack[2*(skip+1):] - } - prunedStack := []string{} - re := regexp.MustCompile(`\/ginkgo\/|\/pkg\/testing\/|\/pkg\/runtime\/`) - for i := 0; i < len(stack)/2; i++ { - // We filter out based on the source code file name. - if !re.Match([]byte(stack[i*2+1])) { - prunedStack = append(prunedStack, stack[i*2]) - prunedStack = append(prunedStack, stack[i*2+1]) - } - } - return strings.Join(prunedStack, "\n") -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/containernode/container_node.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/containernode/container_node.go deleted file mode 100644 index 0737746dcf..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/containernode/container_node.go +++ /dev/null @@ -1,151 +0,0 @@ -package containernode - -import ( - "math/rand" - "sort" - - "github.com/onsi/ginkgo/internal/leafnodes" - "github.com/onsi/ginkgo/types" -) - -type subjectOrContainerNode struct { - containerNode *ContainerNode - subjectNode leafnodes.SubjectNode -} - -func (n subjectOrContainerNode) text() string { - if n.containerNode != nil { - return n.containerNode.Text() - } else { - return n.subjectNode.Text() - } -} - -type CollatedNodes struct { - Containers []*ContainerNode - Subject leafnodes.SubjectNode -} - -type ContainerNode struct { - text string - flag types.FlagType - codeLocation types.CodeLocation - - setupNodes []leafnodes.BasicNode - subjectAndContainerNodes []subjectOrContainerNode -} - -func New(text string, flag types.FlagType, codeLocation types.CodeLocation) *ContainerNode { - return &ContainerNode{ - text: text, - flag: flag, - codeLocation: codeLocation, - } -} - -func (container *ContainerNode) Shuffle(r *rand.Rand) { - sort.Sort(container) - permutation := r.Perm(len(container.subjectAndContainerNodes)) - shuffledNodes := make([]subjectOrContainerNode, len(container.subjectAndContainerNodes)) - for i, j := range permutation { - shuffledNodes[i] = container.subjectAndContainerNodes[j] - } - container.subjectAndContainerNodes = shuffledNodes -} - -func (node *ContainerNode) BackPropagateProgrammaticFocus() bool { - if node.flag == types.FlagTypePending { - return false - } - - shouldUnfocus := false - for _, subjectOrContainerNode := range node.subjectAndContainerNodes { - if subjectOrContainerNode.containerNode != nil { - shouldUnfocus = subjectOrContainerNode.containerNode.BackPropagateProgrammaticFocus() || shouldUnfocus - } else { - shouldUnfocus = (subjectOrContainerNode.subjectNode.Flag() == types.FlagTypeFocused) || shouldUnfocus - } - } - - if shouldUnfocus { - if node.flag == types.FlagTypeFocused { - node.flag = types.FlagTypeNone - } - return true - } - - return node.flag == types.FlagTypeFocused -} - -func (node *ContainerNode) Collate() []CollatedNodes { - return node.collate([]*ContainerNode{}) -} - -func (node *ContainerNode) collate(enclosingContainers []*ContainerNode) []CollatedNodes { - collated := make([]CollatedNodes, 0) - - containers := make([]*ContainerNode, len(enclosingContainers)) - copy(containers, enclosingContainers) - containers = append(containers, node) - - for _, subjectOrContainer := range node.subjectAndContainerNodes { - if subjectOrContainer.containerNode != nil { - collated = append(collated, subjectOrContainer.containerNode.collate(containers)...) - } else { - collated = append(collated, CollatedNodes{ - Containers: containers, - Subject: subjectOrContainer.subjectNode, - }) - } - } - - return collated -} - -func (node *ContainerNode) PushContainerNode(container *ContainerNode) { - node.subjectAndContainerNodes = append(node.subjectAndContainerNodes, subjectOrContainerNode{containerNode: container}) -} - -func (node *ContainerNode) PushSubjectNode(subject leafnodes.SubjectNode) { - node.subjectAndContainerNodes = append(node.subjectAndContainerNodes, subjectOrContainerNode{subjectNode: subject}) -} - -func (node *ContainerNode) PushSetupNode(setupNode leafnodes.BasicNode) { - node.setupNodes = append(node.setupNodes, setupNode) -} - -func (node *ContainerNode) SetupNodesOfType(nodeType types.SpecComponentType) []leafnodes.BasicNode { - nodes := []leafnodes.BasicNode{} - for _, setupNode := range node.setupNodes { - if setupNode.Type() == nodeType { - nodes = append(nodes, setupNode) - } - } - return nodes -} - -func (node *ContainerNode) Text() string { - return node.text -} - -func (node *ContainerNode) CodeLocation() types.CodeLocation { - return node.codeLocation -} - -func (node *ContainerNode) Flag() types.FlagType { - return node.flag -} - -//sort.Interface - -func (node *ContainerNode) Len() int { - return len(node.subjectAndContainerNodes) -} - -func (node *ContainerNode) Less(i, j int) bool { - return node.subjectAndContainerNodes[i].text() < node.subjectAndContainerNodes[j].text() -} - -func (node *ContainerNode) Swap(i, j int) { - node.subjectAndContainerNodes[i], node.subjectAndContainerNodes[j] = node.subjectAndContainerNodes[j], node.subjectAndContainerNodes[i] -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/failer/failer.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/failer/failer.go deleted file mode 100644 index 678ea2514a..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/failer/failer.go +++ /dev/null @@ -1,92 +0,0 @@ -package failer - -import ( - "fmt" - "sync" - - "github.com/onsi/ginkgo/types" -) - -type Failer struct { - lock *sync.Mutex - failure types.SpecFailure - state types.SpecState -} - -func New() *Failer { - return &Failer{ - lock: &sync.Mutex{}, - state: types.SpecStatePassed, - } -} - -func (f *Failer) Panic(location types.CodeLocation, forwardedPanic interface{}) { - f.lock.Lock() - defer f.lock.Unlock() - - if f.state == types.SpecStatePassed { - f.state = types.SpecStatePanicked - f.failure = types.SpecFailure{ - Message: "Test Panicked", - Location: location, - ForwardedPanic: fmt.Sprintf("%v", forwardedPanic), - } - } -} - -func (f *Failer) Timeout(location types.CodeLocation) { - f.lock.Lock() - defer f.lock.Unlock() - - if f.state == types.SpecStatePassed { - f.state = types.SpecStateTimedOut - f.failure = types.SpecFailure{ - Message: "Timed out", - Location: location, - } - } -} - -func (f *Failer) Fail(message string, location types.CodeLocation) { - f.lock.Lock() - defer f.lock.Unlock() - - if f.state == types.SpecStatePassed { - f.state = types.SpecStateFailed - f.failure = types.SpecFailure{ - Message: message, - Location: location, - } - } -} - -func (f *Failer) Drain(componentType types.SpecComponentType, componentIndex int, componentCodeLocation types.CodeLocation) (types.SpecFailure, types.SpecState) { - f.lock.Lock() - defer f.lock.Unlock() - - failure := f.failure - outcome := f.state - if outcome != types.SpecStatePassed { - failure.ComponentType = componentType - failure.ComponentIndex = componentIndex - failure.ComponentCodeLocation = componentCodeLocation - } - - f.state = types.SpecStatePassed - f.failure = types.SpecFailure{} - - return failure, outcome -} - -func (f *Failer) Skip(message string, location types.CodeLocation) { - f.lock.Lock() - defer f.lock.Unlock() - - if f.state == types.SpecStatePassed { - f.state = types.SpecStateSkipped - f.failure = types.SpecFailure{ - Message: message, - Location: location, - } - } -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/global/init.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/global/init.go deleted file mode 100644 index 109f617a5e..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/global/init.go +++ /dev/null @@ -1,22 +0,0 @@ -package global - -import ( - "time" - - "github.com/onsi/ginkgo/internal/failer" - "github.com/onsi/ginkgo/internal/suite" -) - -const DefaultTimeout = time.Duration(1 * time.Second) - -var Suite *suite.Suite -var Failer *failer.Failer - -func init() { - InitializeGlobals() -} - -func InitializeGlobals() { - Failer = failer.New() - Suite = suite.New(Failer) -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/benchmarker.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/benchmarker.go deleted file mode 100644 index 393901e11c..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/benchmarker.go +++ /dev/null @@ -1,103 +0,0 @@ -package leafnodes - -import ( - "math" - "time" - - "sync" - - "github.com/onsi/ginkgo/types" -) - -type benchmarker struct { - mu sync.Mutex - measurements map[string]*types.SpecMeasurement - orderCounter int -} - -func newBenchmarker() *benchmarker { - return &benchmarker{ - measurements: make(map[string]*types.SpecMeasurement), - } -} - -func (b *benchmarker) Time(name string, body func(), info ...interface{}) (elapsedTime time.Duration) { - t := time.Now() - body() - elapsedTime = time.Since(t) - - b.mu.Lock() - defer b.mu.Unlock() - measurement := b.getMeasurement(name, "Fastest Time", "Slowest Time", "Average Time", "s", 3, info...) - measurement.Results = append(measurement.Results, elapsedTime.Seconds()) - - return -} - -func (b *benchmarker) RecordValue(name string, value float64, info ...interface{}) { - b.mu.Lock() - measurement := b.getMeasurement(name, "Smallest", " Largest", " Average", "", 3, info...) - defer b.mu.Unlock() - measurement.Results = append(measurement.Results, value) -} - -func (b *benchmarker) RecordValueWithPrecision(name string, value float64, units string, precision int, info ...interface{}) { - b.mu.Lock() - measurement := b.getMeasurement(name, "Smallest", " Largest", " Average", units, precision, info...) - defer b.mu.Unlock() - measurement.Results = append(measurement.Results, value) -} - -func (b *benchmarker) getMeasurement(name string, smallestLabel string, largestLabel string, averageLabel string, units string, precision int, info ...interface{}) *types.SpecMeasurement { - measurement, ok := b.measurements[name] - if !ok { - var computedInfo interface{} - computedInfo = nil - if len(info) > 0 { - computedInfo = info[0] - } - measurement = &types.SpecMeasurement{ - Name: name, - Info: computedInfo, - Order: b.orderCounter, - SmallestLabel: smallestLabel, - LargestLabel: largestLabel, - AverageLabel: averageLabel, - Units: units, - Precision: precision, - Results: make([]float64, 0), - } - b.measurements[name] = measurement - b.orderCounter++ - } - - return measurement -} - -func (b *benchmarker) measurementsReport() map[string]*types.SpecMeasurement { - b.mu.Lock() - defer b.mu.Unlock() - for _, measurement := range b.measurements { - measurement.Smallest = math.MaxFloat64 - measurement.Largest = -math.MaxFloat64 - sum := float64(0) - sumOfSquares := float64(0) - - for _, result := range measurement.Results { - if result > measurement.Largest { - measurement.Largest = result - } - if result < measurement.Smallest { - measurement.Smallest = result - } - sum += result - sumOfSquares += result * result - } - - n := float64(len(measurement.Results)) - measurement.Average = sum / n - measurement.StdDeviation = math.Sqrt(sumOfSquares/n - (sum/n)*(sum/n)) - } - - return b.measurements -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/interfaces.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/interfaces.go deleted file mode 100644 index 8c3902d601..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/interfaces.go +++ /dev/null @@ -1,19 +0,0 @@ -package leafnodes - -import ( - "github.com/onsi/ginkgo/types" -) - -type BasicNode interface { - Type() types.SpecComponentType - Run() (types.SpecState, types.SpecFailure) - CodeLocation() types.CodeLocation -} - -type SubjectNode interface { - BasicNode - - Text() string - Flag() types.FlagType - Samples() int -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/it_node.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/it_node.go deleted file mode 100644 index 6eded7b763..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/it_node.go +++ /dev/null @@ -1,47 +0,0 @@ -package leafnodes - -import ( - "time" - - "github.com/onsi/ginkgo/internal/failer" - "github.com/onsi/ginkgo/types" -) - -type ItNode struct { - runner *runner - - flag types.FlagType - text string -} - -func NewItNode(text string, body interface{}, flag types.FlagType, codeLocation types.CodeLocation, timeout time.Duration, failer *failer.Failer, componentIndex int) *ItNode { - return &ItNode{ - runner: newRunner(body, codeLocation, timeout, failer, types.SpecComponentTypeIt, componentIndex), - flag: flag, - text: text, - } -} - -func (node *ItNode) Run() (outcome types.SpecState, failure types.SpecFailure) { - return node.runner.run() -} - -func (node *ItNode) Type() types.SpecComponentType { - return types.SpecComponentTypeIt -} - -func (node *ItNode) Text() string { - return node.text -} - -func (node *ItNode) Flag() types.FlagType { - return node.flag -} - -func (node *ItNode) CodeLocation() types.CodeLocation { - return node.runner.codeLocation -} - -func (node *ItNode) Samples() int { - return 1 -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/measure_node.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/measure_node.go deleted file mode 100644 index 3ab9a6d552..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/measure_node.go +++ /dev/null @@ -1,62 +0,0 @@ -package leafnodes - -import ( - "reflect" - - "github.com/onsi/ginkgo/internal/failer" - "github.com/onsi/ginkgo/types" -) - -type MeasureNode struct { - runner *runner - - text string - flag types.FlagType - samples int - benchmarker *benchmarker -} - -func NewMeasureNode(text string, body interface{}, flag types.FlagType, codeLocation types.CodeLocation, samples int, failer *failer.Failer, componentIndex int) *MeasureNode { - benchmarker := newBenchmarker() - - wrappedBody := func() { - reflect.ValueOf(body).Call([]reflect.Value{reflect.ValueOf(benchmarker)}) - } - - return &MeasureNode{ - runner: newRunner(wrappedBody, codeLocation, 0, failer, types.SpecComponentTypeMeasure, componentIndex), - - text: text, - flag: flag, - samples: samples, - benchmarker: benchmarker, - } -} - -func (node *MeasureNode) Run() (outcome types.SpecState, failure types.SpecFailure) { - return node.runner.run() -} - -func (node *MeasureNode) MeasurementsReport() map[string]*types.SpecMeasurement { - return node.benchmarker.measurementsReport() -} - -func (node *MeasureNode) Type() types.SpecComponentType { - return types.SpecComponentTypeMeasure -} - -func (node *MeasureNode) Text() string { - return node.text -} - -func (node *MeasureNode) Flag() types.FlagType { - return node.flag -} - -func (node *MeasureNode) CodeLocation() types.CodeLocation { - return node.runner.codeLocation -} - -func (node *MeasureNode) Samples() int { - return node.samples -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/runner.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/runner.go deleted file mode 100644 index 16cb66c3e4..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/runner.go +++ /dev/null @@ -1,117 +0,0 @@ -package leafnodes - -import ( - "fmt" - "reflect" - "time" - - "github.com/onsi/ginkgo/internal/codelocation" - "github.com/onsi/ginkgo/internal/failer" - "github.com/onsi/ginkgo/types" -) - -type runner struct { - isAsync bool - asyncFunc func(chan<- interface{}) - syncFunc func() - codeLocation types.CodeLocation - timeoutThreshold time.Duration - nodeType types.SpecComponentType - componentIndex int - failer *failer.Failer -} - -func newRunner(body interface{}, codeLocation types.CodeLocation, timeout time.Duration, failer *failer.Failer, nodeType types.SpecComponentType, componentIndex int) *runner { - bodyType := reflect.TypeOf(body) - if bodyType.Kind() != reflect.Func { - panic(fmt.Sprintf("Expected a function but got something else at %v", codeLocation)) - } - - runner := &runner{ - codeLocation: codeLocation, - timeoutThreshold: timeout, - failer: failer, - nodeType: nodeType, - componentIndex: componentIndex, - } - - switch bodyType.NumIn() { - case 0: - runner.syncFunc = body.(func()) - return runner - case 1: - if !(bodyType.In(0).Kind() == reflect.Chan && bodyType.In(0).Elem().Kind() == reflect.Interface) { - panic(fmt.Sprintf("Must pass a Done channel to function at %v", codeLocation)) - } - - wrappedBody := func(done chan<- interface{}) { - bodyValue := reflect.ValueOf(body) - bodyValue.Call([]reflect.Value{reflect.ValueOf(done)}) - } - - runner.isAsync = true - runner.asyncFunc = wrappedBody - return runner - } - - panic(fmt.Sprintf("Too many arguments to function at %v", codeLocation)) -} - -func (r *runner) run() (outcome types.SpecState, failure types.SpecFailure) { - if r.isAsync { - return r.runAsync() - } else { - return r.runSync() - } -} - -func (r *runner) runAsync() (outcome types.SpecState, failure types.SpecFailure) { - done := make(chan interface{}, 1) - - go func() { - finished := false - - defer func() { - if e := recover(); e != nil || !finished { - r.failer.Panic(codelocation.New(2), e) - select { - case <-done: - break - default: - close(done) - } - } - }() - - r.asyncFunc(done) - finished = true - }() - - // If this goroutine gets no CPU time before the select block, - // the <-done case may complete even if the test took longer than the timeoutThreshold. - // This can cause flaky behaviour, but we haven't seen it in the wild. - select { - case <-done: - case <-time.After(r.timeoutThreshold): - r.failer.Timeout(r.codeLocation) - } - - failure, outcome = r.failer.Drain(r.nodeType, r.componentIndex, r.codeLocation) - return -} -func (r *runner) runSync() (outcome types.SpecState, failure types.SpecFailure) { - finished := false - - defer func() { - if e := recover(); e != nil || !finished { - r.failer.Panic(codelocation.New(2), e) - } - - failure, outcome = r.failer.Drain(r.nodeType, r.componentIndex, r.codeLocation) - }() - - r.syncFunc() - finished = true - - return -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/setup_nodes.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/setup_nodes.go deleted file mode 100644 index e3e9cb7c58..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/setup_nodes.go +++ /dev/null @@ -1,48 +0,0 @@ -package leafnodes - -import ( - "time" - - "github.com/onsi/ginkgo/internal/failer" - "github.com/onsi/ginkgo/types" -) - -type SetupNode struct { - runner *runner -} - -func (node *SetupNode) Run() (outcome types.SpecState, failure types.SpecFailure) { - return node.runner.run() -} - -func (node *SetupNode) Type() types.SpecComponentType { - return node.runner.nodeType -} - -func (node *SetupNode) CodeLocation() types.CodeLocation { - return node.runner.codeLocation -} - -func NewBeforeEachNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration, failer *failer.Failer, componentIndex int) *SetupNode { - return &SetupNode{ - runner: newRunner(body, codeLocation, timeout, failer, types.SpecComponentTypeBeforeEach, componentIndex), - } -} - -func NewAfterEachNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration, failer *failer.Failer, componentIndex int) *SetupNode { - return &SetupNode{ - runner: newRunner(body, codeLocation, timeout, failer, types.SpecComponentTypeAfterEach, componentIndex), - } -} - -func NewJustBeforeEachNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration, failer *failer.Failer, componentIndex int) *SetupNode { - return &SetupNode{ - runner: newRunner(body, codeLocation, timeout, failer, types.SpecComponentTypeJustBeforeEach, componentIndex), - } -} - -func NewJustAfterEachNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration, failer *failer.Failer, componentIndex int) *SetupNode { - return &SetupNode{ - runner: newRunner(body, codeLocation, timeout, failer, types.SpecComponentTypeJustAfterEach, componentIndex), - } -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/suite_nodes.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/suite_nodes.go deleted file mode 100644 index 80f16ed786..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/suite_nodes.go +++ /dev/null @@ -1,55 +0,0 @@ -package leafnodes - -import ( - "time" - - "github.com/onsi/ginkgo/internal/failer" - "github.com/onsi/ginkgo/types" -) - -type SuiteNode interface { - Run(parallelNode int, parallelTotal int, syncHost string) bool - Passed() bool - Summary() *types.SetupSummary -} - -type simpleSuiteNode struct { - runner *runner - outcome types.SpecState - failure types.SpecFailure - runTime time.Duration -} - -func (node *simpleSuiteNode) Run(parallelNode int, parallelTotal int, syncHost string) bool { - t := time.Now() - node.outcome, node.failure = node.runner.run() - node.runTime = time.Since(t) - - return node.outcome == types.SpecStatePassed -} - -func (node *simpleSuiteNode) Passed() bool { - return node.outcome == types.SpecStatePassed -} - -func (node *simpleSuiteNode) Summary() *types.SetupSummary { - return &types.SetupSummary{ - ComponentType: node.runner.nodeType, - CodeLocation: node.runner.codeLocation, - State: node.outcome, - RunTime: node.runTime, - Failure: node.failure, - } -} - -func NewBeforeSuiteNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration, failer *failer.Failer) SuiteNode { - return &simpleSuiteNode{ - runner: newRunner(body, codeLocation, timeout, failer, types.SpecComponentTypeBeforeSuite, 0), - } -} - -func NewAfterSuiteNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration, failer *failer.Failer) SuiteNode { - return &simpleSuiteNode{ - runner: newRunner(body, codeLocation, timeout, failer, types.SpecComponentTypeAfterSuite, 0), - } -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/synchronized_after_suite_node.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/synchronized_after_suite_node.go deleted file mode 100644 index a721d0cf7f..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/synchronized_after_suite_node.go +++ /dev/null @@ -1,90 +0,0 @@ -package leafnodes - -import ( - "encoding/json" - "io/ioutil" - "net/http" - "time" - - "github.com/onsi/ginkgo/internal/failer" - "github.com/onsi/ginkgo/types" -) - -type synchronizedAfterSuiteNode struct { - runnerA *runner - runnerB *runner - - outcome types.SpecState - failure types.SpecFailure - runTime time.Duration -} - -func NewSynchronizedAfterSuiteNode(bodyA interface{}, bodyB interface{}, codeLocation types.CodeLocation, timeout time.Duration, failer *failer.Failer) SuiteNode { - return &synchronizedAfterSuiteNode{ - runnerA: newRunner(bodyA, codeLocation, timeout, failer, types.SpecComponentTypeAfterSuite, 0), - runnerB: newRunner(bodyB, codeLocation, timeout, failer, types.SpecComponentTypeAfterSuite, 0), - } -} - -func (node *synchronizedAfterSuiteNode) Run(parallelNode int, parallelTotal int, syncHost string) bool { - node.outcome, node.failure = node.runnerA.run() - - if parallelNode == 1 { - if parallelTotal > 1 { - node.waitUntilOtherNodesAreDone(syncHost) - } - - outcome, failure := node.runnerB.run() - - if node.outcome == types.SpecStatePassed { - node.outcome, node.failure = outcome, failure - } - } - - return node.outcome == types.SpecStatePassed -} - -func (node *synchronizedAfterSuiteNode) Passed() bool { - return node.outcome == types.SpecStatePassed -} - -func (node *synchronizedAfterSuiteNode) Summary() *types.SetupSummary { - return &types.SetupSummary{ - ComponentType: node.runnerA.nodeType, - CodeLocation: node.runnerA.codeLocation, - State: node.outcome, - RunTime: node.runTime, - Failure: node.failure, - } -} - -func (node *synchronizedAfterSuiteNode) waitUntilOtherNodesAreDone(syncHost string) { - for { - if node.canRun(syncHost) { - return - } - - time.Sleep(50 * time.Millisecond) - } -} - -func (node *synchronizedAfterSuiteNode) canRun(syncHost string) bool { - resp, err := http.Get(syncHost + "/RemoteAfterSuiteData") - if err != nil || resp.StatusCode != http.StatusOK { - return false - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return false - } - resp.Body.Close() - - afterSuiteData := types.RemoteAfterSuiteData{} - err = json.Unmarshal(body, &afterSuiteData) - if err != nil { - return false - } - - return afterSuiteData.CanRun -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/synchronized_before_suite_node.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/synchronized_before_suite_node.go deleted file mode 100644 index d5c8893194..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/leafnodes/synchronized_before_suite_node.go +++ /dev/null @@ -1,181 +0,0 @@ -package leafnodes - -import ( - "bytes" - "encoding/json" - "io/ioutil" - "net/http" - "reflect" - "time" - - "github.com/onsi/ginkgo/internal/failer" - "github.com/onsi/ginkgo/types" -) - -type synchronizedBeforeSuiteNode struct { - runnerA *runner - runnerB *runner - - data []byte - - outcome types.SpecState - failure types.SpecFailure - runTime time.Duration -} - -func NewSynchronizedBeforeSuiteNode(bodyA interface{}, bodyB interface{}, codeLocation types.CodeLocation, timeout time.Duration, failer *failer.Failer) SuiteNode { - node := &synchronizedBeforeSuiteNode{} - - node.runnerA = newRunner(node.wrapA(bodyA), codeLocation, timeout, failer, types.SpecComponentTypeBeforeSuite, 0) - node.runnerB = newRunner(node.wrapB(bodyB), codeLocation, timeout, failer, types.SpecComponentTypeBeforeSuite, 0) - - return node -} - -func (node *synchronizedBeforeSuiteNode) Run(parallelNode int, parallelTotal int, syncHost string) bool { - t := time.Now() - defer func() { - node.runTime = time.Since(t) - }() - - if parallelNode == 1 { - node.outcome, node.failure = node.runA(parallelTotal, syncHost) - } else { - node.outcome, node.failure = node.waitForA(syncHost) - } - - if node.outcome != types.SpecStatePassed { - return false - } - node.outcome, node.failure = node.runnerB.run() - - return node.outcome == types.SpecStatePassed -} - -func (node *synchronizedBeforeSuiteNode) runA(parallelTotal int, syncHost string) (types.SpecState, types.SpecFailure) { - outcome, failure := node.runnerA.run() - - if parallelTotal > 1 { - state := types.RemoteBeforeSuiteStatePassed - if outcome != types.SpecStatePassed { - state = types.RemoteBeforeSuiteStateFailed - } - json := (types.RemoteBeforeSuiteData{ - Data: node.data, - State: state, - }).ToJSON() - http.Post(syncHost+"/BeforeSuiteState", "application/json", bytes.NewBuffer(json)) - } - - return outcome, failure -} - -func (node *synchronizedBeforeSuiteNode) waitForA(syncHost string) (types.SpecState, types.SpecFailure) { - failure := func(message string) types.SpecFailure { - return types.SpecFailure{ - Message: message, - Location: node.runnerA.codeLocation, - ComponentType: node.runnerA.nodeType, - ComponentIndex: node.runnerA.componentIndex, - ComponentCodeLocation: node.runnerA.codeLocation, - } - } - for { - resp, err := http.Get(syncHost + "/BeforeSuiteState") - if err != nil || resp.StatusCode != http.StatusOK { - return types.SpecStateFailed, failure("Failed to fetch BeforeSuite state") - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return types.SpecStateFailed, failure("Failed to read BeforeSuite state") - } - resp.Body.Close() - - beforeSuiteData := types.RemoteBeforeSuiteData{} - err = json.Unmarshal(body, &beforeSuiteData) - if err != nil { - return types.SpecStateFailed, failure("Failed to decode BeforeSuite state") - } - - switch beforeSuiteData.State { - case types.RemoteBeforeSuiteStatePassed: - node.data = beforeSuiteData.Data - return types.SpecStatePassed, types.SpecFailure{} - case types.RemoteBeforeSuiteStateFailed: - return types.SpecStateFailed, failure("BeforeSuite on Node 1 failed") - case types.RemoteBeforeSuiteStateDisappeared: - return types.SpecStateFailed, failure("Node 1 disappeared before completing BeforeSuite") - } - - time.Sleep(50 * time.Millisecond) - } -} - -func (node *synchronizedBeforeSuiteNode) Passed() bool { - return node.outcome == types.SpecStatePassed -} - -func (node *synchronizedBeforeSuiteNode) Summary() *types.SetupSummary { - return &types.SetupSummary{ - ComponentType: node.runnerA.nodeType, - CodeLocation: node.runnerA.codeLocation, - State: node.outcome, - RunTime: node.runTime, - Failure: node.failure, - } -} - -func (node *synchronizedBeforeSuiteNode) wrapA(bodyA interface{}) interface{} { - typeA := reflect.TypeOf(bodyA) - if typeA.Kind() != reflect.Func { - panic("SynchronizedBeforeSuite expects a function as its first argument") - } - - takesNothing := typeA.NumIn() == 0 - takesADoneChannel := typeA.NumIn() == 1 && typeA.In(0).Kind() == reflect.Chan && typeA.In(0).Elem().Kind() == reflect.Interface - returnsBytes := typeA.NumOut() == 1 && typeA.Out(0).Kind() == reflect.Slice && typeA.Out(0).Elem().Kind() == reflect.Uint8 - - if !((takesNothing || takesADoneChannel) && returnsBytes) { - panic("SynchronizedBeforeSuite's first argument should be a function that returns []byte and either takes no arguments or takes a Done channel.") - } - - if takesADoneChannel { - return func(done chan<- interface{}) { - out := reflect.ValueOf(bodyA).Call([]reflect.Value{reflect.ValueOf(done)}) - node.data = out[0].Interface().([]byte) - } - } - - return func() { - out := reflect.ValueOf(bodyA).Call([]reflect.Value{}) - node.data = out[0].Interface().([]byte) - } -} - -func (node *synchronizedBeforeSuiteNode) wrapB(bodyB interface{}) interface{} { - typeB := reflect.TypeOf(bodyB) - if typeB.Kind() != reflect.Func { - panic("SynchronizedBeforeSuite expects a function as its second argument") - } - - returnsNothing := typeB.NumOut() == 0 - takesBytesOnly := typeB.NumIn() == 1 && typeB.In(0).Kind() == reflect.Slice && typeB.In(0).Elem().Kind() == reflect.Uint8 - takesBytesAndDone := typeB.NumIn() == 2 && - typeB.In(0).Kind() == reflect.Slice && typeB.In(0).Elem().Kind() == reflect.Uint8 && - typeB.In(1).Kind() == reflect.Chan && typeB.In(1).Elem().Kind() == reflect.Interface - - if !((takesBytesOnly || takesBytesAndDone) && returnsNothing) { - panic("SynchronizedBeforeSuite's second argument should be a function that returns nothing and either takes []byte or ([]byte, Done)") - } - - if takesBytesAndDone { - return func(done chan<- interface{}) { - reflect.ValueOf(bodyB).Call([]reflect.Value{reflect.ValueOf(node.data), reflect.ValueOf(done)}) - } - } - - return func() { - reflect.ValueOf(bodyB).Call([]reflect.Value{reflect.ValueOf(node.data)}) - } -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec/spec.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/spec/spec.go deleted file mode 100644 index 6eef40a0e0..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec/spec.go +++ /dev/null @@ -1,247 +0,0 @@ -package spec - -import ( - "fmt" - "io" - "time" - - "sync" - - "github.com/onsi/ginkgo/internal/containernode" - "github.com/onsi/ginkgo/internal/leafnodes" - "github.com/onsi/ginkgo/types" -) - -type Spec struct { - subject leafnodes.SubjectNode - focused bool - announceProgress bool - - containers []*containernode.ContainerNode - - state types.SpecState - runTime time.Duration - startTime time.Time - failure types.SpecFailure - previousFailures bool - - stateMutex *sync.Mutex -} - -func New(subject leafnodes.SubjectNode, containers []*containernode.ContainerNode, announceProgress bool) *Spec { - spec := &Spec{ - subject: subject, - containers: containers, - focused: subject.Flag() == types.FlagTypeFocused, - announceProgress: announceProgress, - stateMutex: &sync.Mutex{}, - } - - spec.processFlag(subject.Flag()) - for i := len(containers) - 1; i >= 0; i-- { - spec.processFlag(containers[i].Flag()) - } - - return spec -} - -func (spec *Spec) processFlag(flag types.FlagType) { - if flag == types.FlagTypeFocused { - spec.focused = true - } else if flag == types.FlagTypePending { - spec.setState(types.SpecStatePending) - } -} - -func (spec *Spec) Skip() { - spec.setState(types.SpecStateSkipped) -} - -func (spec *Spec) Failed() bool { - return spec.getState() == types.SpecStateFailed || spec.getState() == types.SpecStatePanicked || spec.getState() == types.SpecStateTimedOut -} - -func (spec *Spec) Passed() bool { - return spec.getState() == types.SpecStatePassed -} - -func (spec *Spec) Flaked() bool { - return spec.getState() == types.SpecStatePassed && spec.previousFailures -} - -func (spec *Spec) Pending() bool { - return spec.getState() == types.SpecStatePending -} - -func (spec *Spec) Skipped() bool { - return spec.getState() == types.SpecStateSkipped -} - -func (spec *Spec) Focused() bool { - return spec.focused -} - -func (spec *Spec) IsMeasurement() bool { - return spec.subject.Type() == types.SpecComponentTypeMeasure -} - -func (spec *Spec) Summary(suiteID string) *types.SpecSummary { - componentTexts := make([]string, len(spec.containers)+1) - componentCodeLocations := make([]types.CodeLocation, len(spec.containers)+1) - - for i, container := range spec.containers { - componentTexts[i] = container.Text() - componentCodeLocations[i] = container.CodeLocation() - } - - componentTexts[len(spec.containers)] = spec.subject.Text() - componentCodeLocations[len(spec.containers)] = spec.subject.CodeLocation() - - runTime := spec.runTime - if runTime == 0 && !spec.startTime.IsZero() { - runTime = time.Since(spec.startTime) - } - - return &types.SpecSummary{ - IsMeasurement: spec.IsMeasurement(), - NumberOfSamples: spec.subject.Samples(), - ComponentTexts: componentTexts, - ComponentCodeLocations: componentCodeLocations, - State: spec.getState(), - RunTime: runTime, - Failure: spec.failure, - Measurements: spec.measurementsReport(), - SuiteID: suiteID, - } -} - -func (spec *Spec) ConcatenatedString() string { - s := "" - for _, container := range spec.containers { - s += container.Text() + " " - } - - return s + spec.subject.Text() -} - -func (spec *Spec) Run(writer io.Writer) { - if spec.getState() == types.SpecStateFailed { - spec.previousFailures = true - } - - spec.startTime = time.Now() - defer func() { - spec.runTime = time.Since(spec.startTime) - }() - - for sample := 0; sample < spec.subject.Samples(); sample++ { - spec.runSample(sample, writer) - - if spec.getState() != types.SpecStatePassed { - return - } - } -} - -func (spec *Spec) getState() types.SpecState { - spec.stateMutex.Lock() - defer spec.stateMutex.Unlock() - return spec.state -} - -func (spec *Spec) setState(state types.SpecState) { - spec.stateMutex.Lock() - defer spec.stateMutex.Unlock() - spec.state = state -} - -func (spec *Spec) runSample(sample int, writer io.Writer) { - spec.setState(types.SpecStatePassed) - spec.failure = types.SpecFailure{} - innerMostContainerIndexToUnwind := -1 - - defer func() { - for i := innerMostContainerIndexToUnwind; i >= 0; i-- { - container := spec.containers[i] - for _, justAfterEach := range container.SetupNodesOfType(types.SpecComponentTypeJustAfterEach) { - spec.announceSetupNode(writer, "JustAfterEach", container, justAfterEach) - justAfterEachState, justAfterEachFailure := justAfterEach.Run() - if justAfterEachState != types.SpecStatePassed && spec.state == types.SpecStatePassed { - spec.state = justAfterEachState - spec.failure = justAfterEachFailure - } - } - } - - for i := innerMostContainerIndexToUnwind; i >= 0; i-- { - container := spec.containers[i] - for _, afterEach := range container.SetupNodesOfType(types.SpecComponentTypeAfterEach) { - spec.announceSetupNode(writer, "AfterEach", container, afterEach) - afterEachState, afterEachFailure := afterEach.Run() - if afterEachState != types.SpecStatePassed && spec.getState() == types.SpecStatePassed { - spec.setState(afterEachState) - spec.failure = afterEachFailure - } - } - } - }() - - for i, container := range spec.containers { - innerMostContainerIndexToUnwind = i - for _, beforeEach := range container.SetupNodesOfType(types.SpecComponentTypeBeforeEach) { - spec.announceSetupNode(writer, "BeforeEach", container, beforeEach) - s, f := beforeEach.Run() - spec.failure = f - spec.setState(s) - if spec.getState() != types.SpecStatePassed { - return - } - } - } - - for _, container := range spec.containers { - for _, justBeforeEach := range container.SetupNodesOfType(types.SpecComponentTypeJustBeforeEach) { - spec.announceSetupNode(writer, "JustBeforeEach", container, justBeforeEach) - s, f := justBeforeEach.Run() - spec.failure = f - spec.setState(s) - if spec.getState() != types.SpecStatePassed { - return - } - } - } - - spec.announceSubject(writer, spec.subject) - s, f := spec.subject.Run() - spec.failure = f - spec.setState(s) -} - -func (spec *Spec) announceSetupNode(writer io.Writer, nodeType string, container *containernode.ContainerNode, setupNode leafnodes.BasicNode) { - if spec.announceProgress { - s := fmt.Sprintf("[%s] %s\n %s\n", nodeType, container.Text(), setupNode.CodeLocation().String()) - writer.Write([]byte(s)) - } -} - -func (spec *Spec) announceSubject(writer io.Writer, subject leafnodes.SubjectNode) { - if spec.announceProgress { - nodeType := "" - switch subject.Type() { - case types.SpecComponentTypeIt: - nodeType = "It" - case types.SpecComponentTypeMeasure: - nodeType = "Measure" - } - s := fmt.Sprintf("[%s] %s\n %s\n", nodeType, subject.Text(), subject.CodeLocation().String()) - writer.Write([]byte(s)) - } -} - -func (spec *Spec) measurementsReport() map[string]*types.SpecMeasurement { - if !spec.IsMeasurement() || spec.Failed() { - return map[string]*types.SpecMeasurement{} - } - - return spec.subject.(*leafnodes.MeasureNode).MeasurementsReport() -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec/specs.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/spec/specs.go deleted file mode 100644 index 0a24139fb1..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec/specs.go +++ /dev/null @@ -1,144 +0,0 @@ -package spec - -import ( - "math/rand" - "regexp" - "sort" - "strings" -) - -type Specs struct { - specs []*Spec - names []string - - hasProgrammaticFocus bool - RegexScansFilePath bool -} - -func NewSpecs(specs []*Spec) *Specs { - names := make([]string, len(specs)) - for i, spec := range specs { - names[i] = spec.ConcatenatedString() - } - return &Specs{ - specs: specs, - names: names, - } -} - -func (e *Specs) Specs() []*Spec { - return e.specs -} - -func (e *Specs) HasProgrammaticFocus() bool { - return e.hasProgrammaticFocus -} - -func (e *Specs) Shuffle(r *rand.Rand) { - sort.Sort(e) - permutation := r.Perm(len(e.specs)) - shuffledSpecs := make([]*Spec, len(e.specs)) - names := make([]string, len(e.specs)) - for i, j := range permutation { - shuffledSpecs[i] = e.specs[j] - names[i] = e.names[j] - } - e.specs = shuffledSpecs - e.names = names -} - -func (e *Specs) ApplyFocus(description string, focus, skip []string) { - if len(focus)+len(skip) == 0 { - e.applyProgrammaticFocus() - } else { - e.applyRegExpFocusAndSkip(description, focus, skip) - } -} - -func (e *Specs) applyProgrammaticFocus() { - e.hasProgrammaticFocus = false - for _, spec := range e.specs { - if spec.Focused() && !spec.Pending() { - e.hasProgrammaticFocus = true - break - } - } - - if e.hasProgrammaticFocus { - for _, spec := range e.specs { - if !spec.Focused() { - spec.Skip() - } - } - } -} - -// toMatch returns a byte[] to be used by regex matchers. When adding new behaviours to the matching function, -// this is the place which we append to. -func (e *Specs) toMatch(description string, i int) []byte { - if i > len(e.names) { - return nil - } - if e.RegexScansFilePath { - return []byte( - description + " " + - e.names[i] + " " + - e.specs[i].subject.CodeLocation().FileName) - } else { - return []byte( - description + " " + - e.names[i]) - } -} - -func (e *Specs) applyRegExpFocusAndSkip(description string, focus, skip []string) { - var focusFilter, skipFilter *regexp.Regexp - if len(focus) > 0 { - focusFilter = regexp.MustCompile(strings.Join(focus, "|")) - } - if len(skip) > 0 { - skipFilter = regexp.MustCompile(strings.Join(skip, "|")) - } - - for i, spec := range e.specs { - matchesFocus := true - matchesSkip := false - - toMatch := e.toMatch(description, i) - - if focusFilter != nil { - matchesFocus = focusFilter.Match(toMatch) - } - - if skipFilter != nil { - matchesSkip = skipFilter.Match(toMatch) - } - - if !matchesFocus || matchesSkip { - spec.Skip() - } - } -} - -func (e *Specs) SkipMeasurements() { - for _, spec := range e.specs { - if spec.IsMeasurement() { - spec.Skip() - } - } -} - -//sort.Interface - -func (e *Specs) Len() int { - return len(e.specs) -} - -func (e *Specs) Less(i, j int) bool { - return e.names[i] < e.names[j] -} - -func (e *Specs) Swap(i, j int) { - e.names[i], e.names[j] = e.names[j], e.names[i] - e.specs[i], e.specs[j] = e.specs[j], e.specs[i] -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/index_computer.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/index_computer.go deleted file mode 100644 index 82272554aa..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/index_computer.go +++ /dev/null @@ -1,55 +0,0 @@ -package spec_iterator - -func ParallelizedIndexRange(length int, parallelTotal int, parallelNode int) (startIndex int, count int) { - if length == 0 { - return 0, 0 - } - - // We have more nodes than tests. Trivial case. - if parallelTotal >= length { - if parallelNode > length { - return 0, 0 - } else { - return parallelNode - 1, 1 - } - } - - // This is the minimum amount of tests that a node will be required to run - minTestsPerNode := length / parallelTotal - - // This is the maximum amount of tests that a node will be required to run - // The algorithm guarantees that this would be equal to at least the minimum amount - // and at most one more - maxTestsPerNode := minTestsPerNode - if length%parallelTotal != 0 { - maxTestsPerNode++ - } - - // Number of nodes that will have to run the maximum amount of tests per node - numMaxLoadNodes := length % parallelTotal - - // Number of nodes that precede the current node and will have to run the maximum amount of tests per node - var numPrecedingMaxLoadNodes int - if parallelNode > numMaxLoadNodes { - numPrecedingMaxLoadNodes = numMaxLoadNodes - } else { - numPrecedingMaxLoadNodes = parallelNode - 1 - } - - // Number of nodes that precede the current node and will have to run the minimum amount of tests per node - var numPrecedingMinLoadNodes int - if parallelNode <= numMaxLoadNodes { - numPrecedingMinLoadNodes = 0 - } else { - numPrecedingMinLoadNodes = parallelNode - numMaxLoadNodes - 1 - } - - // Evaluate the test start index and number of tests to run - startIndex = numPrecedingMaxLoadNodes*maxTestsPerNode + numPrecedingMinLoadNodes*minTestsPerNode - if parallelNode > numMaxLoadNodes { - count = minTestsPerNode - } else { - count = maxTestsPerNode - } - return -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/parallel_spec_iterator.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/parallel_spec_iterator.go deleted file mode 100644 index 99f548bca4..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/parallel_spec_iterator.go +++ /dev/null @@ -1,59 +0,0 @@ -package spec_iterator - -import ( - "encoding/json" - "fmt" - "net/http" - - "github.com/onsi/ginkgo/internal/spec" -) - -type ParallelIterator struct { - specs []*spec.Spec - host string - client *http.Client -} - -func NewParallelIterator(specs []*spec.Spec, host string) *ParallelIterator { - return &ParallelIterator{ - specs: specs, - host: host, - client: &http.Client{}, - } -} - -func (s *ParallelIterator) Next() (*spec.Spec, error) { - resp, err := s.client.Get(s.host + "/counter") - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode) - } - - var counter Counter - err = json.NewDecoder(resp.Body).Decode(&counter) - if err != nil { - return nil, err - } - - if counter.Index >= len(s.specs) { - return nil, ErrClosed - } - - return s.specs[counter.Index], nil -} - -func (s *ParallelIterator) NumberOfSpecsPriorToIteration() int { - return len(s.specs) -} - -func (s *ParallelIterator) NumberOfSpecsToProcessIfKnown() (int, bool) { - return -1, false -} - -func (s *ParallelIterator) NumberOfSpecsThatWillBeRunIfKnown() (int, bool) { - return -1, false -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/serial_spec_iterator.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/serial_spec_iterator.go deleted file mode 100644 index a51c93b8b6..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/serial_spec_iterator.go +++ /dev/null @@ -1,45 +0,0 @@ -package spec_iterator - -import ( - "github.com/onsi/ginkgo/internal/spec" -) - -type SerialIterator struct { - specs []*spec.Spec - index int -} - -func NewSerialIterator(specs []*spec.Spec) *SerialIterator { - return &SerialIterator{ - specs: specs, - index: 0, - } -} - -func (s *SerialIterator) Next() (*spec.Spec, error) { - if s.index >= len(s.specs) { - return nil, ErrClosed - } - - spec := s.specs[s.index] - s.index += 1 - return spec, nil -} - -func (s *SerialIterator) NumberOfSpecsPriorToIteration() int { - return len(s.specs) -} - -func (s *SerialIterator) NumberOfSpecsToProcessIfKnown() (int, bool) { - return len(s.specs), true -} - -func (s *SerialIterator) NumberOfSpecsThatWillBeRunIfKnown() (int, bool) { - count := 0 - for _, s := range s.specs { - if !s.Skipped() && !s.Pending() { - count += 1 - } - } - return count, true -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/sharded_parallel_spec_iterator.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/sharded_parallel_spec_iterator.go deleted file mode 100644 index ad4a3ea3c6..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/sharded_parallel_spec_iterator.go +++ /dev/null @@ -1,47 +0,0 @@ -package spec_iterator - -import "github.com/onsi/ginkgo/internal/spec" - -type ShardedParallelIterator struct { - specs []*spec.Spec - index int - maxIndex int -} - -func NewShardedParallelIterator(specs []*spec.Spec, total int, node int) *ShardedParallelIterator { - startIndex, count := ParallelizedIndexRange(len(specs), total, node) - - return &ShardedParallelIterator{ - specs: specs, - index: startIndex, - maxIndex: startIndex + count, - } -} - -func (s *ShardedParallelIterator) Next() (*spec.Spec, error) { - if s.index >= s.maxIndex { - return nil, ErrClosed - } - - spec := s.specs[s.index] - s.index += 1 - return spec, nil -} - -func (s *ShardedParallelIterator) NumberOfSpecsPriorToIteration() int { - return len(s.specs) -} - -func (s *ShardedParallelIterator) NumberOfSpecsToProcessIfKnown() (int, bool) { - return s.maxIndex - s.index, true -} - -func (s *ShardedParallelIterator) NumberOfSpecsThatWillBeRunIfKnown() (int, bool) { - count := 0 - for i := s.index; i < s.maxIndex; i += 1 { - if !s.specs[i].Skipped() && !s.specs[i].Pending() { - count += 1 - } - } - return count, true -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/spec_iterator.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/spec_iterator.go deleted file mode 100644 index 74bffad646..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/spec_iterator/spec_iterator.go +++ /dev/null @@ -1,20 +0,0 @@ -package spec_iterator - -import ( - "errors" - - "github.com/onsi/ginkgo/internal/spec" -) - -var ErrClosed = errors.New("no more specs to run") - -type SpecIterator interface { - Next() (*spec.Spec, error) - NumberOfSpecsPriorToIteration() int - NumberOfSpecsToProcessIfKnown() (int, bool) - NumberOfSpecsThatWillBeRunIfKnown() (int, bool) -} - -type Counter struct { - Index int `json:"index"` -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/specrunner/random_id.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/specrunner/random_id.go deleted file mode 100644 index a0b8b62d52..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/specrunner/random_id.go +++ /dev/null @@ -1,15 +0,0 @@ -package specrunner - -import ( - "crypto/rand" - "fmt" -) - -func randomID() string { - b := make([]byte, 8) - _, err := rand.Read(b) - if err != nil { - return "" - } - return fmt.Sprintf("%x-%x-%x-%x", b[0:2], b[2:4], b[4:6], b[6:8]) -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/specrunner/spec_runner.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/specrunner/spec_runner.go deleted file mode 100644 index c9a0a60d84..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/specrunner/spec_runner.go +++ /dev/null @@ -1,411 +0,0 @@ -package specrunner - -import ( - "fmt" - "os" - "os/signal" - "sync" - "syscall" - - "github.com/onsi/ginkgo/internal/spec_iterator" - - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/internal/leafnodes" - "github.com/onsi/ginkgo/internal/spec" - Writer "github.com/onsi/ginkgo/internal/writer" - "github.com/onsi/ginkgo/reporters" - "github.com/onsi/ginkgo/types" - - "time" -) - -type SpecRunner struct { - description string - beforeSuiteNode leafnodes.SuiteNode - iterator spec_iterator.SpecIterator - afterSuiteNode leafnodes.SuiteNode - reporters []reporters.Reporter - startTime time.Time - suiteID string - runningSpec *spec.Spec - writer Writer.WriterInterface - config config.GinkgoConfigType - interrupted bool - processedSpecs []*spec.Spec - lock *sync.Mutex -} - -func New(description string, beforeSuiteNode leafnodes.SuiteNode, iterator spec_iterator.SpecIterator, afterSuiteNode leafnodes.SuiteNode, reporters []reporters.Reporter, writer Writer.WriterInterface, config config.GinkgoConfigType) *SpecRunner { - return &SpecRunner{ - description: description, - beforeSuiteNode: beforeSuiteNode, - iterator: iterator, - afterSuiteNode: afterSuiteNode, - reporters: reporters, - writer: writer, - config: config, - suiteID: randomID(), - lock: &sync.Mutex{}, - } -} - -func (runner *SpecRunner) Run() bool { - if runner.config.DryRun { - runner.performDryRun() - return true - } - - runner.reportSuiteWillBegin() - signalRegistered := make(chan struct{}) - go runner.registerForInterrupts(signalRegistered) - <-signalRegistered - - suitePassed := runner.runBeforeSuite() - - if suitePassed { - suitePassed = runner.runSpecs() - } - - runner.blockForeverIfInterrupted() - - suitePassed = runner.runAfterSuite() && suitePassed - - runner.reportSuiteDidEnd(suitePassed) - - return suitePassed -} - -func (runner *SpecRunner) performDryRun() { - runner.reportSuiteWillBegin() - - if runner.beforeSuiteNode != nil { - summary := runner.beforeSuiteNode.Summary() - summary.State = types.SpecStatePassed - runner.reportBeforeSuite(summary) - } - - for { - spec, err := runner.iterator.Next() - if err == spec_iterator.ErrClosed { - break - } - if err != nil { - fmt.Println("failed to iterate over tests:\n" + err.Error()) - break - } - - runner.processedSpecs = append(runner.processedSpecs, spec) - - summary := spec.Summary(runner.suiteID) - runner.reportSpecWillRun(summary) - if summary.State == types.SpecStateInvalid { - summary.State = types.SpecStatePassed - } - runner.reportSpecDidComplete(summary, false) - } - - if runner.afterSuiteNode != nil { - summary := runner.afterSuiteNode.Summary() - summary.State = types.SpecStatePassed - runner.reportAfterSuite(summary) - } - - runner.reportSuiteDidEnd(true) -} - -func (runner *SpecRunner) runBeforeSuite() bool { - if runner.beforeSuiteNode == nil || runner.wasInterrupted() { - return true - } - - runner.writer.Truncate() - conf := runner.config - passed := runner.beforeSuiteNode.Run(conf.ParallelNode, conf.ParallelTotal, conf.SyncHost) - if !passed { - runner.writer.DumpOut() - } - runner.reportBeforeSuite(runner.beforeSuiteNode.Summary()) - return passed -} - -func (runner *SpecRunner) runAfterSuite() bool { - if runner.afterSuiteNode == nil { - return true - } - - runner.writer.Truncate() - conf := runner.config - passed := runner.afterSuiteNode.Run(conf.ParallelNode, conf.ParallelTotal, conf.SyncHost) - if !passed { - runner.writer.DumpOut() - } - runner.reportAfterSuite(runner.afterSuiteNode.Summary()) - return passed -} - -func (runner *SpecRunner) runSpecs() bool { - suiteFailed := false - skipRemainingSpecs := false - for { - spec, err := runner.iterator.Next() - if err == spec_iterator.ErrClosed { - break - } - if err != nil { - fmt.Println("failed to iterate over tests:\n" + err.Error()) - suiteFailed = true - break - } - - runner.processedSpecs = append(runner.processedSpecs, spec) - - if runner.wasInterrupted() { - break - } - if skipRemainingSpecs { - spec.Skip() - } - - if !spec.Skipped() && !spec.Pending() { - if passed := runner.runSpec(spec); !passed { - suiteFailed = true - } - } else if spec.Pending() && runner.config.FailOnPending { - runner.reportSpecWillRun(spec.Summary(runner.suiteID)) - suiteFailed = true - runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed()) - } else { - runner.reportSpecWillRun(spec.Summary(runner.suiteID)) - runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed()) - } - - if spec.Failed() && runner.config.FailFast { - skipRemainingSpecs = true - } - } - - return !suiteFailed -} - -func (runner *SpecRunner) runSpec(spec *spec.Spec) (passed bool) { - maxAttempts := 1 - if runner.config.FlakeAttempts > 0 { - // uninitialized configs count as 1 - maxAttempts = runner.config.FlakeAttempts - } - - for i := 0; i < maxAttempts; i++ { - runner.reportSpecWillRun(spec.Summary(runner.suiteID)) - runner.runningSpec = spec - spec.Run(runner.writer) - runner.runningSpec = nil - runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed()) - if !spec.Failed() { - return true - } - } - return false -} - -func (runner *SpecRunner) CurrentSpecSummary() (*types.SpecSummary, bool) { - if runner.runningSpec == nil { - return nil, false - } - - return runner.runningSpec.Summary(runner.suiteID), true -} - -func (runner *SpecRunner) registerForInterrupts(signalRegistered chan struct{}) { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - close(signalRegistered) - - <-c - signal.Stop(c) - runner.markInterrupted() - go runner.registerForHardInterrupts() - runner.writer.DumpOutWithHeader(` -Received interrupt. Emitting contents of GinkgoWriter... ---------------------------------------------------------- -`) - if runner.afterSuiteNode != nil { - fmt.Fprint(os.Stderr, ` ---------------------------------------------------------- -Received interrupt. Running AfterSuite... -^C again to terminate immediately -`) - runner.runAfterSuite() - } - runner.reportSuiteDidEnd(false) - os.Exit(1) -} - -func (runner *SpecRunner) registerForHardInterrupts() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - - <-c - fmt.Fprintln(os.Stderr, "\nReceived second interrupt. Shutting down.") - os.Exit(1) -} - -func (runner *SpecRunner) blockForeverIfInterrupted() { - runner.lock.Lock() - interrupted := runner.interrupted - runner.lock.Unlock() - - if interrupted { - select {} - } -} - -func (runner *SpecRunner) markInterrupted() { - runner.lock.Lock() - defer runner.lock.Unlock() - runner.interrupted = true -} - -func (runner *SpecRunner) wasInterrupted() bool { - runner.lock.Lock() - defer runner.lock.Unlock() - return runner.interrupted -} - -func (runner *SpecRunner) reportSuiteWillBegin() { - runner.startTime = time.Now() - summary := runner.suiteWillBeginSummary() - for _, reporter := range runner.reporters { - reporter.SpecSuiteWillBegin(runner.config, summary) - } -} - -func (runner *SpecRunner) reportBeforeSuite(summary *types.SetupSummary) { - for _, reporter := range runner.reporters { - reporter.BeforeSuiteDidRun(summary) - } -} - -func (runner *SpecRunner) reportAfterSuite(summary *types.SetupSummary) { - for _, reporter := range runner.reporters { - reporter.AfterSuiteDidRun(summary) - } -} - -func (runner *SpecRunner) reportSpecWillRun(summary *types.SpecSummary) { - runner.writer.Truncate() - - for _, reporter := range runner.reporters { - reporter.SpecWillRun(summary) - } -} - -func (runner *SpecRunner) reportSpecDidComplete(summary *types.SpecSummary, failed bool) { - if len(summary.CapturedOutput) == 0 { - summary.CapturedOutput = string(runner.writer.Bytes()) - } - for i := len(runner.reporters) - 1; i >= 1; i-- { - runner.reporters[i].SpecDidComplete(summary) - } - - if failed { - runner.writer.DumpOut() - } - - runner.reporters[0].SpecDidComplete(summary) -} - -func (runner *SpecRunner) reportSuiteDidEnd(success bool) { - summary := runner.suiteDidEndSummary(success) - summary.RunTime = time.Since(runner.startTime) - for _, reporter := range runner.reporters { - reporter.SpecSuiteDidEnd(summary) - } -} - -func (runner *SpecRunner) countSpecsThatRanSatisfying(filter func(ex *spec.Spec) bool) (count int) { - count = 0 - - for _, spec := range runner.processedSpecs { - if filter(spec) { - count++ - } - } - - return count -} - -func (runner *SpecRunner) suiteDidEndSummary(success bool) *types.SuiteSummary { - numberOfSpecsThatWillBeRun := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool { - return !ex.Skipped() && !ex.Pending() - }) - - numberOfPendingSpecs := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool { - return ex.Pending() - }) - - numberOfSkippedSpecs := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool { - return ex.Skipped() - }) - - numberOfPassedSpecs := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool { - return ex.Passed() - }) - - numberOfFlakedSpecs := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool { - return ex.Flaked() - }) - - numberOfFailedSpecs := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool { - return ex.Failed() - }) - - if runner.beforeSuiteNode != nil && !runner.beforeSuiteNode.Passed() && !runner.config.DryRun { - var known bool - numberOfSpecsThatWillBeRun, known = runner.iterator.NumberOfSpecsThatWillBeRunIfKnown() - if !known { - numberOfSpecsThatWillBeRun = runner.iterator.NumberOfSpecsPriorToIteration() - } - numberOfFailedSpecs = numberOfSpecsThatWillBeRun - } - - return &types.SuiteSummary{ - SuiteDescription: runner.description, - SuiteSucceeded: success, - SuiteID: runner.suiteID, - - NumberOfSpecsBeforeParallelization: runner.iterator.NumberOfSpecsPriorToIteration(), - NumberOfTotalSpecs: len(runner.processedSpecs), - NumberOfSpecsThatWillBeRun: numberOfSpecsThatWillBeRun, - NumberOfPendingSpecs: numberOfPendingSpecs, - NumberOfSkippedSpecs: numberOfSkippedSpecs, - NumberOfPassedSpecs: numberOfPassedSpecs, - NumberOfFailedSpecs: numberOfFailedSpecs, - NumberOfFlakedSpecs: numberOfFlakedSpecs, - } -} - -func (runner *SpecRunner) suiteWillBeginSummary() *types.SuiteSummary { - numTotal, known := runner.iterator.NumberOfSpecsToProcessIfKnown() - if !known { - numTotal = -1 - } - - numToRun, known := runner.iterator.NumberOfSpecsThatWillBeRunIfKnown() - if !known { - numToRun = -1 - } - - return &types.SuiteSummary{ - SuiteDescription: runner.description, - SuiteID: runner.suiteID, - - NumberOfSpecsBeforeParallelization: runner.iterator.NumberOfSpecsPriorToIteration(), - NumberOfTotalSpecs: numTotal, - NumberOfSpecsThatWillBeRun: numToRun, - NumberOfPendingSpecs: -1, - NumberOfSkippedSpecs: -1, - NumberOfPassedSpecs: -1, - NumberOfFailedSpecs: -1, - NumberOfFlakedSpecs: -1, - } -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/suite/suite.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/suite/suite.go deleted file mode 100644 index b4a83c432d..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/suite/suite.go +++ /dev/null @@ -1,227 +0,0 @@ -package suite - -import ( - "math/rand" - "net/http" - "time" - - "github.com/onsi/ginkgo/internal/spec_iterator" - - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/internal/containernode" - "github.com/onsi/ginkgo/internal/failer" - "github.com/onsi/ginkgo/internal/leafnodes" - "github.com/onsi/ginkgo/internal/spec" - "github.com/onsi/ginkgo/internal/specrunner" - "github.com/onsi/ginkgo/internal/writer" - "github.com/onsi/ginkgo/reporters" - "github.com/onsi/ginkgo/types" -) - -type ginkgoTestingT interface { - Fail() -} - -type deferredContainerNode struct { - text string - body func() - flag types.FlagType - codeLocation types.CodeLocation -} - -type Suite struct { - topLevelContainer *containernode.ContainerNode - currentContainer *containernode.ContainerNode - - deferredContainerNodes []deferredContainerNode - - containerIndex int - beforeSuiteNode leafnodes.SuiteNode - afterSuiteNode leafnodes.SuiteNode - runner *specrunner.SpecRunner - failer *failer.Failer - running bool - expandTopLevelNodes bool -} - -func New(failer *failer.Failer) *Suite { - topLevelContainer := containernode.New("[Top Level]", types.FlagTypeNone, types.CodeLocation{}) - - return &Suite{ - topLevelContainer: topLevelContainer, - currentContainer: topLevelContainer, - failer: failer, - containerIndex: 1, - deferredContainerNodes: []deferredContainerNode{}, - } -} - -func (suite *Suite) Run(t ginkgoTestingT, description string, reporters []reporters.Reporter, writer writer.WriterInterface, config config.GinkgoConfigType) (bool, bool) { - if config.ParallelTotal < 1 { - panic("ginkgo.parallel.total must be >= 1") - } - - if config.ParallelNode > config.ParallelTotal || config.ParallelNode < 1 { - panic("ginkgo.parallel.node is one-indexed and must be <= ginkgo.parallel.total") - } - - suite.expandTopLevelNodes = true - for _, deferredNode := range suite.deferredContainerNodes { - suite.PushContainerNode(deferredNode.text, deferredNode.body, deferredNode.flag, deferredNode.codeLocation) - } - - r := rand.New(rand.NewSource(config.RandomSeed)) - suite.topLevelContainer.Shuffle(r) - iterator, hasProgrammaticFocus := suite.generateSpecsIterator(description, config) - suite.runner = specrunner.New(description, suite.beforeSuiteNode, iterator, suite.afterSuiteNode, reporters, writer, config) - - suite.running = true - success := suite.runner.Run() - if !success { - t.Fail() - } - return success, hasProgrammaticFocus -} - -func (suite *Suite) generateSpecsIterator(description string, config config.GinkgoConfigType) (spec_iterator.SpecIterator, bool) { - specsSlice := []*spec.Spec{} - suite.topLevelContainer.BackPropagateProgrammaticFocus() - for _, collatedNodes := range suite.topLevelContainer.Collate() { - specsSlice = append(specsSlice, spec.New(collatedNodes.Subject, collatedNodes.Containers, config.EmitSpecProgress)) - } - - specs := spec.NewSpecs(specsSlice) - specs.RegexScansFilePath = config.RegexScansFilePath - - if config.RandomizeAllSpecs { - specs.Shuffle(rand.New(rand.NewSource(config.RandomSeed))) - } - - specs.ApplyFocus(description, config.FocusStrings, config.SkipStrings) - - if config.SkipMeasurements { - specs.SkipMeasurements() - } - - var iterator spec_iterator.SpecIterator - - if config.ParallelTotal > 1 { - iterator = spec_iterator.NewParallelIterator(specs.Specs(), config.SyncHost) - resp, err := http.Get(config.SyncHost + "/has-counter") - if err != nil || resp.StatusCode != http.StatusOK { - iterator = spec_iterator.NewShardedParallelIterator(specs.Specs(), config.ParallelTotal, config.ParallelNode) - } - } else { - iterator = spec_iterator.NewSerialIterator(specs.Specs()) - } - - return iterator, specs.HasProgrammaticFocus() -} - -func (suite *Suite) CurrentRunningSpecSummary() (*types.SpecSummary, bool) { - if !suite.running { - return nil, false - } - return suite.runner.CurrentSpecSummary() -} - -func (suite *Suite) SetBeforeSuiteNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration) { - if suite.beforeSuiteNode != nil { - panic("You may only call BeforeSuite once!") - } - suite.beforeSuiteNode = leafnodes.NewBeforeSuiteNode(body, codeLocation, timeout, suite.failer) -} - -func (suite *Suite) SetAfterSuiteNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration) { - if suite.afterSuiteNode != nil { - panic("You may only call AfterSuite once!") - } - suite.afterSuiteNode = leafnodes.NewAfterSuiteNode(body, codeLocation, timeout, suite.failer) -} - -func (suite *Suite) SetSynchronizedBeforeSuiteNode(bodyA interface{}, bodyB interface{}, codeLocation types.CodeLocation, timeout time.Duration) { - if suite.beforeSuiteNode != nil { - panic("You may only call BeforeSuite once!") - } - suite.beforeSuiteNode = leafnodes.NewSynchronizedBeforeSuiteNode(bodyA, bodyB, codeLocation, timeout, suite.failer) -} - -func (suite *Suite) SetSynchronizedAfterSuiteNode(bodyA interface{}, bodyB interface{}, codeLocation types.CodeLocation, timeout time.Duration) { - if suite.afterSuiteNode != nil { - panic("You may only call AfterSuite once!") - } - suite.afterSuiteNode = leafnodes.NewSynchronizedAfterSuiteNode(bodyA, bodyB, codeLocation, timeout, suite.failer) -} - -func (suite *Suite) PushContainerNode(text string, body func(), flag types.FlagType, codeLocation types.CodeLocation) { - /* - We defer walking the container nodes (which immediately evaluates the `body` function) - until `RunSpecs` is called. We do this by storing off the deferred container nodes. Then, when - `RunSpecs` is called we actually go through and add the container nodes to the test structure. - - This allows us to defer calling all the `body` functions until _after_ the top level functions - have been walked, _after_ func init()s have been called, and _after_ `go test` has called `flag.Parse()`. - - This allows users to load up configuration information in the `TestX` go test hook just before `RunSpecs` - is invoked and solves issues like #693 and makes the lifecycle easier to reason about. - - */ - if !suite.expandTopLevelNodes { - suite.deferredContainerNodes = append(suite.deferredContainerNodes, deferredContainerNode{text, body, flag, codeLocation}) - return - } - - container := containernode.New(text, flag, codeLocation) - suite.currentContainer.PushContainerNode(container) - - previousContainer := suite.currentContainer - suite.currentContainer = container - suite.containerIndex++ - - body() - - suite.containerIndex-- - suite.currentContainer = previousContainer -} - -func (suite *Suite) PushItNode(text string, body interface{}, flag types.FlagType, codeLocation types.CodeLocation, timeout time.Duration) { - if suite.running { - suite.failer.Fail("You may only call It from within a Describe, Context or When", codeLocation) - } - suite.currentContainer.PushSubjectNode(leafnodes.NewItNode(text, body, flag, codeLocation, timeout, suite.failer, suite.containerIndex)) -} - -func (suite *Suite) PushMeasureNode(text string, body interface{}, flag types.FlagType, codeLocation types.CodeLocation, samples int) { - if suite.running { - suite.failer.Fail("You may only call Measure from within a Describe, Context or When", codeLocation) - } - suite.currentContainer.PushSubjectNode(leafnodes.NewMeasureNode(text, body, flag, codeLocation, samples, suite.failer, suite.containerIndex)) -} - -func (suite *Suite) PushBeforeEachNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration) { - if suite.running { - suite.failer.Fail("You may only call BeforeEach from within a Describe, Context or When", codeLocation) - } - suite.currentContainer.PushSetupNode(leafnodes.NewBeforeEachNode(body, codeLocation, timeout, suite.failer, suite.containerIndex)) -} - -func (suite *Suite) PushJustBeforeEachNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration) { - if suite.running { - suite.failer.Fail("You may only call JustBeforeEach from within a Describe, Context or When", codeLocation) - } - suite.currentContainer.PushSetupNode(leafnodes.NewJustBeforeEachNode(body, codeLocation, timeout, suite.failer, suite.containerIndex)) -} - -func (suite *Suite) PushJustAfterEachNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration) { - if suite.running { - suite.failer.Fail("You may only call JustAfterEach from within a Describe or Context", codeLocation) - } - suite.currentContainer.PushSetupNode(leafnodes.NewJustAfterEachNode(body, codeLocation, timeout, suite.failer, suite.containerIndex)) -} - -func (suite *Suite) PushAfterEachNode(body interface{}, codeLocation types.CodeLocation, timeout time.Duration) { - if suite.running { - suite.failer.Fail("You may only call AfterEach from within a Describe, Context or When", codeLocation) - } - suite.currentContainer.PushSetupNode(leafnodes.NewAfterEachNode(body, codeLocation, timeout, suite.failer, suite.containerIndex)) -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/writer/fake_writer.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/writer/fake_writer.go deleted file mode 100644 index 6739c3f605..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/writer/fake_writer.go +++ /dev/null @@ -1,36 +0,0 @@ -package writer - -type FakeGinkgoWriter struct { - EventStream []string -} - -func NewFake() *FakeGinkgoWriter { - return &FakeGinkgoWriter{ - EventStream: []string{}, - } -} - -func (writer *FakeGinkgoWriter) AddEvent(event string) { - writer.EventStream = append(writer.EventStream, event) -} - -func (writer *FakeGinkgoWriter) Truncate() { - writer.EventStream = append(writer.EventStream, "TRUNCATE") -} - -func (writer *FakeGinkgoWriter) DumpOut() { - writer.EventStream = append(writer.EventStream, "DUMP") -} - -func (writer *FakeGinkgoWriter) DumpOutWithHeader(header string) { - writer.EventStream = append(writer.EventStream, "DUMP_WITH_HEADER: "+header) -} - -func (writer *FakeGinkgoWriter) Bytes() []byte { - writer.EventStream = append(writer.EventStream, "BYTES") - return nil -} - -func (writer *FakeGinkgoWriter) Write(data []byte) (n int, err error) { - return 0, nil -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/internal/writer/writer.go b/go-controller/vendor/github.com/onsi/ginkgo/internal/writer/writer.go deleted file mode 100644 index 98eca3bdd2..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/internal/writer/writer.go +++ /dev/null @@ -1,89 +0,0 @@ -package writer - -import ( - "bytes" - "io" - "sync" -) - -type WriterInterface interface { - io.Writer - - Truncate() - DumpOut() - DumpOutWithHeader(header string) - Bytes() []byte -} - -type Writer struct { - buffer *bytes.Buffer - outWriter io.Writer - lock *sync.Mutex - stream bool - redirector io.Writer -} - -func New(outWriter io.Writer) *Writer { - return &Writer{ - buffer: &bytes.Buffer{}, - lock: &sync.Mutex{}, - outWriter: outWriter, - stream: true, - } -} - -func (w *Writer) AndRedirectTo(writer io.Writer) { - w.redirector = writer -} - -func (w *Writer) SetStream(stream bool) { - w.lock.Lock() - defer w.lock.Unlock() - w.stream = stream -} - -func (w *Writer) Write(b []byte) (n int, err error) { - w.lock.Lock() - defer w.lock.Unlock() - - n, err = w.buffer.Write(b) - if w.redirector != nil { - w.redirector.Write(b) - } - if w.stream { - return w.outWriter.Write(b) - } - return n, err -} - -func (w *Writer) Truncate() { - w.lock.Lock() - defer w.lock.Unlock() - w.buffer.Reset() -} - -func (w *Writer) DumpOut() { - w.lock.Lock() - defer w.lock.Unlock() - if !w.stream { - w.buffer.WriteTo(w.outWriter) - } -} - -func (w *Writer) Bytes() []byte { - w.lock.Lock() - defer w.lock.Unlock() - b := w.buffer.Bytes() - copied := make([]byte, len(b)) - copy(copied, b) - return copied -} - -func (w *Writer) DumpOutWithHeader(header string) { - w.lock.Lock() - defer w.lock.Unlock() - if !w.stream && w.buffer.Len() > 0 { - w.outWriter.Write([]byte(header)) - w.buffer.WriteTo(w.outWriter) - } -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/reporters/default_reporter.go b/go-controller/vendor/github.com/onsi/ginkgo/reporters/default_reporter.go deleted file mode 100644 index f0c9f61410..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/reporters/default_reporter.go +++ /dev/null @@ -1,87 +0,0 @@ -/* -Ginkgo's Default Reporter - -A number of command line flags are available to tweak Ginkgo's default output. - -These are documented [here](http://onsi.github.io/ginkgo/#running_tests) -*/ -package reporters - -import ( - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/reporters/stenographer" - "github.com/onsi/ginkgo/types" -) - -type DefaultReporter struct { - config config.DefaultReporterConfigType - stenographer stenographer.Stenographer - specSummaries []*types.SpecSummary -} - -func NewDefaultReporter(config config.DefaultReporterConfigType, stenographer stenographer.Stenographer) *DefaultReporter { - return &DefaultReporter{ - config: config, - stenographer: stenographer, - } -} - -func (reporter *DefaultReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { - reporter.stenographer.AnnounceSuite(summary.SuiteDescription, config.RandomSeed, config.RandomizeAllSpecs, reporter.config.Succinct) - if config.ParallelTotal > 1 { - reporter.stenographer.AnnounceParallelRun(config.ParallelNode, config.ParallelTotal, reporter.config.Succinct) - } else { - reporter.stenographer.AnnounceNumberOfSpecs(summary.NumberOfSpecsThatWillBeRun, summary.NumberOfTotalSpecs, reporter.config.Succinct) - } -} - -func (reporter *DefaultReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) { - if setupSummary.State != types.SpecStatePassed { - reporter.stenographer.AnnounceBeforeSuiteFailure(setupSummary, reporter.config.Succinct, reporter.config.FullTrace) - } -} - -func (reporter *DefaultReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) { - if setupSummary.State != types.SpecStatePassed { - reporter.stenographer.AnnounceAfterSuiteFailure(setupSummary, reporter.config.Succinct, reporter.config.FullTrace) - } -} - -func (reporter *DefaultReporter) SpecWillRun(specSummary *types.SpecSummary) { - if reporter.config.Verbose && !reporter.config.Succinct && specSummary.State != types.SpecStatePending && specSummary.State != types.SpecStateSkipped { - reporter.stenographer.AnnounceSpecWillRun(specSummary) - } -} - -func (reporter *DefaultReporter) SpecDidComplete(specSummary *types.SpecSummary) { - switch specSummary.State { - case types.SpecStatePassed: - if specSummary.IsMeasurement { - reporter.stenographer.AnnounceSuccessfulMeasurement(specSummary, reporter.config.Succinct) - } else if specSummary.RunTime.Seconds() >= reporter.config.SlowSpecThreshold { - reporter.stenographer.AnnounceSuccessfulSlowSpec(specSummary, reporter.config.Succinct) - } else { - reporter.stenographer.AnnounceSuccessfulSpec(specSummary) - if reporter.config.ReportPassed { - reporter.stenographer.AnnounceCapturedOutput(specSummary.CapturedOutput) - } - } - case types.SpecStatePending: - reporter.stenographer.AnnouncePendingSpec(specSummary, reporter.config.NoisyPendings && !reporter.config.Succinct) - case types.SpecStateSkipped: - reporter.stenographer.AnnounceSkippedSpec(specSummary, reporter.config.Succinct || !reporter.config.NoisySkippings, reporter.config.FullTrace) - case types.SpecStateTimedOut: - reporter.stenographer.AnnounceSpecTimedOut(specSummary, reporter.config.Succinct, reporter.config.FullTrace) - case types.SpecStatePanicked: - reporter.stenographer.AnnounceSpecPanicked(specSummary, reporter.config.Succinct, reporter.config.FullTrace) - case types.SpecStateFailed: - reporter.stenographer.AnnounceSpecFailed(specSummary, reporter.config.Succinct, reporter.config.FullTrace) - } - - reporter.specSummaries = append(reporter.specSummaries, specSummary) -} - -func (reporter *DefaultReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { - reporter.stenographer.SummarizeFailures(reporter.specSummaries) - reporter.stenographer.AnnounceSpecRunCompletion(summary, reporter.config.Succinct) -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/reporters/fake_reporter.go b/go-controller/vendor/github.com/onsi/ginkgo/reporters/fake_reporter.go deleted file mode 100644 index 27db479490..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/reporters/fake_reporter.go +++ /dev/null @@ -1,59 +0,0 @@ -package reporters - -import ( - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/types" -) - -//FakeReporter is useful for testing purposes -type FakeReporter struct { - Config config.GinkgoConfigType - - BeginSummary *types.SuiteSummary - BeforeSuiteSummary *types.SetupSummary - SpecWillRunSummaries []*types.SpecSummary - SpecSummaries []*types.SpecSummary - AfterSuiteSummary *types.SetupSummary - EndSummary *types.SuiteSummary - - SpecWillRunStub func(specSummary *types.SpecSummary) - SpecDidCompleteStub func(specSummary *types.SpecSummary) -} - -func NewFakeReporter() *FakeReporter { - return &FakeReporter{ - SpecWillRunSummaries: make([]*types.SpecSummary, 0), - SpecSummaries: make([]*types.SpecSummary, 0), - } -} - -func (fakeR *FakeReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { - fakeR.Config = config - fakeR.BeginSummary = summary -} - -func (fakeR *FakeReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) { - fakeR.BeforeSuiteSummary = setupSummary -} - -func (fakeR *FakeReporter) SpecWillRun(specSummary *types.SpecSummary) { - if fakeR.SpecWillRunStub != nil { - fakeR.SpecWillRunStub(specSummary) - } - fakeR.SpecWillRunSummaries = append(fakeR.SpecWillRunSummaries, specSummary) -} - -func (fakeR *FakeReporter) SpecDidComplete(specSummary *types.SpecSummary) { - if fakeR.SpecDidCompleteStub != nil { - fakeR.SpecDidCompleteStub(specSummary) - } - fakeR.SpecSummaries = append(fakeR.SpecSummaries, specSummary) -} - -func (fakeR *FakeReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) { - fakeR.AfterSuiteSummary = setupSummary -} - -func (fakeR *FakeReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { - fakeR.EndSummary = summary -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/reporters/junit_reporter.go b/go-controller/vendor/github.com/onsi/ginkgo/reporters/junit_reporter.go deleted file mode 100644 index 01ddca6e1d..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/reporters/junit_reporter.go +++ /dev/null @@ -1,178 +0,0 @@ -/* - -JUnit XML Reporter for Ginkgo - -For usage instructions: http://onsi.github.io/ginkgo/#generating_junit_xml_output - -*/ - -package reporters - -import ( - "encoding/xml" - "fmt" - "math" - "os" - "path/filepath" - "strings" - - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/types" -) - -type JUnitTestSuite struct { - XMLName xml.Name `xml:"testsuite"` - TestCases []JUnitTestCase `xml:"testcase"` - Name string `xml:"name,attr"` - Tests int `xml:"tests,attr"` - Failures int `xml:"failures,attr"` - Errors int `xml:"errors,attr"` - Time float64 `xml:"time,attr"` -} - -type JUnitTestCase struct { - Name string `xml:"name,attr"` - ClassName string `xml:"classname,attr"` - FailureMessage *JUnitFailureMessage `xml:"failure,omitempty"` - Skipped *JUnitSkipped `xml:"skipped,omitempty"` - Time float64 `xml:"time,attr"` - SystemOut string `xml:"system-out,omitempty"` -} - -type JUnitFailureMessage struct { - Type string `xml:"type,attr"` - Message string `xml:",chardata"` -} - -type JUnitSkipped struct { - Message string `xml:",chardata"` -} - -type JUnitReporter struct { - suite JUnitTestSuite - filename string - testSuiteName string - ReporterConfig config.DefaultReporterConfigType -} - -//NewJUnitReporter creates a new JUnit XML reporter. The XML will be stored in the passed in filename. -func NewJUnitReporter(filename string) *JUnitReporter { - return &JUnitReporter{ - filename: filename, - } -} - -func (reporter *JUnitReporter) SpecSuiteWillBegin(ginkgoConfig config.GinkgoConfigType, summary *types.SuiteSummary) { - reporter.suite = JUnitTestSuite{ - Name: summary.SuiteDescription, - TestCases: []JUnitTestCase{}, - } - reporter.testSuiteName = summary.SuiteDescription - reporter.ReporterConfig = config.DefaultReporterConfig -} - -func (reporter *JUnitReporter) SpecWillRun(specSummary *types.SpecSummary) { -} - -func (reporter *JUnitReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) { - reporter.handleSetupSummary("BeforeSuite", setupSummary) -} - -func (reporter *JUnitReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) { - reporter.handleSetupSummary("AfterSuite", setupSummary) -} - -func failureMessage(failure types.SpecFailure) string { - return fmt.Sprintf("%s\n%s\n%s", failure.ComponentCodeLocation.String(), failure.Message, failure.Location.String()) -} - -func (reporter *JUnitReporter) handleSetupSummary(name string, setupSummary *types.SetupSummary) { - if setupSummary.State != types.SpecStatePassed { - testCase := JUnitTestCase{ - Name: name, - ClassName: reporter.testSuiteName, - } - - testCase.FailureMessage = &JUnitFailureMessage{ - Type: reporter.failureTypeForState(setupSummary.State), - Message: failureMessage(setupSummary.Failure), - } - testCase.SystemOut = setupSummary.CapturedOutput - testCase.Time = setupSummary.RunTime.Seconds() - reporter.suite.TestCases = append(reporter.suite.TestCases, testCase) - } -} - -func (reporter *JUnitReporter) SpecDidComplete(specSummary *types.SpecSummary) { - testCase := JUnitTestCase{ - Name: strings.Join(specSummary.ComponentTexts[1:], " "), - ClassName: reporter.testSuiteName, - } - if reporter.ReporterConfig.ReportPassed && specSummary.State == types.SpecStatePassed { - testCase.SystemOut = specSummary.CapturedOutput - } - if specSummary.State == types.SpecStateFailed || specSummary.State == types.SpecStateTimedOut || specSummary.State == types.SpecStatePanicked { - testCase.FailureMessage = &JUnitFailureMessage{ - Type: reporter.failureTypeForState(specSummary.State), - Message: failureMessage(specSummary.Failure), - } - if specSummary.State == types.SpecStatePanicked { - testCase.FailureMessage.Message += fmt.Sprintf("\n\nPanic: %s\n\nFull stack:\n%s", - specSummary.Failure.ForwardedPanic, - specSummary.Failure.Location.FullStackTrace) - } - testCase.SystemOut = specSummary.CapturedOutput - } - if specSummary.State == types.SpecStateSkipped || specSummary.State == types.SpecStatePending { - testCase.Skipped = &JUnitSkipped{} - if specSummary.Failure.Message != "" { - testCase.Skipped.Message = failureMessage(specSummary.Failure) - } - } - testCase.Time = specSummary.RunTime.Seconds() - reporter.suite.TestCases = append(reporter.suite.TestCases, testCase) -} - -func (reporter *JUnitReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { - reporter.suite.Tests = summary.NumberOfSpecsThatWillBeRun - reporter.suite.Time = math.Trunc(summary.RunTime.Seconds()*1000) / 1000 - reporter.suite.Failures = summary.NumberOfFailedSpecs - reporter.suite.Errors = 0 - if reporter.ReporterConfig.ReportFile != "" { - reporter.filename = reporter.ReporterConfig.ReportFile - fmt.Printf("\nJUnit path was configured: %s\n", reporter.filename) - } - filePath, _ := filepath.Abs(reporter.filename) - dirPath := filepath.Dir(filePath) - err := os.MkdirAll(dirPath, os.ModePerm) - if err != nil { - fmt.Printf("\nFailed to create JUnit directory: %s\n\t%s", filePath, err.Error()) - } - file, err := os.Create(filePath) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to create JUnit report file: %s\n\t%s", filePath, err.Error()) - } - defer file.Close() - file.WriteString(xml.Header) - encoder := xml.NewEncoder(file) - encoder.Indent(" ", " ") - err = encoder.Encode(reporter.suite) - if err == nil { - fmt.Fprintf(os.Stdout, "\nJUnit report was created: %s\n", filePath) - } else { - fmt.Fprintf(os.Stderr,"\nFailed to generate JUnit report data:\n\t%s", err.Error()) - } -} - -func (reporter *JUnitReporter) failureTypeForState(state types.SpecState) string { - switch state { - case types.SpecStateFailed: - return "Failure" - case types.SpecStateTimedOut: - return "Timeout" - case types.SpecStatePanicked: - return "Panic" - default: - return "" - } -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/reporters/reporter.go b/go-controller/vendor/github.com/onsi/ginkgo/reporters/reporter.go deleted file mode 100644 index 348b9dfce1..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/reporters/reporter.go +++ /dev/null @@ -1,15 +0,0 @@ -package reporters - -import ( - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/types" -) - -type Reporter interface { - SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) - BeforeSuiteDidRun(setupSummary *types.SetupSummary) - SpecWillRun(specSummary *types.SpecSummary) - SpecDidComplete(specSummary *types.SpecSummary) - AfterSuiteDidRun(setupSummary *types.SetupSummary) - SpecSuiteDidEnd(summary *types.SuiteSummary) -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/reporters/stenographer/console_logging.go b/go-controller/vendor/github.com/onsi/ginkgo/reporters/stenographer/console_logging.go deleted file mode 100644 index 45b8f88690..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/reporters/stenographer/console_logging.go +++ /dev/null @@ -1,64 +0,0 @@ -package stenographer - -import ( - "fmt" - "strings" -) - -func (s *consoleStenographer) colorize(colorCode string, format string, args ...interface{}) string { - var out string - - if len(args) > 0 { - out = fmt.Sprintf(format, args...) - } else { - out = format - } - - if s.color { - return fmt.Sprintf("%s%s%s", colorCode, out, defaultStyle) - } else { - return out - } -} - -func (s *consoleStenographer) printBanner(text string, bannerCharacter string) { - fmt.Fprintln(s.w, text) - fmt.Fprintln(s.w, strings.Repeat(bannerCharacter, len(text))) -} - -func (s *consoleStenographer) printNewLine() { - fmt.Fprintln(s.w, "") -} - -func (s *consoleStenographer) printDelimiter() { - fmt.Fprintln(s.w, s.colorize(grayColor, "%s", strings.Repeat("-", 30))) -} - -func (s *consoleStenographer) print(indentation int, format string, args ...interface{}) { - fmt.Fprint(s.w, s.indent(indentation, format, args...)) -} - -func (s *consoleStenographer) println(indentation int, format string, args ...interface{}) { - fmt.Fprintln(s.w, s.indent(indentation, format, args...)) -} - -func (s *consoleStenographer) indent(indentation int, format string, args ...interface{}) string { - var text string - - if len(args) > 0 { - text = fmt.Sprintf(format, args...) - } else { - text = format - } - - stringArray := strings.Split(text, "\n") - padding := "" - if indentation >= 0 { - padding = strings.Repeat(" ", indentation) - } - for i, s := range stringArray { - stringArray[i] = fmt.Sprintf("%s%s", padding, s) - } - - return strings.Join(stringArray, "\n") -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/reporters/stenographer/fake_stenographer.go b/go-controller/vendor/github.com/onsi/ginkgo/reporters/stenographer/fake_stenographer.go deleted file mode 100644 index 1aa5b9db0f..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/reporters/stenographer/fake_stenographer.go +++ /dev/null @@ -1,142 +0,0 @@ -package stenographer - -import ( - "sync" - - "github.com/onsi/ginkgo/types" -) - -func NewFakeStenographerCall(method string, args ...interface{}) FakeStenographerCall { - return FakeStenographerCall{ - Method: method, - Args: args, - } -} - -type FakeStenographer struct { - calls []FakeStenographerCall - lock *sync.Mutex -} - -type FakeStenographerCall struct { - Method string - Args []interface{} -} - -func NewFakeStenographer() *FakeStenographer { - stenographer := &FakeStenographer{ - lock: &sync.Mutex{}, - } - stenographer.Reset() - return stenographer -} - -func (stenographer *FakeStenographer) Calls() []FakeStenographerCall { - stenographer.lock.Lock() - defer stenographer.lock.Unlock() - - return stenographer.calls -} - -func (stenographer *FakeStenographer) Reset() { - stenographer.lock.Lock() - defer stenographer.lock.Unlock() - - stenographer.calls = make([]FakeStenographerCall, 0) -} - -func (stenographer *FakeStenographer) CallsTo(method string) []FakeStenographerCall { - stenographer.lock.Lock() - defer stenographer.lock.Unlock() - - results := make([]FakeStenographerCall, 0) - for _, call := range stenographer.calls { - if call.Method == method { - results = append(results, call) - } - } - - return results -} - -func (stenographer *FakeStenographer) registerCall(method string, args ...interface{}) { - stenographer.lock.Lock() - defer stenographer.lock.Unlock() - - stenographer.calls = append(stenographer.calls, NewFakeStenographerCall(method, args...)) -} - -func (stenographer *FakeStenographer) AnnounceSuite(description string, randomSeed int64, randomizingAll bool, succinct bool) { - stenographer.registerCall("AnnounceSuite", description, randomSeed, randomizingAll, succinct) -} - -func (stenographer *FakeStenographer) AnnounceAggregatedParallelRun(nodes int, succinct bool) { - stenographer.registerCall("AnnounceAggregatedParallelRun", nodes, succinct) -} - -func (stenographer *FakeStenographer) AnnounceParallelRun(node int, nodes int, succinct bool) { - stenographer.registerCall("AnnounceParallelRun", node, nodes, succinct) -} - -func (stenographer *FakeStenographer) AnnounceNumberOfSpecs(specsToRun int, total int, succinct bool) { - stenographer.registerCall("AnnounceNumberOfSpecs", specsToRun, total, succinct) -} - -func (stenographer *FakeStenographer) AnnounceTotalNumberOfSpecs(total int, succinct bool) { - stenographer.registerCall("AnnounceTotalNumberOfSpecs", total, succinct) -} - -func (stenographer *FakeStenographer) AnnounceSpecRunCompletion(summary *types.SuiteSummary, succinct bool) { - stenographer.registerCall("AnnounceSpecRunCompletion", summary, succinct) -} - -func (stenographer *FakeStenographer) AnnounceSpecWillRun(spec *types.SpecSummary) { - stenographer.registerCall("AnnounceSpecWillRun", spec) -} - -func (stenographer *FakeStenographer) AnnounceBeforeSuiteFailure(summary *types.SetupSummary, succinct bool, fullTrace bool) { - stenographer.registerCall("AnnounceBeforeSuiteFailure", summary, succinct, fullTrace) -} - -func (stenographer *FakeStenographer) AnnounceAfterSuiteFailure(summary *types.SetupSummary, succinct bool, fullTrace bool) { - stenographer.registerCall("AnnounceAfterSuiteFailure", summary, succinct, fullTrace) -} -func (stenographer *FakeStenographer) AnnounceCapturedOutput(output string) { - stenographer.registerCall("AnnounceCapturedOutput", output) -} - -func (stenographer *FakeStenographer) AnnounceSuccessfulSpec(spec *types.SpecSummary) { - stenographer.registerCall("AnnounceSuccessfulSpec", spec) -} - -func (stenographer *FakeStenographer) AnnounceSuccessfulSlowSpec(spec *types.SpecSummary, succinct bool) { - stenographer.registerCall("AnnounceSuccessfulSlowSpec", spec, succinct) -} - -func (stenographer *FakeStenographer) AnnounceSuccessfulMeasurement(spec *types.SpecSummary, succinct bool) { - stenographer.registerCall("AnnounceSuccessfulMeasurement", spec, succinct) -} - -func (stenographer *FakeStenographer) AnnouncePendingSpec(spec *types.SpecSummary, noisy bool) { - stenographer.registerCall("AnnouncePendingSpec", spec, noisy) -} - -func (stenographer *FakeStenographer) AnnounceSkippedSpec(spec *types.SpecSummary, succinct bool, fullTrace bool) { - stenographer.registerCall("AnnounceSkippedSpec", spec, succinct, fullTrace) -} - -func (stenographer *FakeStenographer) AnnounceSpecTimedOut(spec *types.SpecSummary, succinct bool, fullTrace bool) { - stenographer.registerCall("AnnounceSpecTimedOut", spec, succinct, fullTrace) -} - -func (stenographer *FakeStenographer) AnnounceSpecPanicked(spec *types.SpecSummary, succinct bool, fullTrace bool) { - stenographer.registerCall("AnnounceSpecPanicked", spec, succinct, fullTrace) -} - -func (stenographer *FakeStenographer) AnnounceSpecFailed(spec *types.SpecSummary, succinct bool, fullTrace bool) { - stenographer.registerCall("AnnounceSpecFailed", spec, succinct, fullTrace) -} - -func (stenographer *FakeStenographer) SummarizeFailures(summaries []*types.SpecSummary) { - stenographer.registerCall("SummarizeFailures", summaries) -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/reporters/stenographer/stenographer.go b/go-controller/vendor/github.com/onsi/ginkgo/reporters/stenographer/stenographer.go deleted file mode 100644 index 638d6fbb1a..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/reporters/stenographer/stenographer.go +++ /dev/null @@ -1,572 +0,0 @@ -/* -The stenographer is used by Ginkgo's reporters to generate output. - -Move along, nothing to see here. -*/ - -package stenographer - -import ( - "fmt" - "io" - "runtime" - "strings" - - "github.com/onsi/ginkgo/types" -) - -const defaultStyle = "\x1b[0m" -const boldStyle = "\x1b[1m" -const redColor = "\x1b[91m" -const greenColor = "\x1b[32m" -const yellowColor = "\x1b[33m" -const cyanColor = "\x1b[36m" -const grayColor = "\x1b[90m" -const lightGrayColor = "\x1b[37m" - -type cursorStateType int - -const ( - cursorStateTop cursorStateType = iota - cursorStateStreaming - cursorStateMidBlock - cursorStateEndBlock -) - -type Stenographer interface { - AnnounceSuite(description string, randomSeed int64, randomizingAll bool, succinct bool) - AnnounceAggregatedParallelRun(nodes int, succinct bool) - AnnounceParallelRun(node int, nodes int, succinct bool) - AnnounceTotalNumberOfSpecs(total int, succinct bool) - AnnounceNumberOfSpecs(specsToRun int, total int, succinct bool) - AnnounceSpecRunCompletion(summary *types.SuiteSummary, succinct bool) - - AnnounceSpecWillRun(spec *types.SpecSummary) - AnnounceBeforeSuiteFailure(summary *types.SetupSummary, succinct bool, fullTrace bool) - AnnounceAfterSuiteFailure(summary *types.SetupSummary, succinct bool, fullTrace bool) - - AnnounceCapturedOutput(output string) - - AnnounceSuccessfulSpec(spec *types.SpecSummary) - AnnounceSuccessfulSlowSpec(spec *types.SpecSummary, succinct bool) - AnnounceSuccessfulMeasurement(spec *types.SpecSummary, succinct bool) - - AnnouncePendingSpec(spec *types.SpecSummary, noisy bool) - AnnounceSkippedSpec(spec *types.SpecSummary, succinct bool, fullTrace bool) - - AnnounceSpecTimedOut(spec *types.SpecSummary, succinct bool, fullTrace bool) - AnnounceSpecPanicked(spec *types.SpecSummary, succinct bool, fullTrace bool) - AnnounceSpecFailed(spec *types.SpecSummary, succinct bool, fullTrace bool) - - SummarizeFailures(summaries []*types.SpecSummary) -} - -func New(color bool, enableFlakes bool, writer io.Writer) Stenographer { - denoter := "•" - if runtime.GOOS == "windows" { - denoter = "+" - } - return &consoleStenographer{ - color: color, - denoter: denoter, - cursorState: cursorStateTop, - enableFlakes: enableFlakes, - w: writer, - } -} - -type consoleStenographer struct { - color bool - denoter string - cursorState cursorStateType - enableFlakes bool - w io.Writer -} - -var alternatingColors = []string{defaultStyle, grayColor} - -func (s *consoleStenographer) AnnounceSuite(description string, randomSeed int64, randomizingAll bool, succinct bool) { - if succinct { - s.print(0, "[%d] %s ", randomSeed, s.colorize(boldStyle, description)) - return - } - s.printBanner(fmt.Sprintf("Running Suite: %s", description), "=") - s.print(0, "Random Seed: %s", s.colorize(boldStyle, "%d", randomSeed)) - if randomizingAll { - s.print(0, " - Will randomize all specs") - } - s.printNewLine() -} - -func (s *consoleStenographer) AnnounceParallelRun(node int, nodes int, succinct bool) { - if succinct { - s.print(0, "- node #%d ", node) - return - } - s.println(0, - "Parallel test node %s/%s.", - s.colorize(boldStyle, "%d", node), - s.colorize(boldStyle, "%d", nodes), - ) - s.printNewLine() -} - -func (s *consoleStenographer) AnnounceAggregatedParallelRun(nodes int, succinct bool) { - if succinct { - s.print(0, "- %d nodes ", nodes) - return - } - s.println(0, - "Running in parallel across %s nodes", - s.colorize(boldStyle, "%d", nodes), - ) - s.printNewLine() -} - -func (s *consoleStenographer) AnnounceNumberOfSpecs(specsToRun int, total int, succinct bool) { - if succinct { - s.print(0, "- %d/%d specs ", specsToRun, total) - s.stream() - return - } - s.println(0, - "Will run %s of %s specs", - s.colorize(boldStyle, "%d", specsToRun), - s.colorize(boldStyle, "%d", total), - ) - - s.printNewLine() -} - -func (s *consoleStenographer) AnnounceTotalNumberOfSpecs(total int, succinct bool) { - if succinct { - s.print(0, "- %d specs ", total) - s.stream() - return - } - s.println(0, - "Will run %s specs", - s.colorize(boldStyle, "%d", total), - ) - - s.printNewLine() -} - -func (s *consoleStenographer) AnnounceSpecRunCompletion(summary *types.SuiteSummary, succinct bool) { - if succinct && summary.SuiteSucceeded { - s.print(0, " %s %s ", s.colorize(greenColor, "SUCCESS!"), summary.RunTime) - return - } - s.printNewLine() - color := greenColor - if !summary.SuiteSucceeded { - color = redColor - } - s.println(0, s.colorize(boldStyle+color, "Ran %d of %d Specs in %.3f seconds", summary.NumberOfSpecsThatWillBeRun, summary.NumberOfTotalSpecs, summary.RunTime.Seconds())) - - status := "" - if summary.SuiteSucceeded { - status = s.colorize(boldStyle+greenColor, "SUCCESS!") - } else { - status = s.colorize(boldStyle+redColor, "FAIL!") - } - - flakes := "" - if s.enableFlakes { - flakes = " | " + s.colorize(yellowColor+boldStyle, "%d Flaked", summary.NumberOfFlakedSpecs) - } - - s.print(0, - "%s -- %s | %s | %s | %s\n", - status, - s.colorize(greenColor+boldStyle, "%d Passed", summary.NumberOfPassedSpecs), - s.colorize(redColor+boldStyle, "%d Failed", summary.NumberOfFailedSpecs)+flakes, - s.colorize(yellowColor+boldStyle, "%d Pending", summary.NumberOfPendingSpecs), - s.colorize(cyanColor+boldStyle, "%d Skipped", summary.NumberOfSkippedSpecs), - ) -} - -func (s *consoleStenographer) AnnounceSpecWillRun(spec *types.SpecSummary) { - s.startBlock() - for i, text := range spec.ComponentTexts[1 : len(spec.ComponentTexts)-1] { - s.print(0, s.colorize(alternatingColors[i%2], text)+" ") - } - - indentation := 0 - if len(spec.ComponentTexts) > 2 { - indentation = 1 - s.printNewLine() - } - index := len(spec.ComponentTexts) - 1 - s.print(indentation, s.colorize(boldStyle, spec.ComponentTexts[index])) - s.printNewLine() - s.print(indentation, s.colorize(lightGrayColor, spec.ComponentCodeLocations[index].String())) - s.printNewLine() - s.midBlock() -} - -func (s *consoleStenographer) AnnounceBeforeSuiteFailure(summary *types.SetupSummary, succinct bool, fullTrace bool) { - s.announceSetupFailure("BeforeSuite", summary, succinct, fullTrace) -} - -func (s *consoleStenographer) AnnounceAfterSuiteFailure(summary *types.SetupSummary, succinct bool, fullTrace bool) { - s.announceSetupFailure("AfterSuite", summary, succinct, fullTrace) -} - -func (s *consoleStenographer) announceSetupFailure(name string, summary *types.SetupSummary, succinct bool, fullTrace bool) { - s.startBlock() - var message string - switch summary.State { - case types.SpecStateFailed: - message = "Failure" - case types.SpecStatePanicked: - message = "Panic" - case types.SpecStateTimedOut: - message = "Timeout" - } - - s.println(0, s.colorize(redColor+boldStyle, "%s [%.3f seconds]", message, summary.RunTime.Seconds())) - - indentation := s.printCodeLocationBlock([]string{name}, []types.CodeLocation{summary.CodeLocation}, summary.ComponentType, 0, summary.State, true) - - s.printNewLine() - s.printFailure(indentation, summary.State, summary.Failure, fullTrace) - - s.endBlock() -} - -func (s *consoleStenographer) AnnounceCapturedOutput(output string) { - if output == "" { - return - } - - s.startBlock() - s.println(0, output) - s.midBlock() -} - -func (s *consoleStenographer) AnnounceSuccessfulSpec(spec *types.SpecSummary) { - s.print(0, s.colorize(greenColor, s.denoter)) - s.stream() -} - -func (s *consoleStenographer) AnnounceSuccessfulSlowSpec(spec *types.SpecSummary, succinct bool) { - s.printBlockWithMessage( - s.colorize(greenColor, "%s [SLOW TEST:%.3f seconds]", s.denoter, spec.RunTime.Seconds()), - "", - spec, - succinct, - ) -} - -func (s *consoleStenographer) AnnounceSuccessfulMeasurement(spec *types.SpecSummary, succinct bool) { - s.printBlockWithMessage( - s.colorize(greenColor, "%s [MEASUREMENT]", s.denoter), - s.measurementReport(spec, succinct), - spec, - succinct, - ) -} - -func (s *consoleStenographer) AnnouncePendingSpec(spec *types.SpecSummary, noisy bool) { - if noisy { - s.printBlockWithMessage( - s.colorize(yellowColor, "P [PENDING]"), - "", - spec, - false, - ) - } else { - s.print(0, s.colorize(yellowColor, "P")) - s.stream() - } -} - -func (s *consoleStenographer) AnnounceSkippedSpec(spec *types.SpecSummary, succinct bool, fullTrace bool) { - // Skips at runtime will have a non-empty spec.Failure. All others should be succinct. - if succinct || spec.Failure == (types.SpecFailure{}) { - s.print(0, s.colorize(cyanColor, "S")) - s.stream() - } else { - s.startBlock() - s.println(0, s.colorize(cyanColor+boldStyle, "S [SKIPPING]%s [%.3f seconds]", s.failureContext(spec.Failure.ComponentType), spec.RunTime.Seconds())) - - indentation := s.printCodeLocationBlock(spec.ComponentTexts, spec.ComponentCodeLocations, spec.Failure.ComponentType, spec.Failure.ComponentIndex, spec.State, succinct) - - s.printNewLine() - s.printSkip(indentation, spec.Failure) - s.endBlock() - } -} - -func (s *consoleStenographer) AnnounceSpecTimedOut(spec *types.SpecSummary, succinct bool, fullTrace bool) { - s.printSpecFailure(fmt.Sprintf("%s... Timeout", s.denoter), spec, succinct, fullTrace) -} - -func (s *consoleStenographer) AnnounceSpecPanicked(spec *types.SpecSummary, succinct bool, fullTrace bool) { - s.printSpecFailure(fmt.Sprintf("%s! Panic", s.denoter), spec, succinct, fullTrace) -} - -func (s *consoleStenographer) AnnounceSpecFailed(spec *types.SpecSummary, succinct bool, fullTrace bool) { - s.printSpecFailure(fmt.Sprintf("%s Failure", s.denoter), spec, succinct, fullTrace) -} - -func (s *consoleStenographer) SummarizeFailures(summaries []*types.SpecSummary) { - failingSpecs := []*types.SpecSummary{} - - for _, summary := range summaries { - if summary.HasFailureState() { - failingSpecs = append(failingSpecs, summary) - } - } - - if len(failingSpecs) == 0 { - return - } - - s.printNewLine() - s.printNewLine() - plural := "s" - if len(failingSpecs) == 1 { - plural = "" - } - s.println(0, s.colorize(redColor+boldStyle, "Summarizing %d Failure%s:", len(failingSpecs), plural)) - for _, summary := range failingSpecs { - s.printNewLine() - if summary.HasFailureState() { - if summary.TimedOut() { - s.print(0, s.colorize(redColor+boldStyle, "[Timeout...] ")) - } else if summary.Panicked() { - s.print(0, s.colorize(redColor+boldStyle, "[Panic!] ")) - } else if summary.Failed() { - s.print(0, s.colorize(redColor+boldStyle, "[Fail] ")) - } - s.printSpecContext(summary.ComponentTexts, summary.ComponentCodeLocations, summary.Failure.ComponentType, summary.Failure.ComponentIndex, summary.State, true) - s.printNewLine() - s.println(0, s.colorize(lightGrayColor, summary.Failure.Location.String())) - } - } -} - -func (s *consoleStenographer) startBlock() { - if s.cursorState == cursorStateStreaming { - s.printNewLine() - s.printDelimiter() - } else if s.cursorState == cursorStateMidBlock { - s.printNewLine() - } -} - -func (s *consoleStenographer) midBlock() { - s.cursorState = cursorStateMidBlock -} - -func (s *consoleStenographer) endBlock() { - s.printDelimiter() - s.cursorState = cursorStateEndBlock -} - -func (s *consoleStenographer) stream() { - s.cursorState = cursorStateStreaming -} - -func (s *consoleStenographer) printBlockWithMessage(header string, message string, spec *types.SpecSummary, succinct bool) { - s.startBlock() - s.println(0, header) - - indentation := s.printCodeLocationBlock(spec.ComponentTexts, spec.ComponentCodeLocations, types.SpecComponentTypeInvalid, 0, spec.State, succinct) - - if message != "" { - s.printNewLine() - s.println(indentation, message) - } - - s.endBlock() -} - -func (s *consoleStenographer) printSpecFailure(message string, spec *types.SpecSummary, succinct bool, fullTrace bool) { - s.startBlock() - s.println(0, s.colorize(redColor+boldStyle, "%s%s [%.3f seconds]", message, s.failureContext(spec.Failure.ComponentType), spec.RunTime.Seconds())) - - indentation := s.printCodeLocationBlock(spec.ComponentTexts, spec.ComponentCodeLocations, spec.Failure.ComponentType, spec.Failure.ComponentIndex, spec.State, succinct) - - s.printNewLine() - s.printFailure(indentation, spec.State, spec.Failure, fullTrace) - s.endBlock() -} - -func (s *consoleStenographer) failureContext(failedComponentType types.SpecComponentType) string { - switch failedComponentType { - case types.SpecComponentTypeBeforeSuite: - return " in Suite Setup (BeforeSuite)" - case types.SpecComponentTypeAfterSuite: - return " in Suite Teardown (AfterSuite)" - case types.SpecComponentTypeBeforeEach: - return " in Spec Setup (BeforeEach)" - case types.SpecComponentTypeJustBeforeEach: - return " in Spec Setup (JustBeforeEach)" - case types.SpecComponentTypeAfterEach: - return " in Spec Teardown (AfterEach)" - } - - return "" -} - -func (s *consoleStenographer) printSkip(indentation int, spec types.SpecFailure) { - s.println(indentation, s.colorize(cyanColor, spec.Message)) - s.printNewLine() - s.println(indentation, spec.Location.String()) -} - -func (s *consoleStenographer) printFailure(indentation int, state types.SpecState, failure types.SpecFailure, fullTrace bool) { - if state == types.SpecStatePanicked { - s.println(indentation, s.colorize(redColor+boldStyle, failure.Message)) - s.println(indentation, s.colorize(redColor, failure.ForwardedPanic)) - s.println(indentation, failure.Location.String()) - s.printNewLine() - s.println(indentation, s.colorize(redColor, "Full Stack Trace")) - s.println(indentation, failure.Location.FullStackTrace) - } else { - s.println(indentation, s.colorize(redColor, failure.Message)) - s.printNewLine() - s.println(indentation, failure.Location.String()) - if fullTrace { - s.printNewLine() - s.println(indentation, s.colorize(redColor, "Full Stack Trace")) - s.println(indentation, failure.Location.FullStackTrace) - } - } -} - -func (s *consoleStenographer) printSpecContext(componentTexts []string, componentCodeLocations []types.CodeLocation, failedComponentType types.SpecComponentType, failedComponentIndex int, state types.SpecState, succinct bool) int { - startIndex := 1 - indentation := 0 - - if len(componentTexts) == 1 { - startIndex = 0 - } - - for i := startIndex; i < len(componentTexts); i++ { - if (state.IsFailure() || state == types.SpecStateSkipped) && i == failedComponentIndex { - color := redColor - if state == types.SpecStateSkipped { - color = cyanColor - } - blockType := "" - switch failedComponentType { - case types.SpecComponentTypeBeforeSuite: - blockType = "BeforeSuite" - case types.SpecComponentTypeAfterSuite: - blockType = "AfterSuite" - case types.SpecComponentTypeBeforeEach: - blockType = "BeforeEach" - case types.SpecComponentTypeJustBeforeEach: - blockType = "JustBeforeEach" - case types.SpecComponentTypeAfterEach: - blockType = "AfterEach" - case types.SpecComponentTypeIt: - blockType = "It" - case types.SpecComponentTypeMeasure: - blockType = "Measurement" - } - if succinct { - s.print(0, s.colorize(color+boldStyle, "[%s] %s ", blockType, componentTexts[i])) - } else { - s.println(indentation, s.colorize(color+boldStyle, "%s [%s]", componentTexts[i], blockType)) - s.println(indentation, s.colorize(grayColor, "%s", componentCodeLocations[i])) - } - } else { - if succinct { - s.print(0, s.colorize(alternatingColors[i%2], "%s ", componentTexts[i])) - } else { - s.println(indentation, componentTexts[i]) - s.println(indentation, s.colorize(grayColor, "%s", componentCodeLocations[i])) - } - } - indentation++ - } - - return indentation -} - -func (s *consoleStenographer) printCodeLocationBlock(componentTexts []string, componentCodeLocations []types.CodeLocation, failedComponentType types.SpecComponentType, failedComponentIndex int, state types.SpecState, succinct bool) int { - indentation := s.printSpecContext(componentTexts, componentCodeLocations, failedComponentType, failedComponentIndex, state, succinct) - - if succinct { - if len(componentTexts) > 0 { - s.printNewLine() - s.print(0, s.colorize(lightGrayColor, "%s", componentCodeLocations[len(componentCodeLocations)-1])) - } - s.printNewLine() - indentation = 1 - } else { - indentation-- - } - - return indentation -} - -func (s *consoleStenographer) orderedMeasurementKeys(measurements map[string]*types.SpecMeasurement) []string { - orderedKeys := make([]string, len(measurements)) - for key, measurement := range measurements { - orderedKeys[measurement.Order] = key - } - return orderedKeys -} - -func (s *consoleStenographer) measurementReport(spec *types.SpecSummary, succinct bool) string { - if len(spec.Measurements) == 0 { - return "Found no measurements" - } - - message := []string{} - orderedKeys := s.orderedMeasurementKeys(spec.Measurements) - - if succinct { - message = append(message, fmt.Sprintf("%s samples:", s.colorize(boldStyle, "%d", spec.NumberOfSamples))) - for _, key := range orderedKeys { - measurement := spec.Measurements[key] - message = append(message, fmt.Sprintf(" %s - %s: %s%s, %s: %s%s ± %s%s, %s: %s%s", - s.colorize(boldStyle, "%s", measurement.Name), - measurement.SmallestLabel, - s.colorize(greenColor, measurement.PrecisionFmt(), measurement.Smallest), - measurement.Units, - measurement.AverageLabel, - s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.Average), - measurement.Units, - s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.StdDeviation), - measurement.Units, - measurement.LargestLabel, - s.colorize(redColor, measurement.PrecisionFmt(), measurement.Largest), - measurement.Units, - )) - } - } else { - message = append(message, fmt.Sprintf("Ran %s samples:", s.colorize(boldStyle, "%d", spec.NumberOfSamples))) - for _, key := range orderedKeys { - measurement := spec.Measurements[key] - info := "" - if measurement.Info != nil { - message = append(message, fmt.Sprintf("%v", measurement.Info)) - } - - message = append(message, fmt.Sprintf("%s:\n%s %s: %s%s\n %s: %s%s\n %s: %s%s ± %s%s", - s.colorize(boldStyle, "%s", measurement.Name), - info, - measurement.SmallestLabel, - s.colorize(greenColor, measurement.PrecisionFmt(), measurement.Smallest), - measurement.Units, - measurement.LargestLabel, - s.colorize(redColor, measurement.PrecisionFmt(), measurement.Largest), - measurement.Units, - measurement.AverageLabel, - s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.Average), - measurement.Units, - s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.StdDeviation), - measurement.Units, - )) - } - } - - return strings.Join(message, "\n") -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/reporters/teamcity_reporter.go b/go-controller/vendor/github.com/onsi/ginkgo/reporters/teamcity_reporter.go deleted file mode 100644 index 84fd8aff87..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/reporters/teamcity_reporter.go +++ /dev/null @@ -1,106 +0,0 @@ -/* - -TeamCity Reporter for Ginkgo - -Makes use of TeamCity's support for Service Messages -http://confluence.jetbrains.com/display/TCD7/Build+Script+Interaction+with+TeamCity#BuildScriptInteractionwithTeamCity-ReportingTests -*/ - -package reporters - -import ( - "fmt" - "io" - "strings" - - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/types" -) - -const ( - messageId = "##teamcity" -) - -type TeamCityReporter struct { - writer io.Writer - testSuiteName string - ReporterConfig config.DefaultReporterConfigType -} - -func NewTeamCityReporter(writer io.Writer) *TeamCityReporter { - return &TeamCityReporter{ - writer: writer, - } -} - -func (reporter *TeamCityReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { - reporter.testSuiteName = escape(summary.SuiteDescription) - fmt.Fprintf(reporter.writer, "%s[testSuiteStarted name='%s']\n", messageId, reporter.testSuiteName) -} - -func (reporter *TeamCityReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) { - reporter.handleSetupSummary("BeforeSuite", setupSummary) -} - -func (reporter *TeamCityReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) { - reporter.handleSetupSummary("AfterSuite", setupSummary) -} - -func (reporter *TeamCityReporter) handleSetupSummary(name string, setupSummary *types.SetupSummary) { - if setupSummary.State != types.SpecStatePassed { - testName := escape(name) - fmt.Fprintf(reporter.writer, "%s[testStarted name='%s']\n", messageId, testName) - message := reporter.failureMessage(setupSummary.Failure) - details := reporter.failureDetails(setupSummary.Failure) - fmt.Fprintf(reporter.writer, "%s[testFailed name='%s' message='%s' details='%s']\n", messageId, testName, message, details) - durationInMilliseconds := setupSummary.RunTime.Seconds() * 1000 - fmt.Fprintf(reporter.writer, "%s[testFinished name='%s' duration='%v']\n", messageId, testName, durationInMilliseconds) - } -} - -func (reporter *TeamCityReporter) SpecWillRun(specSummary *types.SpecSummary) { - testName := escape(strings.Join(specSummary.ComponentTexts[1:], " ")) - fmt.Fprintf(reporter.writer, "%s[testStarted name='%s']\n", messageId, testName) -} - -func (reporter *TeamCityReporter) SpecDidComplete(specSummary *types.SpecSummary) { - testName := escape(strings.Join(specSummary.ComponentTexts[1:], " ")) - - if reporter.ReporterConfig.ReportPassed && specSummary.State == types.SpecStatePassed { - details := escape(specSummary.CapturedOutput) - fmt.Fprintf(reporter.writer, "%s[testPassed name='%s' details='%s']\n", messageId, testName, details) - } - if specSummary.State == types.SpecStateFailed || specSummary.State == types.SpecStateTimedOut || specSummary.State == types.SpecStatePanicked { - message := reporter.failureMessage(specSummary.Failure) - details := reporter.failureDetails(specSummary.Failure) - fmt.Fprintf(reporter.writer, "%s[testFailed name='%s' message='%s' details='%s']\n", messageId, testName, message, details) - } - if specSummary.State == types.SpecStateSkipped || specSummary.State == types.SpecStatePending { - fmt.Fprintf(reporter.writer, "%s[testIgnored name='%s']\n", messageId, testName) - } - - durationInMilliseconds := specSummary.RunTime.Seconds() * 1000 - fmt.Fprintf(reporter.writer, "%s[testFinished name='%s' duration='%v']\n", messageId, testName, durationInMilliseconds) -} - -func (reporter *TeamCityReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { - fmt.Fprintf(reporter.writer, "%s[testSuiteFinished name='%s']\n", messageId, reporter.testSuiteName) -} - -func (reporter *TeamCityReporter) failureMessage(failure types.SpecFailure) string { - return escape(failure.ComponentCodeLocation.String()) -} - -func (reporter *TeamCityReporter) failureDetails(failure types.SpecFailure) string { - return escape(fmt.Sprintf("%s\n%s", failure.Message, failure.Location.String())) -} - -func escape(output string) string { - output = strings.Replace(output, "|", "||", -1) - output = strings.Replace(output, "'", "|'", -1) - output = strings.Replace(output, "\n", "|n", -1) - output = strings.Replace(output, "\r", "|r", -1) - output = strings.Replace(output, "[", "|[", -1) - output = strings.Replace(output, "]", "|]", -1) - return output -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/types/code_location.go b/go-controller/vendor/github.com/onsi/ginkgo/types/code_location.go deleted file mode 100644 index 935a89e136..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/types/code_location.go +++ /dev/null @@ -1,15 +0,0 @@ -package types - -import ( - "fmt" -) - -type CodeLocation struct { - FileName string - LineNumber int - FullStackTrace string -} - -func (codeLocation CodeLocation) String() string { - return fmt.Sprintf("%s:%d", codeLocation.FileName, codeLocation.LineNumber) -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/types/deprecation_support.go b/go-controller/vendor/github.com/onsi/ginkgo/types/deprecation_support.go deleted file mode 100644 index d5a6658f35..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/types/deprecation_support.go +++ /dev/null @@ -1,160 +0,0 @@ -package types - -import ( - "os" - "strconv" - "strings" - "unicode" - - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/formatter" -) - -type Deprecation struct { - Message string - DocLink string - Version string -} - -type deprecations struct{} - -var Deprecations = deprecations{} - -func (d deprecations) CustomReporter() Deprecation { - return Deprecation{ - Message: "You are using a custom reporter. Support for custom reporters will likely be removed in V2. Most users were using them to generate junit or teamcity reports and this functionality will be merged into the core reporter. In addition, Ginkgo 2.0 will support emitting a JSON-formatted report that users can then manipulate to generate custom reports.\n\n{{red}}{{bold}}If this change will be impactful to you please leave a comment on {{cyan}}{{underline}}https://github.com/onsi/ginkgo/issues/711{{/}}", - DocLink: "removed-custom-reporters", - Version: "1.16.0", - } -} - -func (d deprecations) V1Reporter() Deprecation { - return Deprecation{ - Message: "You are using a V1 Ginkgo Reporter. Please update your custom reporter to the new V2 Reporter interface.", - DocLink: "changed-reporter-interface", - Version: "1.16.0", - } -} - -func (d deprecations) Async() Deprecation { - return Deprecation{ - Message: "You are passing a Done channel to a test node to test asynchronous behavior. This is deprecated in Ginkgo V2. Your test will run synchronously and the timeout will be ignored.", - DocLink: "removed-async-testing", - Version: "1.16.0", - } -} - -func (d deprecations) Measure() Deprecation { - return Deprecation{ - Message: "Measure is deprecated and will be removed in Ginkgo V2. Please migrate to gomega/gmeasure.", - DocLink: "removed-measure", - Version: "1.16.3", - } -} - -func (d deprecations) ParallelNode() Deprecation { - return Deprecation{ - Message: "GinkgoParallelNode is deprecated and will be removed in Ginkgo V2. Please use GinkgoParallelProcess instead.", - DocLink: "renamed-ginkgoparallelnode", - Version: "1.16.5", - } -} - -func (d deprecations) Convert() Deprecation { - return Deprecation{ - Message: "The convert command is deprecated in Ginkgo V2", - DocLink: "removed-ginkgo-convert", - Version: "1.16.0", - } -} - -func (d deprecations) Blur() Deprecation { - return Deprecation{ - Message: "The blur command is deprecated in Ginkgo V2. Use 'ginkgo unfocus' instead.", - Version: "1.16.0", - } -} - -type DeprecationTracker struct { - deprecations map[Deprecation][]CodeLocation -} - -func NewDeprecationTracker() *DeprecationTracker { - return &DeprecationTracker{ - deprecations: map[Deprecation][]CodeLocation{}, - } -} - -func (d *DeprecationTracker) TrackDeprecation(deprecation Deprecation, cl ...CodeLocation) { - ackVersion := os.Getenv("ACK_GINKGO_DEPRECATIONS") - if deprecation.Version != "" && ackVersion != "" { - ack := ParseSemVer(ackVersion) - version := ParseSemVer(deprecation.Version) - if ack.GreaterThanOrEqualTo(version) { - return - } - } - - if len(cl) == 1 { - d.deprecations[deprecation] = append(d.deprecations[deprecation], cl[0]) - } else { - d.deprecations[deprecation] = []CodeLocation{} - } -} - -func (d *DeprecationTracker) DidTrackDeprecations() bool { - return len(d.deprecations) > 0 -} - -func (d *DeprecationTracker) DeprecationsReport() string { - out := formatter.F("\n{{light-yellow}}You're using deprecated Ginkgo functionality:{{/}}\n") - out += formatter.F("{{light-yellow}}============================================={{/}}\n") - out += formatter.F("{{bold}}{{green}}Ginkgo 2.0{{/}} is under active development and will introduce several new features, improvements, and a small handful of breaking changes.\n") - out += formatter.F("A release candidate for 2.0 is now available and 2.0 should GA in Fall 2021. {{bold}}Please give the RC a try and send us feedback!{{/}}\n") - out += formatter.F(" - To learn more, view the migration guide at {{cyan}}{{underline}}https://github.com/onsi/ginkgo/blob/ver2/docs/MIGRATING_TO_V2.md{{/}}\n") - out += formatter.F(" - For instructions on using the Release Candidate visit {{cyan}}{{underline}}https://github.com/onsi/ginkgo/blob/ver2/docs/MIGRATING_TO_V2.md#using-the-beta{{/}}\n") - out += formatter.F(" - To comment, chime in at {{cyan}}{{underline}}https://github.com/onsi/ginkgo/issues/711{{/}}\n\n") - - for deprecation, locations := range d.deprecations { - out += formatter.Fi(1, "{{yellow}}"+deprecation.Message+"{{/}}\n") - if deprecation.DocLink != "" { - out += formatter.Fi(1, "{{bold}}Learn more at:{{/}} {{cyan}}{{underline}}https://github.com/onsi/ginkgo/blob/ver2/docs/MIGRATING_TO_V2.md#%s{{/}}\n", deprecation.DocLink) - } - for _, location := range locations { - out += formatter.Fi(2, "{{gray}}%s{{/}}\n", location) - } - } - out += formatter.F("\n{{gray}}To silence deprecations that can be silenced set the following environment variable:{{/}}\n") - out += formatter.Fi(1, "{{gray}}ACK_GINKGO_DEPRECATIONS=%s{{/}}\n", config.VERSION) - return out -} - -type SemVer struct { - Major int - Minor int - Patch int -} - -func (s SemVer) GreaterThanOrEqualTo(o SemVer) bool { - return (s.Major > o.Major) || - (s.Major == o.Major && s.Minor > o.Minor) || - (s.Major == o.Major && s.Minor == o.Minor && s.Patch >= o.Patch) -} - -func ParseSemVer(semver string) SemVer { - out := SemVer{} - semver = strings.TrimFunc(semver, func(r rune) bool { - return !(unicode.IsNumber(r) || r == '.') - }) - components := strings.Split(semver, ".") - if len(components) > 0 { - out.Major, _ = strconv.Atoi(components[0]) - } - if len(components) > 1 { - out.Minor, _ = strconv.Atoi(components[1]) - } - if len(components) > 2 { - out.Patch, _ = strconv.Atoi(components[2]) - } - return out -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/types/synchronization.go b/go-controller/vendor/github.com/onsi/ginkgo/types/synchronization.go deleted file mode 100644 index fdd6ed5bdf..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/types/synchronization.go +++ /dev/null @@ -1,30 +0,0 @@ -package types - -import ( - "encoding/json" -) - -type RemoteBeforeSuiteState int - -const ( - RemoteBeforeSuiteStateInvalid RemoteBeforeSuiteState = iota - - RemoteBeforeSuiteStatePending - RemoteBeforeSuiteStatePassed - RemoteBeforeSuiteStateFailed - RemoteBeforeSuiteStateDisappeared -) - -type RemoteBeforeSuiteData struct { - Data []byte - State RemoteBeforeSuiteState -} - -func (r RemoteBeforeSuiteData) ToJSON() []byte { - data, _ := json.Marshal(r) - return data -} - -type RemoteAfterSuiteData struct { - CanRun bool -} diff --git a/go-controller/vendor/github.com/onsi/ginkgo/types/types.go b/go-controller/vendor/github.com/onsi/ginkgo/types/types.go deleted file mode 100644 index c143e02d84..0000000000 --- a/go-controller/vendor/github.com/onsi/ginkgo/types/types.go +++ /dev/null @@ -1,174 +0,0 @@ -package types - -import ( - "strconv" - "time" -) - -const GINKGO_FOCUS_EXIT_CODE = 197 - -/* -SuiteSummary represents the a summary of the test suite and is passed to both -Reporter.SpecSuiteWillBegin -Reporter.SpecSuiteDidEnd - -this is unfortunate as these two methods should receive different objects. When running in parallel -each node does not deterministically know how many specs it will end up running. - -Unfortunately making such a change would break backward compatibility. - -Until Ginkgo 2.0 comes out we will continue to reuse this struct but populate unknown fields -with -1. -*/ -type SuiteSummary struct { - SuiteDescription string - SuiteSucceeded bool - SuiteID string - - NumberOfSpecsBeforeParallelization int - NumberOfTotalSpecs int - NumberOfSpecsThatWillBeRun int - NumberOfPendingSpecs int - NumberOfSkippedSpecs int - NumberOfPassedSpecs int - NumberOfFailedSpecs int - // Flaked specs are those that failed initially, but then passed on a - // subsequent try. - NumberOfFlakedSpecs int - RunTime time.Duration -} - -type SpecSummary struct { - ComponentTexts []string - ComponentCodeLocations []CodeLocation - - State SpecState - RunTime time.Duration - Failure SpecFailure - IsMeasurement bool - NumberOfSamples int - Measurements map[string]*SpecMeasurement - - CapturedOutput string - SuiteID string -} - -func (s SpecSummary) HasFailureState() bool { - return s.State.IsFailure() -} - -func (s SpecSummary) TimedOut() bool { - return s.State == SpecStateTimedOut -} - -func (s SpecSummary) Panicked() bool { - return s.State == SpecStatePanicked -} - -func (s SpecSummary) Failed() bool { - return s.State == SpecStateFailed -} - -func (s SpecSummary) Passed() bool { - return s.State == SpecStatePassed -} - -func (s SpecSummary) Skipped() bool { - return s.State == SpecStateSkipped -} - -func (s SpecSummary) Pending() bool { - return s.State == SpecStatePending -} - -type SetupSummary struct { - ComponentType SpecComponentType - CodeLocation CodeLocation - - State SpecState - RunTime time.Duration - Failure SpecFailure - - CapturedOutput string - SuiteID string -} - -type SpecFailure struct { - Message string - Location CodeLocation - ForwardedPanic string - - ComponentIndex int - ComponentType SpecComponentType - ComponentCodeLocation CodeLocation -} - -type SpecMeasurement struct { - Name string - Info interface{} - Order int - - Results []float64 - - Smallest float64 - Largest float64 - Average float64 - StdDeviation float64 - - SmallestLabel string - LargestLabel string - AverageLabel string - Units string - Precision int -} - -func (s SpecMeasurement) PrecisionFmt() string { - if s.Precision == 0 { - return "%f" - } - - str := strconv.Itoa(s.Precision) - - return "%." + str + "f" -} - -type SpecState uint - -const ( - SpecStateInvalid SpecState = iota - - SpecStatePending - SpecStateSkipped - SpecStatePassed - SpecStateFailed - SpecStatePanicked - SpecStateTimedOut -) - -func (state SpecState) IsFailure() bool { - return state == SpecStateTimedOut || state == SpecStatePanicked || state == SpecStateFailed -} - -type SpecComponentType uint - -const ( - SpecComponentTypeInvalid SpecComponentType = iota - - SpecComponentTypeContainer - SpecComponentTypeBeforeSuite - SpecComponentTypeAfterSuite - SpecComponentTypeBeforeEach - SpecComponentTypeJustBeforeEach - SpecComponentTypeJustAfterEach - SpecComponentTypeAfterEach - SpecComponentTypeIt - SpecComponentTypeMeasure -) - -type FlagType uint - -const ( - FlagTypeNone FlagType = iota - FlagTypeFocused - FlagTypePending -) diff --git a/test/Makefile b/test/Makefile index 8f98c54b8a..4a543cb0c1 100644 --- a/test/Makefile +++ b/test/Makefile @@ -45,3 +45,8 @@ conformance: .PHONY: tools tools: ./scripts/test-ovnkube-trace.sh + +.PHONY: traffic-flow-tests +traffic-flow-tests: + TRAFFIC_FLOW_TESTS=$(TRAFFIC_FLOW_TESTS) \ + ./scripts/traffic-flow-tests.sh $(WHAT) diff --git a/test/e2e/egressip.go b/test/e2e/egressip.go index ae061bd30c..5e54eaacd5 100644 --- a/test/e2e/egressip.go +++ b/test/e2e/egressip.go @@ -12,17 +12,19 @@ import ( "sort" "strconv" "strings" + "time" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + nadclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" "github.com/onsi/ginkgo/v2" "github.com/onsi/ginkgo/v2/dsl/table" "github.com/onsi/gomega" - - nadclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" e2enode "k8s.io/kubernetes/test/e2e/framework/node" @@ -40,6 +42,7 @@ const ( secondaryIPV4Subnet = "10.10.10.0/24" secondaryIPV6Subnet = "2001:db8:abcd:1234::/64" secondaryNetworkName = "secondary-network" + httpdContainerImageName = "docker.io/httpd:latest" ) func labelNodeForEgress(f *framework.Framework, nodeName string) { @@ -212,7 +215,7 @@ func configNetworkAndGetTarget(subnet string, nodesToAttachNet []string, v6 bool for _, nodeName := range nodesToAttachNet { attachNetwork(secondaryNetworkName, nodeName) } - v4Addr, v6Addr := createClusterExternalContainer(targetSecondaryNode.name, "docker.io/httpd", []string{"--network", secondaryNetworkName, "-P"}, []string{}) + v4Addr, v6Addr := createClusterExternalContainer(targetSecondaryNode.name, httpdContainerImageName, []string{"--network", secondaryNetworkName, "-P"}, []string{}) if v4Addr == "" && !v6 { panic("failed to get v4 address") } @@ -295,7 +298,9 @@ func targetExternalContainerAndTest(targetNode node, podName, podNamespace strin } } -var _ = ginkgo.DescribeTableSubtree("e2e egress IP validation", func(netConfigParams networkAttachmentConfigParams, isHostNetwork bool) { +var _ = ginkgo.DescribeTableSubtree("e2e egress IP validation", func(netConfigParams networkAttachmentConfigParams) { + //FIXME: tests for CDN are designed for single stack clusters (IPv4 or IPv6) and must choose a single IP family for dual stack clusters. + // Remove this restriction and allow the tests to detect if an IP family support is available. const ( servicePort int32 = 9999 echoServerPodPortMin = 9900 @@ -321,6 +326,7 @@ var _ = ginkgo.DescribeTableSubtree("e2e egress IP validation", func(netConfigPa pod1Name = "e2e-egressip-pod-1" pod2Name = "e2e-egressip-pod-2" usedEgressNodeAvailabilityHandler egressNodeAvailabilityHandler + isIPv6TestRun bool ) targetPodAndTest := func(namespace, fromName, toName, toIP string) wait.ConditionFunc { @@ -497,19 +503,137 @@ var _ = ginkgo.DescribeTableSubtree("e2e egress IP validation", func(netConfigPa return v4, v6 } + getNodesInternalAddresses := func(nodes *corev1.NodeList, family corev1.IPFamily) []string { + ips := make([]string, 0, 3) + for _, node := range nodes.Items { + ips = append(ips, e2enode.GetAddressesByTypeAndFamily(&node, corev1.NodeInternalIP, family)...) + } + return ips + } + + isNodeInternalAddressesPresentForIPFamily := func(nodes *corev1.NodeList, ipFamily corev1.IPFamily) bool { + if len(getNodesInternalAddresses(nodes, ipFamily)) > 0 { + return true + } + return false + } + + isNetworkSupported := func(nodes *corev1.NodeList, netConfigParams networkAttachmentConfigParams) (bool, string) { + // cluster default network + if netConfigParams.networkName == types.DefaultNetworkName { + return true, "cluster default network is always supported" + } + // user defined networks + if !isNetworkSegmentationEnabled() { + return false, "network segmentation is disabled. Environment variable 'ENABLE_NETWORK_SEGMENTATION' must have value true" + } + if !isInterconnectEnabled() { + return false, "interconnect is disabled. Environment variable 'OVN_ENABLE_INTERCONNECT' must have value true" + } + if netConfigParams.topology == types.LocalnetTopology { + return false, "unsupported network topology" + } + if netConfigParams.cidr == "" { + return false, "UDN network must have subnet specified" + } + if utilnet.IsIPv4CIDRString(netConfigParams.cidr) && !isNodeInternalAddressesPresentForIPFamily(nodes, corev1.IPv4Protocol) { + return false, "cluster must have IPv4 Node internal address" + } + if utilnet.IsIPv6CIDRString(netConfigParams.cidr) && !isNodeInternalAddressesPresentForIPFamily(nodes, corev1.IPv6Protocol) { + return false, "cluster must have IPv6 Node internal address" + } + return true, "network is supported" + } + + getNodeIPs := func(nodes *corev1.NodeList, netConfigParams networkAttachmentConfigParams) []string { + isIPv4Cluster := isNodeInternalAddressesPresentForIPFamily(nodes, corev1.IPv4Protocol) + isIPv6Cluster := isNodeInternalAddressesPresentForIPFamily(nodes, corev1.IPv6Protocol) + var ipFamily corev1.IPFamily + // cluster default network + if netConfigParams.networkName == types.DefaultNetworkName { + // we do not create a CDN, we utilize the network within the cluster. + // The current e2e tests assume a single stack, therefore if dual stack, default to IPv4 + // until the tests are refactored to accommodate dual stack. + if isIPv6Cluster { + ipFamily = corev1.IPv6Protocol + } + if isIPv4Cluster { + ipFamily = corev1.IPv4Protocol + } + } else { + // user defined network + if netConfigParams.cidr == "" { + framework.Failf("network config must have subnet defined") + } + if utilnet.IsIPv4CIDRString(netConfigParams.cidr) && isIPv4Cluster { + ipFamily = corev1.IPv4Protocol + } + if utilnet.IsIPv6CIDRString(netConfigParams.cidr) && isIPv6Cluster { + ipFamily = corev1.IPv6Protocol + } + } + if ipFamily == corev1.IPFamilyUnknown { + framework.Failf("network config is not supported by the cluster") + } + return getNodesInternalAddresses(nodes, ipFamily) + } + + getPodIPWithRetry := func(clientSet clientset.Interface, v6 bool, namespace, name string) (net.IP, error) { + var srcPodIP net.IP + err := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { + pod, err := clientSet.CoreV1().Pods(namespace).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + return false, err + } + ips, err := util.DefaultNetworkPodIPs(pod) + if err != nil { + return false, err + } + srcPodIP, err = util.MatchFirstIPFamily(isIPv6TestRun, ips) + if err != nil { + return false, err + } + return true, nil + }) + if err != nil || srcPodIP == nil { + return srcPodIP, fmt.Errorf("unable to fetch pod %s/%s IP after retrying: %v", namespace, name, err) + } + return srcPodIP, nil + } + + isUserDefinedNetwork := func(netParams networkAttachmentConfigParams) bool { + if netParams.networkName == types.DefaultNetworkName { + return false + } + return true + } + + isClusterDefaultNetwork := func(netParams networkAttachmentConfigParams) bool { + if netParams.networkName == types.DefaultNetworkName { + return true + } + return false + } + f := wrappedTestFramework(egressIPName) // Determine what mode the CI is running in and get relevant endpoint information for the tests ginkgo.BeforeEach(func() { - if !isNetworkSegmentationEnabled() && netConfigParams.networkName != types.DefaultNetworkName { - ginkgo.Skip("Network segmentation is disabled") - } nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) framework.ExpectNoError(err) if len(nodes.Items) < 3 { framework.Failf("Test requires >= 3 Ready nodes, but there are only %v nodes", len(nodes.Items)) } - ips := e2enode.CollectAddresses(nodes, corev1.NodeInternalIP) + if isSupported, reason := isNetworkSupported(nodes, netConfigParams); !isSupported { + ginkgo.Skip(reason) + } + // tests are configured to introspect the Nodes Internal IP address family and then create an EgressIP of + // the same IP family. If dual stack, we default to IPv4 because the tests aren't configured to handle dual stack. + ips := getNodeIPs(nodes, netConfigParams) + if len(ips) == 0 { + framework.Failf("expect at least one IP address") + } + isIPv6TestRun = utilnet.IsIPv6String(ips[0]) egress1Node = node{ name: nodes.Items[1].Name, nodeIP: ips[1], @@ -537,13 +661,13 @@ var _ = ginkgo.DescribeTableSubtree("e2e egress IP validation", func(netConfigPa } isV6 := utilnet.IsIPv6String(egress1Node.nodeIP) if isV6 { - _, targetNode.nodeIP = createClusterExternalContainer(targetNode.name, "docker.io/httpd", []string{"--network", ciNetworkName, "-P"}, []string{}) - _, deniedTargetNode.nodeIP = createClusterExternalContainer(deniedTargetNode.name, "docker.io/httpd", []string{"--network", ciNetworkName, "-P"}, []string{}) + _, targetNode.nodeIP = createClusterExternalContainer(targetNode.name, httpdContainerImageName, []string{"--network", ciNetworkName, "-P"}, []string{}) + _, deniedTargetNode.nodeIP = createClusterExternalContainer(deniedTargetNode.name, httpdContainerImageName, []string{"--network", ciNetworkName, "-P"}, []string{}) // configure and add additional network to worker containers for EIP multi NIC feature _, targetSecondaryNode.nodeIP = configNetworkAndGetTarget(secondaryIPV6Subnet, []string{egress1Node.name, egress2Node.name}, isV6, targetSecondaryNode) } else { - targetNode.nodeIP, _ = createClusterExternalContainer(targetNode.name, "docker.io/httpd", []string{"--network", ciNetworkName, "-P"}, []string{}) - deniedTargetNode.nodeIP, _ = createClusterExternalContainer(deniedTargetNode.name, "docker.io/httpd", []string{"--network", ciNetworkName, "-P"}, []string{}) + targetNode.nodeIP, _ = createClusterExternalContainer(targetNode.name, httpdContainerImageName, []string{"--network", ciNetworkName, "-P"}, []string{}) + deniedTargetNode.nodeIP, _ = createClusterExternalContainer(deniedTargetNode.name, httpdContainerImageName, []string{"--network", ciNetworkName, "-P"}, []string{}) // configure and add additional network to worker containers for EIP multi NIC feature targetSecondaryNode.nodeIP, _ = configNetworkAndGetTarget(secondaryIPV4Subnet, []string{egress1Node.name, egress2Node.name}, isV6, targetSecondaryNode) } @@ -552,14 +676,14 @@ var _ = ginkgo.DescribeTableSubtree("e2e egress IP validation", func(netConfigPa for _, node := range nodes.Items { setNodeReady(node.Name, true) setNodeReachable("iptables", node.Name, true) - if IsIPv6Cluster(f.ClientSet) { + if isV6 { setNodeReachable("ip6tables", node.Name, true) } waitForNoTaint(node.Name, "node.kubernetes.io/unreachable") waitForNoTaint(node.Name, "node.kubernetes.io/not-ready") } // no further network creation is required if CDN - if netConfigParams.networkName == types.DefaultNetworkName { + if isClusterDefaultNetwork(netConfigParams) { return } // configure UDN @@ -576,8 +700,13 @@ var _ = ginkgo.DescribeTableSubtree("e2e egress IP validation", func(netConfigPa }) ginkgo.AfterEach(func() { - if !isNetworkSegmentationEnabled() && netConfigParams.networkName != types.DefaultNetworkName { - ginkgo.Skip("Network segmentation is disabled") + nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) + framework.ExpectNoError(err) + if len(nodes.Items) < 3 { + framework.Failf("Test requires >= 3 Ready nodes, but there are only %v nodes", len(nodes.Items)) + } + if isSupported, reason := isNetworkSupported(nodes, netConfigParams); !isSupported { + ginkgo.Skip(reason) } e2ekubectl.RunKubectlOrDie("default", "delete", "eip", egressIPName, "--ignore-not-found=true") e2ekubectl.RunKubectlOrDie("default", "delete", "eip", egressIPName2, "--ignore-not-found=true") @@ -590,7 +719,7 @@ var _ = ginkgo.DescribeTableSubtree("e2e egress IP validation", func(netConfigPa for _, node := range []string{egress1Node.name, egress2Node.name} { setNodeReady(node, true) setNodeReachable("iptables", node, true) - if IsIPv6Cluster(f.ClientSet) { + if isIPv6TestRun { setNodeReachable("ip6tables", node, true) } waitForNoTaint(node, "node.kubernetes.io/unreachable") @@ -708,7 +837,7 @@ spec: }) framework.ExpectNoError(err, "Step 3. Create two pods matching the EgressIP: one running on each of the egress nodes, failed, err: %v", err) var pod2IP string - if netConfigParams.networkName == types.DefaultNetworkName { + if isClusterDefaultNetwork(netConfigParams) { pod2IP = getPodAddress(pod2Name, f.Namespace.Name) } else { pod2IP, err = podIPsForUserDefinedPrimaryNetwork( @@ -732,8 +861,15 @@ spec: framework.ExpectNoError(err, "Step 5. Check connectivity from one pod to the other and verify that the connection is achieved, failed, err: %v", err) ginkgo.By("6. Check connectivity from both pods to the api-server (running hostNetwork:true) and verifying that the connection is achieved") - err = wait.PollImmediate(retryInterval, retryTimeout, targetDestinationAndTest(podNamespace.Name, fmt.Sprintf("https://%s/version", net.JoinHostPort(getApiAddress(), "443")), []string{pod1Name, pod2Name})) - framework.ExpectNoError(err, "Step 6. Check connectivity from both pods to the api-server (running hostNetwork:true) and verifying that the connection is achieved, failed, err: %v", err) + // CDN exposes either IPv4 and/or IPv6 API endpoint depending on cluster configuration. The network which we are testing may not support this IP family. Skip if unsupported. + apiAddress := getApiAddress() + if utilnet.IsIPv6String(apiAddress) == isIPv6TestRun { + err = wait.PollImmediate(retryInterval, retryTimeout, targetDestinationAndTest(podNamespace.Name, + fmt.Sprintf("https://%s/version", net.JoinHostPort(apiAddress, "443")), []string{pod1Name, pod2Name})) + framework.ExpectNoError(err, "6. Check connectivity from pod to the api-server (running hostNetwork:true) and verifying that the connection is achieved, failed, err: %v", err) + } else { + framework.Logf("Skipping API server reachability check because IP family does not equal IP family of the EgressIP") + } ginkgo.By("7. Update one of the pods, unmatching the EgressIP") pod2 := getPod(f, pod2Name) @@ -814,7 +950,6 @@ spec: 20. Check connectivity from second pod to another node (egress2Node) secondary IP and verify that the srcIP is the expected nodeIP (this verifies SNAT's towards nodeIP are not deleted for pods unless pod is on its own egressNode) */ ginkgo.It("[OVN network] Should validate the egress IP SNAT functionality against host-networked pods", func() { - command := []string{"/agnhost", "netexec", fmt.Sprintf("--http-port=%s", podHTTPPort)} ginkgo.By("0. Add the \"k8s.ovn.org/egress-assignable\" label to egress1Node node") @@ -846,11 +981,21 @@ spec: } ginkgo.By("2. Creating host-networked pod, on non-egress node acting as \"another node\"") - _, err = createPod(f, egress2Node.name+"-host-net-pod", egress2Node.name, f.Namespace.Name, []string{}, map[string]string{}, func(p *corev1.Pod) { + hostNetPodName := egress2Node.name + "-host-net-pod" + p, err := createPod(f, hostNetPodName, egress2Node.name, f.Namespace.Name, []string{}, map[string]string{}, func(p *corev1.Pod) { p.Spec.HostNetwork = true - p.Spec.Containers[0].Image = "docker.io/httpd" + p.Spec.Containers[0].Image = httpdContainerImageName }) framework.ExpectNoError(err) + // block until host network pod is fully deleted because subsequent tests that require binding to the same port may fail + defer func() { + ctxWithTimeout, cancelFn := context.WithTimeout(context.Background(), time.Second*60) + defer cancelFn() + err = pod.DeletePodWithWait(ctxWithTimeout, f.ClientSet, p) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "deletion of host network pod must succeed") + err = pod.WaitForPodNotFoundInNamespace(ctxWithTimeout, f.ClientSet, hostNetPodName, f.Namespace.Name, time.Second*59) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "pod must be fully deleted within 60 seconds") + }() hostNetPod := node{ name: egress2Node.name + "-host-net-pod", nodeIP: egress2Node.nodeIP, @@ -904,15 +1049,7 @@ spec: ginkgo.By("5. Create one pod matching the EgressIP: running on egress1Node") createGenericPodWithLabel(f, pod1Name, pod2Node.name, f.Namespace.Name, command, podEgressLabel) - - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - kubectlOut := getPodAddress(pod1Name, f.Namespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - return true, nil - }) + _, err = getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, pod1Name) framework.ExpectNoError(err, "Step 5. Create one pod matching the EgressIP: running on egress1Node, failed, err: %v", err) framework.Logf("Created pod %s on node %s", pod1Name, pod2Node.name) @@ -959,14 +1096,7 @@ spec: ginkgo.By("15. Create second pod not matching the EgressIP: running on egress1Node") createGenericPodWithLabel(f, pod2Name, pod2Node.name, f.Namespace.Name, command, map[string]string{}) - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - kubectlOut := getPodAddress(pod2Name, f.Namespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - return true, nil - }) + _, err = getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, pod2Name) framework.ExpectNoError(err, "Step 15. Create second pod not matching the EgressIP: running on egress1Node, failed, err: %v", err) framework.Logf("Created pod %s on node %s", pod2Name, pod2Node.name) @@ -1059,15 +1189,7 @@ spec: ginkgo.By("3. Create one pod matching the EgressIP: running on egress1Node") createGenericPodWithLabel(f, pod1Name, pod2Node.name, f.Namespace.Name, command, podEgressLabel) - - err := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - kubectlOut := getPodAddress(pod1Name, f.Namespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - return true, nil - }) + _, err := getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, pod1Name) framework.ExpectNoError(err, "Step 3. Create one pod matching the EgressIP: running on egress1Node, failed, err: %v", err) framework.Logf("Created pod %s on node %s", pod1Name, pod2Node.name) @@ -1084,14 +1206,7 @@ spec: _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "delete", "pod", pod1Name, "--grace-period=0", "--force") framework.ExpectNoError(err, "5. Run %d: Delete the egressPod and recreate it immediately with the same name, failed: %v", i, err) createGenericPodWithLabel(f, pod1Name, nodeSwapName, f.Namespace.Name, command, podEgressLabel) - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - kubectlOut := getPodAddress(pod1Name, f.Namespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - return true, nil - }) + _, err := getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, pod1Name) framework.ExpectNoError(err, "5. Run %d: Delete the egressPod and recreate it immediately with the same name, failed, err: %v", i, err) framework.Logf("Created pod %s on node %s", pod1Name, nodeSwapName) ginkgo.By("6. Check connectivity from pod to an external container and verify that the srcIP is the expected egressIP") @@ -1129,7 +1244,7 @@ spec: 20. Check connectivity from pod to an external container and verify that the srcIP is the expected nodeIP */ ginkgo.It("Should validate egress IP logic when one pod is managed by more than one egressIP object", func() { - if netConfigParams.networkName != types.DefaultNetworkName { + if isUserDefinedNetwork(netConfigParams) { ginkgo.Skip("Unsupported for UDNs") } @@ -1148,16 +1263,7 @@ spec: ginkgo.By("1. Create one pod matching the EgressIP: running on node2 (pod2Node, egress1Node)") createGenericPodWithLabel(f, pod1Name, pod2Node.name, f.Namespace.Name, command, podEgressLabel) - - var srcPodIP net.IP - err := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - kubectlOut := getPodAddress(pod1Name, f.Namespace.Name) - srcPodIP = net.ParseIP(kubectlOut) - if srcPodIP == nil { - return false, nil - } - return true, nil - }) + srcPodIP, err := getPodIPWithRetry(f.ClientSet, isIPv6TestRun, podNamespace.Name, pod1Name) framework.ExpectNoError(err, "Step 1. Create one pod matching the EgressIP: running on node2 (pod2Node, egress1Node), failed, err: %v", err) framework.Logf("Created pod %s on node %s", pod1Name, pod2Node.name) @@ -1288,7 +1394,7 @@ spec: framework.Failf("Error: Check the OVN DB to ensure no SNATs are added for the standby egressIP, err: %v", err) } logicalIP := fmt.Sprintf("logical_ip=%s", srcPodIP.String()) - if IsIPv6Cluster(f.ClientSet) { + if isIPv6TestRun { logicalIP = fmt.Sprintf("logical_ip=\"%s\"", srcPodIP.String()) } snats, err := e2ekubectl.RunKubectl(ovnNamespace, "exec", dbPod, "-c", dbContainerName, "--", "ovn-nbctl", "--no-leader-only", "--columns=external_ip", "find", "nat", logicalIP) @@ -1440,7 +1546,6 @@ spec: ginkgo.By("20. Check connectivity from pod to an external container and verify that the srcIP is the expected nodeIP") err = wait.PollImmediate(retryInterval, retryTimeout, targetExternalContainerAndTest(targetNode, pod1Name, podNamespace.Name, true, []string{pod2Node.nodeIP})) framework.ExpectNoError(err, "Step 20. Check connectivity from pod to an external container and verify that the srcIP is the expected nodeIP, failed: %v", err) - }) /* This test does the following: @@ -1527,7 +1632,7 @@ spec: ginkgo.By(fmt.Sprintf("4. Make egress node: %s unreachable", node1)) setNodeReachable("iptables", node1, false) - if IsIPv6Cluster(f.ClientSet) { + if isIPv6TestRun { setNodeReachable("ip6tables", node1, false) } @@ -1547,12 +1652,17 @@ spec: framework.ExpectNoError(err, "6. Check connectivity from pod to an external \"node\" and verify that the IP is the egress IP, failed, err: %v", err) ginkgo.By("7. Check connectivity from pod to the api-server (running hostNetwork:true) and verifying that the connection is achieved") - err = wait.PollImmediate(retryInterval, retryTimeout, targetDestinationAndTest(podNamespace.Name, fmt.Sprintf("https://%s/version", net.JoinHostPort(getApiAddress(), "443")), []string{pod1Name})) - framework.ExpectNoError(err, "7. Check connectivity from pod to the api-server (running hostNetwork:true) and verifying that the connection is achieved, failed, err: %v", err) - + // CDN exposes either IPv4 and/or IPv6 API endpoint depending on cluster configuration. The network which we are testing may not support this IP family. Skip if unsupported. + apiAddress := getApiAddress() + if utilnet.IsIPv6String(apiAddress) == isIPv6TestRun { + err = wait.PollImmediate(retryInterval, retryTimeout, targetDestinationAndTest(podNamespace.Name, fmt.Sprintf("https://%s/version", net.JoinHostPort(apiAddress, "443")), []string{pod1Name})) + framework.ExpectNoError(err, "7. Check connectivity from pod to the api-server (running hostNetwork:true) and verifying that the connection is achieved, failed, err: %v", err) + } else { + framework.Logf("Skipping API server reachability check because IP family does not equal IP family of the EgressIP") + } ginkgo.By("8, Make node 2 unreachable") setNodeReachable("iptables", node2, false) - if IsIPv6Cluster(f.ClientSet) { + if isIPv6TestRun { setNodeReachable("ip6tables", node2, false) } @@ -1565,7 +1675,7 @@ spec: ginkgo.By("11. Make node 1 reachable again") setNodeReachable("iptables", node1, true) - if IsIPv6Cluster(f.ClientSet) { + if isIPv6TestRun { setNodeReachable("ip6tables", node1, true) } @@ -1581,7 +1691,7 @@ spec: ginkgo.By("14. Make node 2 reachable again") setNodeReachable("iptables", node2, true) - if IsIPv6Cluster(f.ClientSet) { + if isIPv6TestRun { setNodeReachable("ip6tables", node2, true) } @@ -1606,7 +1716,7 @@ spec: ginkgo.By("20. Make node 1 not reachable") setNodeReachable("iptables", node1, false) - if IsIPv6Cluster(f.ClientSet) { + if isIPv6TestRun { setNodeReachable("ip6tables", node1, false) } @@ -1624,7 +1734,7 @@ spec: ginkgo.By("25. Make node 1 reachable again") setNodeReachable("iptables", node1, true) - if IsIPv6Cluster(f.ClientSet) { + if isIPv6TestRun { setNodeReachable("ip6tables", node1, true) } @@ -1656,7 +1766,7 @@ spec: 8. Check connectivity to the service IP and verify that it works */ ginkgo.It("Should validate the egress IP functionality against remote hosts with egress firewall applied", func() { - if netConfigParams.networkName != types.DefaultNetworkName { + if isUserDefinedNetwork(netConfigParams) { ginkgo.Skip("Unsupported for UDNs") } command := []string{"/agnhost", "netexec", fmt.Sprintf("--http-port=%s", podHTTPPort)} @@ -1748,20 +1858,10 @@ spec: createGenericPodWithLabel(f, pod2Name, pod2Node.name, f.Namespace.Name, command, podEgressLabel) serviceIP, err := createServiceForPodsWithLabel(f, f.Namespace.Name, servicePort, podHTTPPort, "ClusterIP", podEgressLabel) framework.ExpectNoError(err, "Step 3. Create two pods, and matching service, matching both egress firewall and egress IP, failed creating service, err: %v", err) - - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - for _, podName := range []string{pod1Name, pod2Name} { - kubectlOut := getPodAddress(podName, f.Namespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - } - return true, nil - }) - framework.ExpectNoError(err, "Step 3. Create two pods matching both egress firewall and egress IP, failed, err: %v", err) - - pod2IP := getPodAddress(pod2Name, f.Namespace.Name) + for _, podName := range []string{pod1Name, pod2Name} { + _, err = getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, podName) + framework.ExpectNoError(err, "Step 3. Create two pods matching both egress firewall and egress IP, failed for pod %s, err: %v", podName, err) + } ginkgo.By("Checking that the status is of length one") verifyEgressIPStatusLengthEquals(1, nil) @@ -1783,7 +1883,9 @@ spec: // framework.ExpectNoError(err, "Step 6. Check connectivity to the kubernetes API IP and verify that it works, failed, err %v", err) ginkgo.By("7. Check connectivity to the other pod IP and verify that it works") - err = wait.PollImmediate(retryInterval, retryTimeout, targetPodAndTest(f.Namespace.Name, pod1Name, pod2Name, pod2IP)) + pod2IP, err := getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, pod2Name) + framework.ExpectNoError(err, "Step 7. Check connectivity to the other pod IP and verify that it works, err retrieving pod %s IP: %v", err, pod2Name) + err = wait.PollImmediate(retryInterval, retryTimeout, targetPodAndTest(f.Namespace.Name, pod1Name, pod2Name, pod2IP.String())) framework.ExpectNoError(err, "Step 7. Check connectivity to the other pod IP and verify that it works, err: %v", err) ginkgo.By("8. Check connectivity to the service IP and verify that it works") @@ -1803,6 +1905,14 @@ spec: // verifies it. // This test is specific to IPv4 LGW mode. ginkgo.It("of replies to egress IP packets that require fragmentation [LGW][IPv4]", func() { + if isIPv6TestRun { + ginkgo.Skip("IPv4 only") + } + if isUserDefinedNetwork(netConfigParams) { + //FIXME: Fragmentation is broken for user defined networks + // Remove when https://issues.redhat.com/browse/OCPBUGS-46476 is resolved + ginkgo.Skip("Fragmentation is not working for user defined networks") + } usedEgressNodeAvailabilityHandler = &egressNodeAvailabilityHandlerViaLabel{f} ginkgo.By("Setting a node as available for egress") @@ -1964,17 +2074,17 @@ spec: 26. Check connectivity from the other pod to an external "node" on the secondary host network and verify the expected src IPs */ table.DescribeTable("[secondary-host-eip] Using different methods to disable a node or pod availability for egress", func(egressIPIP1, egressIPIP2 string) { - if netConfigParams.networkName != types.DefaultNetworkName { + if isUserDefinedNetwork(netConfigParams) { ginkgo.Skip("Unsupported for UDNs") } // get v4, v6 from eips // check that node has both of them v4, v6 := getIPVersions(egressIPIP1, egressIPIP2) - if v4 && utilnet.IsIPv6(net.ParseIP(egress1Node.nodeIP)) { - ginkgo.Skip("Node does not have IPv4 address") + if v4 && isIPv6TestRun { + ginkgo.Skip("IPv4 EIP but IPv6 test run") } - if v6 && !utilnet.IsIPv6(net.ParseIP(egress1Node.nodeIP)) { - ginkgo.Skip("Node does not have IPv6 address") + if v6 && !isIPv6TestRun { + ginkgo.Skip("IPv6 EIP but IPv4 test run") } egressNodeAvailabilityHandler := egressNodeAvailabilityHandlerViaLabel{f} ginkgo.By("0. Set two nodes as available for egress") @@ -2033,24 +2143,17 @@ spec: ginkgo.By("4. Create two pods matching the EgressIP: one running on each of the egress nodes") createGenericPodWithLabel(f, pod1Name, pod1Node.name, f.Namespace.Name, command, podEgressLabel) createGenericPodWithLabel(f, pod2Name, pod2Node.name, f.Namespace.Name, command, podEgressLabel) - err := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - for _, podName := range []string{pod1Name, pod2Name} { - kubectlOut := getPodAddress(podName, f.Namespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - } - return true, nil - }) - framework.ExpectNoError(err, "Step 4. Create two pods matching an EgressIP - running pod(s) failed to get "+ - "their IP(s), failed, err: %v", err) + for _, podName := range []string{pod1Name, pod2Name} { + _, err := getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, podName) + framework.ExpectNoError(err, "Step 4. Create two pods matching an EgressIP - running pod(s) failed to get "+ + "pod %s IP(s), failed, err: %v", podName, err) + } framework.Logf("Created two pods - pod %s on node %s and pod %s on node %s", pod1Name, pod1Node.name, pod2Name, pod2Node.name) ginkgo.By("5. Check connectivity from both pods to an external \"node\" hosted on the secondary host network " + "and verify the expected IPs") - err = wait.PollImmediate(retryInterval, retryTimeout, targetExternalContainerAndTest(targetSecondaryNode, pod1Name, + err := wait.PollImmediate(retryInterval, retryTimeout, targetExternalContainerAndTest(targetSecondaryNode, pod1Name, podNamespace.Name, true, []string{egressIPIP1, egressIPIP2})) framework.ExpectNoError(err, "Step 5. Check connectivity from pod (%s/%s) to an external container attached to "+ "a network that is a secondary host network and verify that the src IP is the expected egressIP, failed: %v", @@ -2061,17 +2164,24 @@ spec: "a network that is a secondary host network and verify that the src IP is the expected egressIP, failed: %v", podNamespace.Name, pod2Name, err) ginkgo.By("6. Check connectivity from one pod to the other and verify that the connection is achieved") - pod2IP := getPodAddress(pod2Name, f.Namespace.Name) - err = wait.PollImmediate(retryInterval, retryTimeout, targetPodAndTest(f.Namespace.Name, pod1Name, pod2Name, pod2IP)) + pod2IP, err := getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, pod2Name) + framework.ExpectNoError(err, "Step 6. Check connectivity from one pod to the other and verify that the connection "+ + "is achieved, failed for pod %s, err: %v", pod2Name, err) + err = wait.PollImmediate(retryInterval, retryTimeout, targetPodAndTest(f.Namespace.Name, pod1Name, pod2Name, pod2IP.String())) framework.ExpectNoError(err, "Step 6. Check connectivity from one pod to the other and verify that the connection "+ "is achieved, failed, err: %v", err) ginkgo.By("7. Check connectivity from both pods to the api-server (running hostNetwork:true) and verifying that " + "the connection is achieved") - err = wait.PollImmediate(retryInterval, retryTimeout, targetDestinationAndTest(podNamespace.Name, - fmt.Sprintf("https://%s/version", net.JoinHostPort(getApiAddress(), "443")), []string{pod1Name, pod2Name})) - framework.ExpectNoError(err, "Step 7. Check connectivity from both pods to the api-server (running hostNetwork:true) "+ - "and verifying that the connection is achieved, failed, err: %v", err) + // CDN exposes either IPv4 and/or IPv6 API endpoint depending on cluster configuration. The network which we are testing may not support this IP family. Skip if unsupported. + apiAddress := getApiAddress() + if utilnet.IsIPv6String(apiAddress) == isIPv6TestRun { + err = wait.PollImmediate(retryInterval, retryTimeout, targetDestinationAndTest(podNamespace.Name, + fmt.Sprintf("https://%s/version", net.JoinHostPort(apiAddress, "443")), []string{pod1Name, pod2Name})) + framework.ExpectNoError(err, "7. Check connectivity from pod to the api-server (running hostNetwork:true) and verifying that the connection is achieved, failed, err: %v", err) + } else { + framework.Logf("Skipping API server reachability check because IP family does not equal IP family of the EgressIP") + } ginkgo.By("8. Update one of the pods, unmatching the EgressIP") pod2 := getPod(f, pod2Name) @@ -2207,7 +2317,7 @@ spec: 28. Check connectivity both pods to an external "node" on the secondary host network and verify the src IP is the expected egressIP */ ginkgo.It("[secondary-host-eip] Using different methods to disable a node or pod availability for egress", func() { - if netConfigParams.networkName != types.DefaultNetworkName { + if isUserDefinedNetwork(netConfigParams) { ginkgo.Skip("Unsupported for UDNs") } if utilnet.IsIPv6(net.ParseIP(egress1Node.nodeIP)) { @@ -2273,24 +2383,17 @@ spec: ginkgo.By("4. Create two pods matching the EgressIP: one running on each of the egress nodes") createGenericPodWithLabel(f, pod1Name, pod1Node.name, f.Namespace.Name, command, podEgressLabel) createGenericPodWithLabel(f, pod2Name, pod2Node.name, f.Namespace.Name, command, podEgressLabel) - err := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - for _, podName := range []string{pod1Name, pod2Name} { - kubectlOut := getPodAddress(podName, f.Namespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - } - return true, nil - }) - framework.ExpectNoError(err, "Step 4. Create two pods matching an EgressIP - running pod(s) failed to get "+ - "their IP(s), failed, err: %v", err) + for _, podName := range []string{pod1Name, pod2Name} { + _, err := getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, podName) + framework.ExpectNoError(err, "Step 4. Create two pods matching an EgressIP - running pod(s) failed to get "+ + "pod %s IP(s), failed, err: %v", podName, err) + } framework.Logf("Created two pods - pod %s on node %s and pod %s on node %s", pod1Name, pod1Node.name, pod2Name, pod2Node.name) ginkgo.By("5. Check connectivity a pod to an external \"node\" hosted on the OVN network " + "and verify the expected IP") - err = wait.PollImmediate(retryInterval, retryTimeout, targetExternalContainerAndTest(targetNode, pod1Name, + err := wait.PollImmediate(retryInterval, retryTimeout, targetExternalContainerAndTest(targetNode, pod1Name, podNamespace.Name, true, []string{egressIPOVN})) framework.ExpectNoError(err, "Step 5. Check connectivity from pod (%s/%s) to an external container attached to "+ "a network that is OVN network and verify that the src IP is the expected egressIP, failed: %v", @@ -2314,17 +2417,24 @@ spec: podNamespace.Name, pod2Name, err) ginkgo.By("7. Check connectivity from one pod to the other and verify that the connection is achieved") - pod2IP := getPodAddress(pod2Name, f.Namespace.Name) - err = wait.PollImmediate(retryInterval, retryTimeout, targetPodAndTest(f.Namespace.Name, pod1Name, pod2Name, pod2IP)) + pod2IP, err := getPodIPWithRetry(f.ClientSet, isIPv6TestRun, podNamespace.Name, pod2Name) + framework.ExpectNoError(err, "Step 7. Check connectivity from one pod to the other and verify that the connection "+ + "is achieved, failed to get Pod %s IP(s), err: %v", pod2Name, err) + err = wait.PollImmediate(retryInterval, retryTimeout, targetPodAndTest(f.Namespace.Name, pod1Name, pod2Name, pod2IP.String())) framework.ExpectNoError(err, "Step 7. Check connectivity from one pod to the other and verify that the connection "+ "is achieved, failed, err: %v", err) ginkgo.By("8. Check connectivity from both pods to the api-server (running hostNetwork:true) and verifying that " + "the connection is achieved") - err = wait.PollImmediate(retryInterval, retryTimeout, targetDestinationAndTest(podNamespace.Name, - fmt.Sprintf("https://%s/version", net.JoinHostPort(getApiAddress(), "443")), []string{pod1Name, pod2Name})) - framework.ExpectNoError(err, "Step 8. Check connectivity from both pods to the api-server (running hostNetwork:true) "+ - "and verifying that the connection is achieved, failed, err: %v", err) + // CDN exposes either IPv4 and/or IPv6 API endpoint depending on cluster configuration. The network which we are testing may not support this IP family. Skip if unsupported. + apiAddress := getApiAddress() + if utilnet.IsIPv6String(apiAddress) == isIPv6TestRun { + err = wait.PollImmediate(retryInterval, retryTimeout, targetDestinationAndTest(podNamespace.Name, + fmt.Sprintf("https://%s/version", net.JoinHostPort(apiAddress, "443")), []string{pod1Name, pod2Name})) + framework.ExpectNoError(err, "8. Check connectivity from pod to the api-server (running hostNetwork:true) and verifying that the connection is achieved, failed, err: %v", err) + } else { + framework.Logf("Skipping API server reachability check because IP family does not equal IP family of the EgressIP") + } ginkgo.By("9. Update one of the pods, unmatching the EgressIP") pod2 := getPod(f, pod2Name) @@ -2461,7 +2571,7 @@ spec: // 6. Check connectivity to the host on the secondary host network from the pod selected by the other EgressIP // 7. Check connectivity to the host on the OVN network from the pod not selected by EgressIP ginkgo.It("[secondary-host-eip] Multiple EgressIP objects and their Egress IP hosted on the same interface", func() { - if netConfigParams.networkName != types.DefaultNetworkName { + if isUserDefinedNetwork(netConfigParams) { ginkgo.Skip("Unsupported for UDNs") } var egressIP1, egressIP2 string @@ -2538,21 +2648,14 @@ spec: "wants": "egress2", } createGenericPodWithLabel(f, pod2Name, pod2Node.name, f.Namespace.Name, command, podEgressLabel2) - err := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - for _, podName := range []string{pod1Name, pod2Name} { - kubectlOut := getPodAddress(podName, f.Namespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - } - return true, nil - }) - framework.ExpectNoError(err, "Step 3. Create two pods - one matching each EgressIP, failed, err: %v", err) + for _, podName := range []string{pod1Name, pod2Name} { + _, err := getPodIPWithRetry(f.ClientSet, isIPv6TestRun, podNamespace.Name, podName) + framework.ExpectNoError(err, "Step 3. Create two pods - one matching each EgressIP, failed for pod %s, err: %v", podName, err) + } ginkgo.By("4. Check connectivity from both pods to an external \"node\" hosted on a secondary host network " + "and verify the expected IPs") - err = wait.PollImmediate(retryInterval, retryTimeout, targetExternalContainerAndTest(targetSecondaryNode, pod1Name, + err := wait.PollImmediate(retryInterval, retryTimeout, targetExternalContainerAndTest(targetSecondaryNode, pod1Name, podNamespace.Name, true, []string{egressIP1})) framework.ExpectNoError(err, "4. Check connectivity from both pods to an external \"node\" hosted on a secondary host network "+ "and verify the expected IPs, failed for EgressIP %s: %v", egressIPName, err) @@ -2587,7 +2690,7 @@ spec: if !isKernelModuleLoaded(egress1Node.name, "vrf") { ginkgo.Skip("Node doesn't have VRF kernel module loaded") } - if netConfigParams.networkName != types.DefaultNetworkName { + if isUserDefinedNetwork(netConfigParams) { ginkgo.Skip("Unsupported for UDNs") } var egressIP1 string @@ -2696,14 +2799,7 @@ spec: verifySpecificEgressIPStatusLengthEquals(egressIPName, 1, nil) ginkgo.By("4. Create a pod matching the EgressIP") createGenericPodWithLabel(f, pod1Name, pod1Node.name, f.Namespace.Name, command, podEgressLabel) - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - kubectlOut := getPodAddress(pod1Name, f.Namespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - return true, nil - }) + _, err = getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, pod1Name) framework.ExpectNoError(err, "Step 4. Create a pod matching the EgressIP, failed, err: %v", err) ginkgo.By("5. Check connectivity from a pod to an external \"node\" hosted on a secondary host network " + "and verify the expected IP") @@ -2716,9 +2812,9 @@ spec: // two pods attached to different namespaces but the same role primary user defined network // One pod is deleted and ensure connectivity for the other pod is ok // The previous pod namespace is deleted and again, ensure connectivity for the other pod is ok - ginkgo.It("[OVN network] multiple namespaces sharing a primary networks", func() { - if !isNetworkSegmentationEnabled() || netConfigParams.role != "primary" { - ginkgo.Skip("Network segmentation is disabled or network isn't a role primary UDN") + ginkgo.It("[OVN network] multiple namespaces sharing a role primary network", func() { + if !isNetworkSegmentationEnabled() || isClusterDefaultNetwork(netConfigParams) { + ginkgo.Skip("network segmentation disabled or unsupported for cluster default network") } ginkgo.By(fmt.Sprintf("Building another namespace api object, basename %s", f.BaseName)) otherNetworkNamespace, err := f.CreateNamespace(context.Background(), f.BaseName, map[string]string{ @@ -2726,7 +2822,7 @@ spec: }) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - ginkgo.By("namespace is connected to UDN, create a namespace attached to this primary as a layer3 UDN") + ginkgo.By(fmt.Sprintf("namespace is connected to UDN, create a namespace attached to this primary as a %s UDN", netConfigParams.topology)) nadClient, err := nadclient.NewForConfig(f.ClientConfig()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) netConfig := newNetworkAttachmentConfig(netConfigParams) @@ -2830,16 +2926,22 @@ spec: ginkgo.DescribeTable("[OVN network] multiple namespaces with different primary networks", func(otherNetworkAttachParms networkAttachmentConfigParams) { if !isNetworkSegmentationEnabled() { - ginkgo.Skip("Network segmentation is disabled") + ginkgo.Skip("network segmentation is disabled") } ginkgo.By(fmt.Sprintf("Building a namespace api object, basename %s", f.BaseName)) otherNetworkNamespace, err := f.CreateNamespace(context.Background(), f.BaseName, map[string]string{ "e2e-framework": f.BaseName, }) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - if netConfigParams.networkName == types.DefaultNetworkName { - ginkgo.By("namespace is connected to CDN, create a namespace with primary as a layer3 UDN") - // create L3 Primary UDN + isOtherNetworkIPv6 := utilnet.IsIPv6CIDRString(otherNetworkAttachParms.cidr) + // The EgressIP IP must match both networks IP family + if isOtherNetworkIPv6 != isIPv6TestRun { + ginkgo.Skip(fmt.Sprintf("Test run IP family (is IPv6: %v) doesn't match other networks IP family (is IPv6: %v)", isIPv6TestRun, isOtherNetworkIPv6)) + } + // is the test namespace a CDN? If so create the UDN + if isClusterDefaultNetwork(netConfigParams) { + ginkgo.By(fmt.Sprintf("namespace is connected to CDN, create a namespace with %s primary UDN", otherNetworkAttachParms.topology)) + // create primary UDN nadClient, err := nadclient.NewForConfig(f.ClientConfig()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) netConfig := newNetworkAttachmentConfig(otherNetworkAttachParms) @@ -2851,7 +2953,7 @@ spec: ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } else { - // if network is L2,L3 or other, then other network is CDN + // if network is L3 or L2 UDN, then other network is CDN } egressNodeAvailabilityHandler := egressNodeAvailabilityHandlerViaLabel{f} ginkgo.By("1. Set one node as available for egress") @@ -2912,26 +3014,10 @@ spec: framework.ExpectNoError(err, "5. Create one pod matching the EgressIP: running on egress1Node, failed: %v", err) _, err = createGenericPodWithLabel(f, pod2Name, pod2Node.name, otherNetworkNamespace.Name, command, podEgressLabel) framework.ExpectNoError(err, "5. Create one pod matching the EgressIP: running on egress2Node, failed: %v", err) - - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - kubectlOut := getPodAddress(pod1Name, f.Namespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - return true, nil - }) + _, err = getPodIPWithRetry(f.ClientSet, isIPv6TestRun, f.Namespace.Name, pod1Name) framework.ExpectNoError(err, "Step 5. Create one pod matching the EgressIP: running on egress1Node, failed, err: %v", err) framework.Logf("Created pod %s on node %s", pod1Name, pod1Node.name) - - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - kubectlOut := getPodAddress(pod2Name, otherNetworkNamespace.Name) - srcIP := net.ParseIP(kubectlOut) - if srcIP == nil { - return false, nil - } - return true, nil - }) + _, err = getPodIPWithRetry(f.ClientSet, isIPv6TestRun, otherNetworkNamespace.Name, pod2Name) framework.ExpectNoError(err, "Step 5. Create one pod matching the EgressIP: running on egress2Node, failed, err: %v", err) framework.Logf("Created pod %s on node %s", pod2Name, pod2Node.name) @@ -2942,28 +3028,74 @@ spec: ginkgo.By("7. Check connectivity from pod connected to a different network and verify that the srcIP is the expected egressIP") err = wait.PollImmediate(retryInterval, retryTimeout, targetExternalContainerAndTest(targetNode, pod2Name, pod2OtherNetworkNamespace, true, []string{egressIP1.String()})) framework.ExpectNoError(err, "Step 7. Check connectivity from pod connected to a different network and verify that the srcIP is the expected nodeIP, failed: %v", err) - }, ginkgo.Entry("L3 Primary UDN", networkAttachmentConfigParams{ - name: "l3primary", - topology: types.Layer3Topology, - cidr: "10.10.0.0/16", - role: "primary", - })) + }, + ginkgo.Entry("IPv4 L3 Primary UDN", networkAttachmentConfigParams{ + name: "l3primary", + topology: types.Layer3Topology, + cidr: "30.10.0.0/16", + role: "primary", + }), + ginkgo.Entry("IPv6 L3 Primary UDN", networkAttachmentConfigParams{ + name: "l3primary", + topology: types.Layer3Topology, + cidr: "2014:100:200::0/60", + }), + ginkgo.Entry("IPv4 L2 Primary UDN", networkAttachmentConfigParams{ + name: "l2primary", + topology: types.Layer2Topology, + cidr: "10.10.0.0/16", + role: "primary", + }), + ginkgo.Entry("IPv6 L2 Primary UDN", networkAttachmentConfigParams{ + name: "l2primary", + topology: types.Layer2Topology, + cidr: "2014:100:200::0/60", + role: "primary", + }), + ) }, ginkgo.Entry( - "L3 CDN", // No UDN attachments + "Cluster Default Network", networkAttachmentConfigParams{ networkName: types.DefaultNetworkName, + topology: types.Layer3Topology, }, - false, ), + // FIXME: fix tests for CDN to specify IPv4 and IPv6 entries in-order to enable testing all IP families on dual stack clusters ginkgo.Entry( - "L3 UDN role primary", + "Network Segmentation: IPv4 L3 role primary", networkAttachmentConfigParams{ - name: "l3primary", + name: "l3primaryv4", topology: types.Layer3Topology, cidr: "10.10.0.0/16", role: "primary", }, - false, + ), + ginkgo.Entry( + "Network Segmentation: IPv6 L3 role primary", + networkAttachmentConfigParams{ + name: "l3primaryv6", + topology: types.Layer3Topology, + cidr: "2014:100:200::0/60", + role: "primary", + }, + ), + ginkgo.Entry( + "Network Segmentation: IPv4 L2 role primary", + networkAttachmentConfigParams{ + name: "l2primary", + topology: types.Layer2Topology, + cidr: "20.10.0.0/16", + role: "primary", + }, + ), + ginkgo.Entry( + "Network Segmentation: IPv6 L2 role primary", + networkAttachmentConfigParams{ + name: "l2primary", + topology: types.Layer2Topology, + cidr: "2015:100:200::0/60", + role: "primary", + }, ), ) diff --git a/test/e2e/kubevirt.go b/test/e2e/kubevirt.go index 187c132ba6..9bbcab6a03 100644 --- a/test/e2e/kubevirt.go +++ b/test/e2e/kubevirt.go @@ -887,7 +887,7 @@ passwd: Eventually(func() error { endpoints, err = dialServiceNodePort(svc) return err - }).WithPolling(time.Second).WithTimeout(20*time.Second).Should(Succeed(), "Should dial service port once service settled") + }).WithPolling(3*time.Second).WithTimeout(60*time.Second).Should(Succeed(), "Should dial service port once service settled") checkConnectivityAndNetworkPolicies(vm.Name, endpoints, "before live migration") // Do just one migration that will fail diff --git a/test/e2e/network_segmentation_services.go b/test/e2e/network_segmentation_services.go index c9b3e1f29c..3eee561a6e 100644 --- a/test/e2e/network_segmentation_services.go +++ b/test/e2e/network_segmentation_services.go @@ -10,6 +10,7 @@ import ( nadclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + kapi "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,12 +36,12 @@ var _ = Describe("Network Segmentation: services", func() { serviceTargetPort = 80 userDefinedNetworkIPv4Subnet = "10.128.0.0/16" userDefinedNetworkIPv6Subnet = "2014:100:200::0/60" + clientContainer = "frr" ) var ( - cs clientset.Interface - nadClient nadclient.K8sCniCncfIoV1Interface - defaultNetNamespace string + cs clientset.Interface + nadClient nadclient.K8sCniCncfIoV1Interface ) BeforeEach(func() { @@ -49,22 +50,6 @@ var _ = Describe("Network Segmentation: services", func() { var err error nadClient, err = nadclient.NewForConfig(f.ClientConfig()) Expect(err).NotTo(HaveOccurred()) - defaultNetNamespace = "" - }) - - cleanupFn := func() { - By("Removing the namespace so all resources get deleted") - err := cs.CoreV1().Namespaces().Delete(context.TODO(), f.Namespace.Name, metav1.DeleteOptions{}) - framework.ExpectNoError(err, "Failed to remove the namespace %s %v", f.Namespace.Name, err) - if defaultNetNamespace != "" { - err = cs.CoreV1().Namespaces().Delete(context.TODO(), defaultNetNamespace, metav1.DeleteOptions{}) - framework.ExpectNoError(err, "Failed to remove the namespace %v", defaultNetNamespace, err) - } - - } - - AfterEach(func() { - cleanupFn() }) DescribeTable( @@ -89,7 +74,7 @@ var _ = Describe("Network Segmentation: services", func() { // + clusterIP fails // + nodeIP:nodePort fails FOR NOW, when we only target the local node - "should be reachable through their cluster IP and node port", + "should be reachable through their cluster IP, node port and load balancer", func( netConfigParams networkAttachmentConfigParams, ) { @@ -122,7 +107,7 @@ var _ = Describe("Network Segmentation: services", func() { ) Expect(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Creating a UDN NodePort service")) + By(fmt.Sprintf("Creating a UDN LoadBalancer service")) policy := v1.IPFamilyPolicyPreferDualStack udnService, err := jig.CreateUDPService(context.TODO(), func(s *v1.Service) { s.Spec.Ports = []v1.ServicePort{ @@ -133,18 +118,28 @@ var _ = Describe("Network Segmentation: services", func() { TargetPort: intstr.FromInt(serviceTargetPort), }, } - s.Spec.Type = v1.ServiceTypeNodePort + s.Spec.Type = v1.ServiceTypeLoadBalancer s.Spec.IPFamilyPolicy = &policy }) framework.ExpectNoError(err) + By("Wait for UDN LoadBalancer Ingress to pop up") + udnService, err = jig.WaitForLoadBalancer(context.TODO(), 180*time.Second) + framework.ExpectNoError(err) + By("Creating a UDN backend pod") udnServerPod := e2epod.NewAgnhostPod( namespace, "backend-pod", nil, nil, []v1.ContainerPort{ {ContainerPort: (serviceTargetPort), Protocol: "UDP"}}, - "netexec", - "--udp-port="+fmt.Sprint(serviceTargetPort)) + "-c", + fmt.Sprintf(` +set -xe +iface=ovn-udn1 +ips=$(ip -o addr show dev $iface| grep global |awk '{print $4}' | cut -d/ -f1 | paste -sd, -) +./agnhost netexec --udp-port=%d --udp-listen-addresses=$ips +`, serviceTargetPort)) + udnServerPod.Spec.Containers[0].Command = []string{"/bin/bash"} udnServerPod.Labels = jig.Labels udnServerPod.Spec.NodeName = serverPodNodeName @@ -159,6 +154,7 @@ var _ = Describe("Network Segmentation: services", func() { By("Connect to the UDN service cluster IP from the UDN client pod on the same node") checkConnectionToClusterIPs(f, udnClientPod, udnService, udnServerPod.Name) By("Connect to the UDN service nodePort on all 3 nodes from the UDN client pod") + checkConnectionToLoadBalancers(f, udnClientPod, udnService, udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[0], "endpoint node", udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[1], "other node", udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[2], "other node", udnServerPod.Name) @@ -169,27 +165,35 @@ var _ = Describe("Network Segmentation: services", func() { By("Connect to the UDN service from the UDN client pod on a different node") checkConnectionToClusterIPs(f, udnClientPod2, udnService, udnServerPod.Name) + checkConnectionToLoadBalancers(f, udnClientPod2, udnService, udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[1], "local node", udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[0], "server node", udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + By("Connect to the UDN service from the UDN client external container") + checkConnectionToLoadBalancersFromExternalContainer(f, clientContainer, udnService, udnServerPod.Name) + checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[0], "server node", udnServerPod.Name) + checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[1], "other node", udnServerPod.Name) + checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + // Default network -> UDN // Check that it cannot connect By(fmt.Sprintf("Create a client pod in the default network on node %s", clientNode)) - defaultNetNamespace = f.Namespace.Name + "-default" - _, err = cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ + defaultNetNamespace := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: defaultNetNamespace, + Name: f.Namespace.Name + "-default", }, - }, metav1.CreateOptions{}) + } + f.AddNamespacesToDelete(defaultNetNamespace) + _, err = cs.CoreV1().Namespaces().Create(context.Background(), defaultNetNamespace, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - defaultClient, err := createPod(f, "default-net-pod", clientNode, defaultNetNamespace, []string{"sleep", "2000000"}, nil) + defaultClient, err := createPod(f, "default-net-pod", clientNode, defaultNetNamespace.Name, []string{"sleep", "2000000"}, nil) Expect(err).NotTo(HaveOccurred()) By("Verify the connection of the client in the default network to the UDN service") checkNoConnectionToClusterIPs(f, defaultClient, udnService) - + checkNoConnectionToLoadBalancers(f, defaultClient, udnService) checkNoConnectionToNodePort(f, defaultClient, udnService, &nodes.Items[1], "local node") // TODO change to checkConnectionToNodePort when we have full UDN support in ovnkube-node checkConnectionToNodePort(f, defaultClient, udnService, &nodes.Items[0], "server node", udnServerPod.Name) @@ -202,7 +206,7 @@ var _ = Describe("Network Segmentation: services", func() { defaultLabels := map[string]string{"app": "default-app"} defaultServerPod, err := createPod(f, "backend-pod-default", serverPodNodeName, - defaultNetNamespace, []string{"/agnhost", "netexec", "--udp-port=" + fmt.Sprint(serviceTargetPort)}, defaultLabels, + defaultNetNamespace.Name, []string{"/agnhost", "netexec", "--udp-port=" + fmt.Sprint(serviceTargetPort)}, defaultLabels, func(pod *v1.Pod) { pod.Spec.Containers[0].Ports = []v1.ContainerPort{{ContainerPort: (serviceTargetPort), Protocol: "UDP"}} }) @@ -226,10 +230,11 @@ var _ = Describe("Network Segmentation: services", func() { }, } - defaultService, err = f.ClientSet.CoreV1().Services(defaultNetNamespace).Create(context.TODO(), defaultService, metav1.CreateOptions{}) + defaultService, err = f.ClientSet.CoreV1().Services(defaultNetNamespace.Name).Create(context.TODO(), defaultService, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) By("Verify the UDN client connection to the default network service") + checkNoConnectionToLoadBalancers(f, udnClientPod2, defaultService) checkConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[0], "server node", defaultServerPod.Name) checkNoConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[1], "local node") checkConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[2], "other node", defaultServerPod.Name) @@ -422,3 +427,74 @@ func checkConnectionOrNoConnectionToNodePort(f *framework.Framework, clientPod * framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg)) } } + +func checkConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string) { + checkConnectionOrNoConnectionToLoadBalancers(f, clientPod, service, expectedOutput, true) +} + +func checkNoConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service) { + checkConnectionOrNoConnectionToLoadBalancers(f, clientPod, service, "", false) +} + +func checkConnectionOrNoConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string, shouldConnect bool) { + var err error + port := service.Spec.Ports[0].Port + notStr := "" + if !shouldConnect { + notStr = "not " + } + for _, lbIngress := range service.Status.LoadBalancer.Ingress { + msg := fmt.Sprintf("Client %s/%s should %sreach service %s/%s on LoadBalancer IP %s port %d", + clientPod.Namespace, clientPod.Name, notStr, service.Namespace, service.Name, lbIngress.IP, port) + By(msg) + + cmd := fmt.Sprintf(`/bin/sh -c 'echo hostname | nc -u -w 1 %s %d '`, lbIngress.IP, port) + + if shouldConnect { + err = checkConnectionToAgnhostPod(f, clientPod, expectedOutput, cmd) + } else { + err = checkNoConnectionToAgnhostPod(f, clientPod, cmd) + } + framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg)) + } +} + +func checkConnectionToNodePortFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, node *v1.Node, nodeRoleMsg, expectedOutput string) { + GinkgoHelper() + var err error + nodePort := service.Spec.Ports[0].NodePort + nodeIPs, err := ParseNodeHostIPDropNetMask(node) + Expect(err).NotTo(HaveOccurred()) + + for nodeIP := range nodeIPs { + msg := fmt.Sprintf("Client at external container %s should connect to NodePort service %s/%s on %s:%d (node %s, %s)", + containerName, service.Namespace, service.Name, nodeIP, nodePort, node.Name, nodeRoleMsg) + By(msg) + cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %d", nodeIP, nodePort)} + Eventually(func() (string, error) { + return runCommand(cmd...) + }). + WithTimeout(5*time.Second). + WithPolling(200*time.Millisecond). + Should(Equal(expectedOutput), "Failed to verify that %s", msg) + } +} + +func checkConnectionToLoadBalancersFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, expectedOutput string) { + GinkgoHelper() + port := service.Spec.Ports[0].Port + + for _, lbIngress := range service.Status.LoadBalancer.Ingress { + msg := fmt.Sprintf("Client at external container %s should reach service %s/%s on LoadBalancer IP %s port %d", + containerName, service.Namespace, service.Name, lbIngress.IP, port) + By(msg) + cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %d", lbIngress.IP, port)} + Eventually(func() (string, error) { + return runCommand(cmd...) + }). + // It takes some time for the container to receive the dynamic routing + WithTimeout(20*time.Second). + WithPolling(200*time.Millisecond). + Should(Equal(expectedOutput), "Failed to verify that %s", msg) + } +} diff --git a/test/scripts/traffic-flow-tests.sh b/test/scripts/traffic-flow-tests.sh new file mode 100755 index 0000000000..c33acd80ef --- /dev/null +++ b/test/scripts/traffic-flow-tests.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Set default values +export KUBECONFIG="${KUBECONFIG:-${HOME}/ovn.conf}" +export OCI_BIN="${KIND_EXPERIMENTAL_PROVIDER:-docker}" +export TFT_TEST_IMAGE="ghcr.io/wizhaoredhat/ocp-traffic-flow-tests:latest" +export TRAFFIC_FLOW_TESTS_DIRNAME="ocp-traffic-flow-tests" +export TRAFFIC_FLOW_TESTS_REPO="https://github.com/wizhaoredhat/ocp-traffic-flow-tests.git" +export TRAFFIC_FLOW_TESTS_COMMIT="eb46da7a3ee1c3cfab5e141f69a3ccd1cdbfbabd" +export TRAFFIC_FLOW_TESTS="${TRAFFIC_FLOW_TESTS:-1,2,3}" + +export SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" +export TOP_DIR="/mnt/runner" +export TRAFFIC_FLOW_TESTS_FULL_PATH="${TOP_DIR}/${TRAFFIC_FLOW_TESTS_DIRNAME}" + +log() { + local level="$1" + shift + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" +} + +error_exit() { + log "ERROR" "$*" + exit 1 +} + +check_dependencies() { + local dependencies=(sed pip git kubectl kind "${OCI_BIN}") + for cmd in "${dependencies[@]}"; do + if ! command -v "${cmd}" &> /dev/null; then + error_exit "Dependency not met: ${cmd}" + fi + done +} + +usage() { + echo "Usage: $(basename "$0") {setup|run}" +} + +create_config_file_defaults() { + local file_name="${1:-config.yaml}" + + cat < "$file_name" +tft: + - name: "Github Workflow Test" + namespace: "default" + test_cases: "TBD" + duration: "10" + connections: + - name: "Connection_1" + type: "iperf-tcp" + instances: 1 + server: + - name: "ovn-worker" + persistent: "false" + sriov: "false" + client: + - name: "ovn-worker2" + sriov: "false" + plugins: [] +kubeconfig: "TBD" +kubeconfig_infra: +EOT +} + +process_test_results() { + local result_file="$1" + + log "INFO" "Processing test results from: ${result_file}" + ./print_results.py ${result_file} || error_exit "Results evaluation failed." +} + +setup() { + check_dependencies + + "${OCI_BIN}" pull "${TFT_TEST_IMAGE}" || error_exit "Failed to pull the test image." + local KIND_CLUSTER_NAME + KIND_CLUSTER_NAME=$(kind get clusters) + kind load docker-image "${TFT_TEST_IMAGE}" --name "${KIND_CLUSTER_NAME}" || error_exit "Failed to load the test image." + + if [ -d "${TRAFFIC_FLOW_TESTS_FULL_PATH}" ]; then + error_exit "Install folder already exists: ${TRAFFIC_FLOW_TESTS_FULL_PATH}" + fi + mkdir -pv "${TRAFFIC_FLOW_TESTS_FULL_PATH}" + cd "${TRAFFIC_FLOW_TESTS_FULL_PATH}" + + git init + git remote add origin "${TRAFFIC_FLOW_TESTS_REPO}" + git fetch --depth=1 origin "${TRAFFIC_FLOW_TESTS_COMMIT}" + git checkout "${TRAFFIC_FLOW_TESTS_COMMIT}" + + python -m venv tft-venv + source tft-venv/bin/activate + pip install --upgrade pip + pip install -r requirements.txt + + local CONFIG_FILE="config.yaml" + create_config_file_defaults "$CONFIG_FILE" + + sed -i "s|^kubeconfig:.*|kubeconfig: \"$KUBECONFIG\"|" "$CONFIG_FILE" + sed -i "s|test_cases:.*|test_cases: \"$TRAFFIC_FLOW_TESTS\"|" "$CONFIG_FILE" + + log "INFO" "Setup complete. Configuration file created: $CONFIG_FILE" + + echo + echo "---" + cat "$CONFIG_FILE" +} + +run() { + cd "${TRAFFIC_FLOW_TESTS_FULL_PATH}" || error_exit "Run folder not found. Missing setup?" + source tft-venv/bin/activate || error_exit "Python environment missing. Missing setup?" + + local OUTPUT_BASE="${TRAFFIC_FLOW_TESTS_FULL_PATH}/ft-logs/result-" + time ./tft.py config.yaml -o "$OUTPUT_BASE" || error_exit "Test execution FAILED." + local RESULT_FILE=$(ls -rt "${OUTPUT_BASE}"*.json | tail -1) + cp -vf "${RESULT_FILE}" /tmp/traffic_flow_test_result.json || error_exit "Unable to locate results.json." + + log "INFO" "Results saved to /tmp/traffic_flow_test_result.json" + + process_test_results "${RESULT_FILE}" +} + +case "${1:-}" in + setup) + setup + ;; + run) + run + ;; + -h|--help) + usage + ;; + *) + usage + exit 1 + ;; +esac