From 7f7898abbc63e9ac51f74b0a1fa41542948df3c1 Mon Sep 17 00:00:00 2001 From: Francesco Cheinasso Date: Mon, 18 Nov 2024 11:06:11 +0100 Subject: [PATCH] nftables monitor disable flag --- .github/workflows/integration.yml | 5 +++-- cmd/fabric/main.go | 2 +- cmd/gateway/main.go | 2 +- deployments/liqo/README.md | 1 + deployments/liqo/templates/liqo-fabric-daemonset.yaml | 1 + deployments/liqo/values.yaml | 5 +++++ docs/contributing/contributing.md | 2 +- docs/installation/install.md | 6 ++++++ pkg/fabric/flags.go | 4 ++++ pkg/fabric/options.go | 3 ++- pkg/firewall/firewallconfiguration_controller.go | 10 ++++++---- pkg/liqoctl/install/k3s/provider.go | 10 +++++++++- 12 files changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bffb9f38e5..d7a431bba9 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -6,6 +6,7 @@ on: branches: - master - release + - frc/nftmonitor repository_dispatch: types: - test-command @@ -32,7 +33,7 @@ jobs: # The ref of the commit to checkout (do not use the merge commit if repository dispatch) if [ "${{ github.event_name }}" == "repository_dispatch" ]; then echo "master=false" >> $GITHUB_OUTPUT - echo "architectures=linux/amd64" >> $GITHUB_OUTPUT + echo "architectures=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT echo "commit-ref=${{ github.event.client_payload.pull_request.head.sha }}" >> $GITHUB_OUTPUT echo "pr-number=${{ github.event.client_payload.github.payload.issue.number }}" >> $GITHUB_OUTPUT elif [ "${{ steps.get_version.outputs.VERSION }}" != "" ]; then @@ -45,7 +46,7 @@ jobs: echo "commit-ref=${{ github.sha }}" >> $GITHUB_OUTPUT else echo "master=false" >> $GITHUB_OUTPUT - echo "architectures=linux/amd64" >> $GITHUB_OUTPUT + echo "architectures=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT echo "commit-ref=${{ github.sha }}" >> $GITHUB_OUTPUT fi diff --git a/cmd/fabric/main.go b/cmd/fabric/main.go index 7cb93a7bb2..d4c8a1f504 100644 --- a/cmd/fabric/main.go +++ b/cmd/fabric/main.go @@ -160,7 +160,7 @@ func run(cmd *cobra.Command, _ []string) error { return fmt.Errorf("unable to create firewall configuration reconciler: %w", err) } - if err := fwcr.SetupWithManager(cmd.Context(), mgr); err != nil { + if err := fwcr.SetupWithManager(cmd.Context(), mgr, options.EnableNftMonitor); err != nil { return fmt.Errorf("unable to setup firewall configuration reconciler: %w", err) } diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index 3bcf045b3d..64c5cff5ad 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -201,7 +201,7 @@ func run(cmd *cobra.Command, _ []string) error { return fmt.Errorf("unable to create firewall configuration reconciler: %w", err) } - if err := fwcr.SetupWithManager(cmd.Context(), mgr); err != nil { + if err := fwcr.SetupWithManager(cmd.Context(), mgr, true); err != nil { return fmt.Errorf("unable to setup firewall configuration reconciler: %w", err) } diff --git a/deployments/liqo/README.md b/deployments/liqo/README.md index aaee2c71ba..27e4d83ba5 100644 --- a/deployments/liqo/README.md +++ b/deployments/liqo/README.md @@ -81,6 +81,7 @@ | networking.enabled | bool | `true` | Use the default Liqo networking module. | | networking.fabric.config.fullMasquerade | bool | `false` | Enabe/Disable the full masquerade mode for the fabric pod. It means that all traffic will be masquerade using the first external cidr IP, instead of using the pod IP. Full masquerade is useful when the cluster nodeports uses a PodCIDR IP to masqerade the incoming traffic. IMPORTANT: Please consider that enabling this feature will masquerade the source IP of traffic towards a remote cluster, making impossible for a pod that receives the traffic to know the original source IP. | | networking.fabric.config.gatewayMasqueradeBypass | bool | `false` | Enable/Disable the masquerade bypass for the gateway pods. It means that the packets from gateway pods will not be masqueraded from the host where the pod is scheduled. This is useful in scenarios where CNIs masquerade the traffic from pod to nodes. For example this is required when using the Azure CNI or Kindnet. | +| networking.fabric.config.nftablesMonitor | bool | `true` | Enable/Disable the nftables monitor for the fabric pod. It means that the fabric pod will monitor the nftables rules and will restore them in case of changes. In some cases (like K3S), this monitor can cause a huge amount of CPU usage. If you are experiencing high CPU usage, you can disable this feature. | | networking.fabric.image.name | string | `"ghcr.io/liqotech/fabric"` | Image repository for the fabric pod. | | networking.fabric.image.version | string | `""` | Custom version for the fabric image. If not specified, the global tag is used. | | networking.fabric.pod.annotations | object | `{}` | Annotations for the fabric pod. | diff --git a/deployments/liqo/templates/liqo-fabric-daemonset.yaml b/deployments/liqo/templates/liqo-fabric-daemonset.yaml index ec744650c7..bf800f904e 100644 --- a/deployments/liqo/templates/liqo-fabric-daemonset.yaml +++ b/deployments/liqo/templates/liqo-fabric-daemonset.yaml @@ -48,6 +48,7 @@ spec: {{- if .Values.requirements.kernel.disabled }} - --disable-kernel-version-check {{- end }} + - --enable-nft-monitor={{ .Values.networking.fabric.config.nftablesMonitor }} {{- if .Values.common.extraArgs }} {{- toYaml .Values.common.extraArgs | nindent 10 }} {{- end }} diff --git a/deployments/liqo/values.yaml b/deployments/liqo/values.yaml index 7058fc0bb0..15031fc8ff 100644 --- a/deployments/liqo/values.yaml +++ b/deployments/liqo/values.yaml @@ -130,6 +130,11 @@ networking: # This is useful in scenarios where CNIs masquerade the traffic from pod to nodes. # For example this is required when using the Azure CNI or Kindnet. gatewayMasqueradeBypass: false + # -- Enable/Disable the nftables monitor for the fabric pod. + # It means that the fabric pod will monitor the nftables rules and will restore them in case of changes. + # In some cases (like K3S), this monitor can cause a huge amount of CPU usage. + # If you are experiencing high CPU usage, you can disable this feature. + nftablesMonitor: true authentication: # -- Enable/Disable the authentication module. diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md index 3e40de9f5d..cdd951ef11 100644 --- a/docs/contributing/contributing.md +++ b/docs/contributing/contributing.md @@ -137,4 +137,4 @@ When executing the unit tests from the *liqo-test* container, it is possible to --accept-multiclient ./path/to/test/directory ``` -4. From the host, connect to *localhost:2345* with your remote debugging client of choice (e.g. [GoLand](https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html#step-3-create-the-remote-run-debug-configuration-on-the-client-computer)), and enjoy! +4. From the host, connect to *localhost:2345* with your remote debugging client of choice, and enjoy! diff --git a/docs/installation/install.md b/docs/installation/install.md index 4d14297c24..5febb3ac3c 100644 --- a/docs/installation/install.md +++ b/docs/installation/install.md @@ -303,6 +303,12 @@ By default, the K3s installer stores the kubeconfig to access your cluster in th Make sure to properly refer to it when using *liqoctl* (e.g., setting the `KUBECONFIG` variable), and that the current user has permissions to read it. ``` +```{warning} +- Due to an issue with K3s certificates, the `kubectl exec' command doesn't work properly when used on a pod scheduled on a virtual node. +- Due to an issue with the [nftables golang library](https://github.com/google/nftables) and the pod running in *host network* in K3s, the firewall monitoring feature is disabled by default. +This means that the firewall rules on the node will not be monitored and enforced by Liqo. If these rules are deleted or changed, Liqo won't restore them. +``` + **Installation** Liqo can be installed on a K3s cluster with the following command: diff --git a/pkg/fabric/flags.go b/pkg/fabric/flags.go index 07e4e87d04..5e5c8d38c1 100644 --- a/pkg/fabric/flags.go +++ b/pkg/fabric/flags.go @@ -40,6 +40,9 @@ const ( // FlagNameDisableARP is the flag to enable ARP. FlagNameDisableARP FlagName = "disable-arp" + // FlagNameEnableNftMonitor is the flag to enable the nftables monitor. + FlagNameEnableNftMonitor FlagName = "enable-nft-monitor" + // FlagNameDisableKernelVersionCheck is the flag to enable the kernel version check. FlagNameDisableKernelVersionCheck FlagName = "disable-kernel-version-check" // FlagNameMinimumKernelVersion is the minimum kernel version required to run the wireguard interface. @@ -63,6 +66,7 @@ func InitFlags(flagset *pflag.FlagSet, opts *Options) { flagset.StringVar(&opts.ProbeAddr, FlagNameProbeAddr.String(), ":8081", "Address for the health probe endpoint") flagset.BoolVar(&opts.DisableARP, FlagNameDisableARP.String(), false, "Disable ARP") + flagset.BoolVar(&opts.EnableNftMonitor, FlagNameEnableNftMonitor.String(), true, "Enable nftables monitor") flagset.BoolVar(&opts.DisableKernelVersionCheck, FlagNameDisableKernelVersionCheck.String(), false, "Disable the kernel version check") flagset.Var(&opts.MinimumKernelVersion, string(FlagNameMinimumKernelVersion), "Minimum kernel version required to run the wireguard interface") diff --git a/pkg/fabric/options.go b/pkg/fabric/options.go index 41a2febf51..4b54af3b9f 100644 --- a/pkg/fabric/options.go +++ b/pkg/fabric/options.go @@ -26,7 +26,8 @@ type Options struct { MetricsAddress string ProbeAddr string - DisableARP bool + DisableARP bool + EnableNftMonitor bool DisableKernelVersionCheck bool MinimumKernelVersion kernelversion.KernelVersion diff --git a/pkg/firewall/firewallconfiguration_controller.go b/pkg/firewall/firewallconfiguration_controller.go index 5c730cfb8a..fed1d13c77 100644 --- a/pkg/firewall/firewallconfiguration_controller.go +++ b/pkg/firewall/firewallconfiguration_controller.go @@ -162,7 +162,7 @@ func (r *FirewallConfigurationReconciler) Reconcile(ctx context.Context, req ctr } // SetupWithManager register the FirewallConfigurationReconciler to the manager. -func (r *FirewallConfigurationReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { +func (r *FirewallConfigurationReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, enableNftMonitor bool) error { klog.Infof("Starting FirewallConfiguration controller with labels %v", r.LabelsSets) filterByLabelsPredicate, err := forgeLabelsPredicate(r.LabelsSets) if err != nil { @@ -170,9 +170,11 @@ func (r *FirewallConfigurationReconciler) SetupWithManager(ctx context.Context, } src := make(chan event.GenericEvent) - go func() { - utilruntime.Must(netmonitor.InterfacesMonitoring(ctx, src, &netmonitor.Options{Nftables: &netmonitor.OptionsNftables{Delete: true}})) - }() + if enableNftMonitor { + go func() { + utilruntime.Must(netmonitor.InterfacesMonitoring(ctx, src, &netmonitor.Options{Nftables: &netmonitor.OptionsNftables{Delete: true}})) + }() + } return ctrl.NewControllerManagedBy(mgr).Named(consts.CtrlFirewallConfiguration). For(&networkingv1beta1.FirewallConfiguration{}, builder.WithPredicates(filterByLabelsPredicate)). WatchesRawSource(NewFirewallWatchSource(src, NewFirewallWatchEventHandler(r.Client, r.LabelsSets))). diff --git a/pkg/liqoctl/install/k3s/provider.go b/pkg/liqoctl/install/k3s/provider.go index b1dbef391e..ab1f9161ba 100644 --- a/pkg/liqoctl/install/k3s/provider.go +++ b/pkg/liqoctl/install/k3s/provider.go @@ -67,5 +67,13 @@ func (o *Options) Initialize(_ context.Context) error { // Values returns the customized provider-specifc values file parameters. func (o *Options) Values() map[string]interface{} { - return map[string]interface{}{} + return map[string]interface{}{ + "networking": map[string]interface{}{ + "fabric": map[string]interface{}{ + "config": map[string]interface{}{ + "nftablesMonitor": false, + }, + }, + }, + } }