From b20f915bc823b5b201f49303664e316b37ec4ab7 Mon Sep 17 00:00:00 2001 From: Jesse Thompson Date: Tue, 7 Jan 2025 08:16:25 -0700 Subject: [PATCH] Give more time to nightlies (#499) Running our nightlies with race conditions enabled in the integration tests is taking a very long time. This change gives more time to the tests to prevent timing out. --- .github/workflows/nightly-tests.yaml | 2 +- pkg/testutils/alerter/alerter.go | 2 -- pkg/testutils/alerter/alerter_test.go | 2 +- pkg/testutils/collector/collector.go | 8 ++++++-- pkg/testutils/collector/collector_test.go | 2 +- pkg/testutils/ingestor/ingestor.go | 2 -- pkg/testutils/ingestor/ingestor_test.go | 2 +- pkg/testutils/integration_test.go | 22 ++++------------------ 8 files changed, 14 insertions(+), 28 deletions(-) diff --git a/.github/workflows/nightly-tests.yaml b/.github/workflows/nightly-tests.yaml index 7a8abee4..8eb5c8a6 100644 --- a/.github/workflows/nightly-tests.yaml +++ b/.github/workflows/nightly-tests.yaml @@ -24,4 +24,4 @@ jobs: run: sudo apt install -y libsystemd-dev # for collector journal input - name: Test - run: INTEGRATION=1 ROLLBACK=1 go test -race -mod=vendor -v -timeout=30m ./... + run: INTEGRATION=1 ROLLBACK=1 go test -race -mod=vendor -v -timeout=120m ./... diff --git a/pkg/testutils/alerter/alerter.go b/pkg/testutils/alerter/alerter.go index ffde8353..4bc0ebdd 100644 --- a/pkg/testutils/alerter/alerter.go +++ b/pkg/testutils/alerter/alerter.go @@ -32,7 +32,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Aler } req := testcontainers.ContainerRequest{ - Name: "alerter" + testcontainers.SessionID(), FromDockerfile: testcontainers.FromDockerfile{ Repo: DefaultImage, Tag: DefaultTag, @@ -44,7 +43,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Aler genericContainerReq := testcontainers.GenericContainerRequest{ ContainerRequest: req, - Reuse: true, } for _, opt := range opts { diff --git a/pkg/testutils/alerter/alerter_test.go b/pkg/testutils/alerter/alerter_test.go index 0b0b4b75..d741cd30 100644 --- a/pkg/testutils/alerter/alerter_test.go +++ b/pkg/testutils/alerter/alerter_test.go @@ -26,7 +26,7 @@ func TestAlerter(t *testing.T) { func TestInCluster(t *testing.T) { testutils.IntegrationTest(t) - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) t.Cleanup(cancel) k3sContainer, err := k3s.Run(ctx, "rancher/k3s:v1.31.2-k3s1") diff --git a/pkg/testutils/collector/collector.go b/pkg/testutils/collector/collector.go index b5447688..b2a0157d 100644 --- a/pkg/testutils/collector/collector.go +++ b/pkg/testutils/collector/collector.go @@ -17,6 +17,7 @@ import ( "github.com/testcontainers/testcontainers-go/modules/k3s" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kwait "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/yaml" @@ -43,7 +44,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Coll } req := testcontainers.ContainerRequest{ - Name: "collector" + testcontainers.SessionID(), FromDockerfile: testcontainers.FromDockerfile{ Repo: DefaultImage, Tag: DefaultTag, @@ -55,7 +55,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Coll genericContainerReq := testcontainers.GenericContainerRequest{ ContainerRequest: req, - Reuse: true, } for _, opt := range opts { @@ -175,6 +174,11 @@ func WithCluster(ctx context.Context, k *k3s.K3sContainer) testcontainers.Custom } if err := ctrlCli.Create(ctx, collectorFunction); err != nil { + if meta.IsNoMatchError(err) { + // Ingestor installs the CRD, so if we get a no match error, it's because the CRD + // hasn't been installed yet. We can just skip creating the function in this case. + return nil + } return fmt.Errorf("failed to create collector function: %w", err) } diff --git a/pkg/testutils/collector/collector_test.go b/pkg/testutils/collector/collector_test.go index b4609eaf..8a2b3cde 100644 --- a/pkg/testutils/collector/collector_test.go +++ b/pkg/testutils/collector/collector_test.go @@ -17,7 +17,7 @@ import ( func TestCollector(t *testing.T) { testutils.IntegrationTest(t) - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() t.Run("outside cluster", func(t *testing.T) { diff --git a/pkg/testutils/ingestor/ingestor.go b/pkg/testutils/ingestor/ingestor.go index 175f9eeb..a77db62e 100644 --- a/pkg/testutils/ingestor/ingestor.go +++ b/pkg/testutils/ingestor/ingestor.go @@ -42,7 +42,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Inge } req := testcontainers.ContainerRequest{ - Name: "ingestor" + testcontainers.SessionID(), FromDockerfile: testcontainers.FromDockerfile{ Repo: DefaultImage, Tag: DefaultTag, @@ -54,7 +53,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Inge genericContainerReq := testcontainers.GenericContainerRequest{ ContainerRequest: req, - Reuse: true, } for _, opt := range opts { diff --git a/pkg/testutils/ingestor/ingestor_test.go b/pkg/testutils/ingestor/ingestor_test.go index ceeafb94..7aec5d71 100644 --- a/pkg/testutils/ingestor/ingestor_test.go +++ b/pkg/testutils/ingestor/ingestor_test.go @@ -17,7 +17,7 @@ import ( func TestIngestor(t *testing.T) { testutils.IntegrationTest(t) - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() t.Run("outside cluster", func(t *testing.T) { diff --git a/pkg/testutils/integration_test.go b/pkg/testutils/integration_test.go index 457bc57c..cad7fbe3 100644 --- a/pkg/testutils/integration_test.go +++ b/pkg/testutils/integration_test.go @@ -57,7 +57,6 @@ func TestIntegration(t *testing.T) { func StartCluster(ctx context.Context, t *testing.T) (kustoUrl string, k3sContainer *k3s.K3sContainer) { t.Helper() - wg := sync.WaitGroup{} k3sContainer, err := k3s.Run(ctx, "rancher/k3s:v1.31.2-k3s1") testcontainers.CleanupContainer(t, k3sContainer) @@ -76,10 +75,7 @@ func StartCluster(ctx context.Context, t *testing.T) (kustoUrl string, k3sContai t.Logf("Kubeconfig: %s", kubeconfig) t.Logf("Kustainer: %s", kustoContainer.ConnectionUrl()) - wg.Add(1) - go t.Run("Configure Kusto", func(t *testing.T) { - defer wg.Done() - + t.Run("Configure Kusto", func(t *testing.T) { opts := kustainer.IngestionBatchingPolicy{ MaximumBatchingTimeSpan: 30 * time.Second, } @@ -89,28 +85,19 @@ func StartCluster(ctx context.Context, t *testing.T) (kustoUrl string, k3sContai } }) - wg.Add(1) - go t.Run("Build and install Ingestor", func(tt *testing.T) { - defer wg.Done() - + t.Run("Build and install Ingestor and Collector", func(tt *testing.T) { ingestorContainer, err := ingestor.Run(ctx, ingestor.WithCluster(ctx, k3sContainer)) testcontainers.CleanupContainer(t, ingestorContainer) require.NoError(tt, err) }) - wg.Add(1) - go t.Run("Build and install Collector", func(tt *testing.T) { - defer wg.Done() - + t.Run("Build and install Collector", func(tt *testing.T) { collectorContainer, err := collector.Run(ctx, collector.WithCluster(ctx, k3sContainer)) testcontainers.CleanupContainer(t, collectorContainer) require.NoError(tt, err) }) - wg.Add(1) - go t.Run("Build and install Alerter", func(tt *testing.T) { - defer wg.Done() - + t.Run("Build and install Alerter", func(tt *testing.T) { crdPath := filepath.Join(t.TempDir(), "crd.yaml") require.NoError(t, testutils.CopyFile("../../kustomize/bases/alertrules_crd.yaml", crdPath)) require.NoError(t, k3sContainer.CopyFileToContainer(ctx, crdPath, filepath.Join(testutils.K3sManifests, "crd.yaml"), 0644)) @@ -121,7 +108,6 @@ func StartCluster(ctx context.Context, t *testing.T) (kustoUrl string, k3sContai }) kustoUrl = kustoContainer.ConnectionUrl() - wg.Wait() return }