Give more time to nightlies (#499)

Running our nightlies with race conditions enabled in the integration tests is taking a very long time. This change gives more time to the tests to prevent timing out.
Azure · Jan 7, 2025 · b20f915 · b20f915
1 parent 9577216
commit b20f915
Show file tree

Hide file tree

Showing 8 changed files with 14 additions and 28 deletions.
diff --git a/.github/workflows/nightly-tests.yaml b/.github/workflows/nightly-tests.yaml
@@ -24,4 +24,4 @@ jobs:
       run: sudo apt install -y libsystemd-dev # for collector journal input
 
     - name: Test
-      run: INTEGRATION=1 ROLLBACK=1 go test -race -mod=vendor -v -timeout=30m ./...
+      run: INTEGRATION=1 ROLLBACK=1 go test -race -mod=vendor -v -timeout=120m ./...
diff --git a/pkg/testutils/alerter/alerter.go b/pkg/testutils/alerter/alerter.go
@@ -32,7 +32,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Aler
 	}
 
 	req := testcontainers.ContainerRequest{
-		Name: "alerter" + testcontainers.SessionID(),
 		FromDockerfile: testcontainers.FromDockerfile{
 			Repo:       DefaultImage,
 			Tag:        DefaultTag,
@@ -44,7 +43,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Aler
 
 	genericContainerReq := testcontainers.GenericContainerRequest{
 		ContainerRequest: req,
-		Reuse:            true,
 	}
 
 	for _, opt := range opts {

diff --git a/pkg/testutils/alerter/alerter_test.go b/pkg/testutils/alerter/alerter_test.go
@@ -26,7 +26,7 @@ func TestAlerter(t *testing.T) {
 func TestInCluster(t *testing.T) {
 	testutils.IntegrationTest(t)
 
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	t.Cleanup(cancel)
 
 	k3sContainer, err := k3s.Run(ctx, "rancher/k3s:v1.31.2-k3s1")

diff --git a/pkg/testutils/collector/collector.go b/pkg/testutils/collector/collector.go
@@ -17,6 +17,7 @@ import (
 	"github.com/testcontainers/testcontainers-go/modules/k3s"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kwait "k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/util/yaml"
@@ -43,7 +44,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Coll
 	}
 
 	req := testcontainers.ContainerRequest{
-		Name: "collector" + testcontainers.SessionID(),
 		FromDockerfile: testcontainers.FromDockerfile{
 			Repo:       DefaultImage,
 			Tag:        DefaultTag,
@@ -55,7 +55,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Coll
 
 	genericContainerReq := testcontainers.GenericContainerRequest{
 		ContainerRequest: req,
-		Reuse:            true,
 	}
 
 	for _, opt := range opts {
@@ -175,6 +174,11 @@ func WithCluster(ctx context.Context, k *k3s.K3sContainer) testcontainers.Custom
 					}
 
 					if err := ctrlCli.Create(ctx, collectorFunction); err != nil {
+						if meta.IsNoMatchError(err) {
+							// Ingestor installs the CRD, so if we get a no match error, it's because the CRD
+							// hasn't been installed yet. We can just skip creating the function in this case.
+							return nil
+						}
 						return fmt.Errorf("failed to create collector function: %w", err)
 					}
 

diff --git a/pkg/testutils/collector/collector_test.go b/pkg/testutils/collector/collector_test.go
@@ -17,7 +17,7 @@ import (
 func TestCollector(t *testing.T) {
 	testutils.IntegrationTest(t)
 
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
 
 	t.Run("outside cluster", func(t *testing.T) {

diff --git a/pkg/testutils/ingestor/ingestor.go b/pkg/testutils/ingestor/ingestor.go
@@ -42,7 +42,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Inge
 	}
 
 	req := testcontainers.ContainerRequest{
-		Name: "ingestor" + testcontainers.SessionID(),
 		FromDockerfile: testcontainers.FromDockerfile{
 			Repo:       DefaultImage,
 			Tag:        DefaultTag,
@@ -54,7 +53,6 @@ func Run(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*Inge
 
 	genericContainerReq := testcontainers.GenericContainerRequest{
 		ContainerRequest: req,
-		Reuse:            true,
 	}
 
 	for _, opt := range opts {

diff --git a/pkg/testutils/ingestor/ingestor_test.go b/pkg/testutils/ingestor/ingestor_test.go
@@ -17,7 +17,7 @@ import (
 func TestIngestor(t *testing.T) {
 	testutils.IntegrationTest(t)
 
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
 
 	t.Run("outside cluster", func(t *testing.T) {

diff --git a/pkg/testutils/integration_test.go b/pkg/testutils/integration_test.go
@@ -57,7 +57,6 @@ func TestIntegration(t *testing.T) {
 
 func StartCluster(ctx context.Context, t *testing.T) (kustoUrl string, k3sContainer *k3s.K3sContainer) {
 	t.Helper()
-	wg := sync.WaitGroup{}
 
 	k3sContainer, err := k3s.Run(ctx, "rancher/k3s:v1.31.2-k3s1")
 	testcontainers.CleanupContainer(t, k3sContainer)
@@ -76,10 +75,7 @@ func StartCluster(ctx context.Context, t *testing.T) (kustoUrl string, k3sContai
 	t.Logf("Kubeconfig: %s", kubeconfig)
 	t.Logf("Kustainer: %s", kustoContainer.ConnectionUrl())
 
-	wg.Add(1)
-	go t.Run("Configure Kusto", func(t *testing.T) {
-		defer wg.Done()
-
+	t.Run("Configure Kusto", func(t *testing.T) {
 		opts := kustainer.IngestionBatchingPolicy{
 			MaximumBatchingTimeSpan: 30 * time.Second,
 		}
@@ -89,28 +85,19 @@ func StartCluster(ctx context.Context, t *testing.T) (kustoUrl string, k3sContai
 		}
 	})
 
-	wg.Add(1)
-	go t.Run("Build and install Ingestor", func(tt *testing.T) {
-		defer wg.Done()
-
+	t.Run("Build and install Ingestor and Collector", func(tt *testing.T) {
 		ingestorContainer, err := ingestor.Run(ctx, ingestor.WithCluster(ctx, k3sContainer))
 		testcontainers.CleanupContainer(t, ingestorContainer)
 		require.NoError(tt, err)
 	})
 
-	wg.Add(1)
-	go t.Run("Build and install Collector", func(tt *testing.T) {
-		defer wg.Done()
-
+	t.Run("Build and install Collector", func(tt *testing.T) {
 		collectorContainer, err := collector.Run(ctx, collector.WithCluster(ctx, k3sContainer))
 		testcontainers.CleanupContainer(t, collectorContainer)
 		require.NoError(tt, err)
 	})
 
-	wg.Add(1)
-	go t.Run("Build and install Alerter", func(tt *testing.T) {
-		defer wg.Done()
-
+	t.Run("Build and install Alerter", func(tt *testing.T) {
 		crdPath := filepath.Join(t.TempDir(), "crd.yaml")
 		require.NoError(t, testutils.CopyFile("../../kustomize/bases/alertrules_crd.yaml", crdPath))
 		require.NoError(t, k3sContainer.CopyFileToContainer(ctx, crdPath, filepath.Join(testutils.K3sManifests, "crd.yaml"), 0644))
@@ -121,7 +108,6 @@ func StartCluster(ctx context.Context, t *testing.T) (kustoUrl string, k3sContai
 	})
 
 	kustoUrl = kustoContainer.ConnectionUrl()
-	wg.Wait()
 	return
 }