conditionally run flaky tests

.github/workflows/run-flaky-tests.yml

@@ -0,0 +1,26 @@
+name: All Tests Including Flaky
+
+on:
+  push: # todo: remove this trigger later
+    branches:
+      - "cdf/run-flaky-tests-nightly"
+
+  schedule:
+    # Run once a day at midnight
+    - cron:  '0 0 * * *'
+
+  workflow_dispatch:
+
+env:
+  COMMIT: ${{ github.sha }}
+  PR_BASE_COMMIT: ${{ github.event.push.base.sha }}
+  DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
+  TEMPORAL_VERSION_CHECK_DISABLED: 1
+  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
+
+jobs:
+  call-run-tests-workflow-with-flaky:
+    uses: temporalio/temporal/.github/workflows/run-tests.yml@cdf/run-flaky-tests-nightly
+    with:
+      include_flaky_tests: true
+    secrets: inherit

.github/workflows/run-tests.yml

@@ -8,6 +8,12 @@ on:
       - release/**
       - cloud/**
 
+  workflow_call:
+    inputs:
+      include_flaky_tests:
+        description: 'Whether to include flaky tests'
+        type: boolean
+
   workflow_dispatch:
     inputs:
       commit:
@@ -47,6 +53,9 @@ on:
         description: 'List of DBs to test on (i.e. ["sqlite", "cassandra", "mysql8", "postgres12"])'
         type: string
         default: '["sqlite"]'
+      include_flaky_tests:
+        description: 'Whether to include flaky tests'
+        type: boolean
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -152,6 +161,7 @@ jobs:
       UNIT_TEST_DIR: ${{ inputs.unit_test_directory }}
       TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
       RUN_SINGLE_UNIT_TEST: ${{ inputs.run_single_unit_test }}
+      INCLUDE_FLAKY_TESTS: ${{ inputs.include_flaky_tests }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -188,6 +198,7 @@ jobs:
     runs-on: ${{ matrix.runs-on }}
     env:
       BUILDKITE_MESSAGE: "{\"job\": \"integration-test\"}"
+      INCLUDE_FLAKY_TESTS: ${{ inputs.include_flaky_tests }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -274,6 +285,7 @@ jobs:
       SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
       TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
       BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
+      INCLUDE_FLAKY_TESTS: ${{ inputs.include_flaky_tests }}
     steps:
       - name: Print functional test
         run: echo "${{ needs.set-up-single-test.outputs.dbs }}" && echo "$SINGLE_TEST_ARGS"
@@ -346,6 +358,7 @@ jobs:
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
       BUILDKITE_MESSAGE: "{\"job\": \"functional-test-xdc\", \"db\": \"${{ matrix.persistence_driver }}\"}"
+      INCLUDE_FLAKY_TESTS: ${{ inputs.include_flaky_tests }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -415,6 +428,7 @@ jobs:
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
       ES_VERSION: ${{ matrix.es_version }}
+      INCLUDE_FLAKY_TESTS: ${{ inputs.include_flaky_tests }}
     steps:
       - uses: actions/checkout@v4
         with:

common/rpc/test/rpc_localstore_tls_test.go

@@ -119,7 +119,9 @@ type localStoreRPCSuite struct {
 }
 
 func TestLocalStoreTLSSuite(t *testing.T) {
-	t.Skip("Skipping flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		t.Skip("flaky test")
+	}
 	suite.Run(t, &localStoreRPCSuite{
 		Suite: &suite.Suite{},
 	})

tests/acquire_shard_test.go

@@ -227,7 +227,9 @@ func (s *EventualSuccessSuite) SetupSuite() {
 // TestEventuallySucceeds verifies that we eventually succeed in acquiring the shard when we get a deadline exceeded
 // error followed by a successful acquire shard call.
 func (s *EventualSuccessSuite) TestEventuallySucceeds() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 	defer cancel()

tests/describe_task_queue_test.go

@@ -25,6 +25,7 @@ package tests
 import (
 	"context"
 	"flag"
+	"os"
 	"testing"
 	"time"
 
@@ -80,7 +81,9 @@ func (s *DescribeTaskQueueSuite) TestAddNoTasks_ValidateStats() {
 }
 
 func (s *DescribeTaskQueueSuite) TestAddSingleTask_ValidateStats() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.OverrideDynamicConfig(dynamicconfig.MatchingUpdateAckInterval, 5*time.Second)
 	s.RunTestWithMatchingBehavior(func() { s.publishConsumeWorkflowTasksValidateStats(1, true) })

tests/http_api_test.go

@@ -308,7 +308,9 @@ func (s *HttpApiTestSuite) runHTTPAPIBasicsTest_Shorthand(contentType string, pr
 }
 
 func (s *HttpApiTestSuite) TestHTTPAPIHeaders() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	if s.HttpAPIAddress() == "" {
 		s.T().Skip("HTTP API server not enabled")

tests/max_buffered_event_test.go

@@ -26,6 +26,7 @@ package tests
 
 import (
 	"context"
+	"os"
 	"sync"
 	"testing"
 	"time"
@@ -138,7 +139,9 @@ func (s *MaxBufferedEventSuite) TestMaxBufferedEventsLimit() {
 }
 
 func (s *MaxBufferedEventSuite) TestBufferedEventsMutableStateSizeLimit() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	/*
 			This test starts a workflow, and block its workflow task, then sending

tests/versioning_test.go

@@ -1047,7 +1047,9 @@ func (s *VersioningIntegSuite) firstWorkflowTaskAssignmentSyncMatch() {
 }
 
 func (s *VersioningIntegSuite) TestIndependentActivityTaskAssignment_Spooled_VersionedWorkflow() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.independentActivityTaskAssignmentSpooled(true) })
 }
@@ -1239,7 +1241,9 @@ func (s *VersioningIntegSuite) independentActivityTaskAssignmentSpooled(versione
 }
 
 func (s *VersioningIntegSuite) TestIndependentActivityTaskAssignment_SyncMatch_VersionedWorkflow() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.independentActivityTaskAssignmentSyncMatch(true) })
 }
@@ -1421,7 +1425,9 @@ func (s *VersioningIntegSuite) independentActivityTaskAssignmentSyncMatch(versio
 }
 
 func (s *VersioningIntegSuite) TestWorkflowTaskRedirectInRetryFirstTask() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.testWorkflowTaskRedirectInRetry(true) })
 }
@@ -1732,7 +1738,9 @@ func (s *VersioningIntegSuite) TestDispatchUpgradeStopOld() {
 }
 
 func (s *VersioningIntegSuite) TestDispatchUpgradeWait() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.dispatchUpgrade(true, false) })
 }
@@ -1835,19 +1843,25 @@ const (
 )
 
 func (s *VersioningIntegSuite) TestDispatchActivityOld() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.dispatchActivity(dontFailActivity, false, false) })
 }
 
 func (s *VersioningIntegSuite) TestDispatchActivityFailOld() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.dispatchActivity(failActivity, false, false) })
 }
 
 func (s *VersioningIntegSuite) TestDispatchActivityTimeoutOld() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.dispatchActivity(timeoutActivity, false, false) })
 }
@@ -2602,13 +2616,17 @@ func (s *VersioningIntegSuite) TestDispatchActivityCrossTQFails() {
 }
 
 func (s *VersioningIntegSuite) TestDispatchChildWorkflowOld() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.dispatchChildWorkflow(false, false) })
 }
 
 func (s *VersioningIntegSuite) TestDispatchChildWorkflow() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.dispatchChildWorkflow(true, false) })
 }
@@ -3084,7 +3102,9 @@ func (s *VersioningIntegSuite) dispatchQuery(newVersioning bool) {
 }
 
 func (s *VersioningIntegSuite) TestDispatchContinueAsNewOld() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.dispatchContinueAsNew(false, false) })
 }
@@ -3248,7 +3268,9 @@ func (s *VersioningIntegSuite) dispatchContinueAsNew(newVersioning bool, crossTq
 }
 
 func (s *VersioningIntegSuite) TestDispatchContinueAsNewUpgradeOld() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.dispatchContinueAsNewUpgrade(false) })
 }
@@ -3600,13 +3622,17 @@ func (s *VersioningIntegSuite) dispatchRetry() {
 }
 
 func (s *VersioningIntegSuite) TestDispatchCronOld() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.dispatchCron(false) })
 }
 
 func (s *VersioningIntegSuite) TestDispatchCron() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	s.RunTestWithMatchingBehavior(func() { s.dispatchCron(true) })
 }

tests/workflow_test.go

@@ -575,7 +575,9 @@ func (s *WorkflowTestSuite) TestCompleteWorkflowTaskAndCreateNewOne() {
 }
 
 func (s *WorkflowTestSuite) TestWorkflowTaskAndActivityTaskTimeoutsWorkflow() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	id := "functional-timeouts-workflow-test"
 	wt := "functional-timeouts-workflow-test-type"

tests/xdc/failover_test.go

@@ -1857,7 +1857,9 @@ func (s *FunctionalClustersTestSuite) TestTransientWorkflowTaskFailover() {
 }
 
 func (s *FunctionalClustersTestSuite) TestCronWorkflowStartAndFailover() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	namespace := "test-cron-workflow-start-and-failover-" + common.GenerateRandomString(5)
 	client1 := s.cluster1.FrontendClient() // active
@@ -1953,7 +1955,9 @@ func (s *FunctionalClustersTestSuite) TestCronWorkflowStartAndFailover() {
 }
 
 func (s *FunctionalClustersTestSuite) TestCronWorkflowCompleteAndFailover() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	namespace := "test-cron-workflow-complete-and-failover-" + common.GenerateRandomString(5)
 	client1 := s.cluster1.FrontendClient() // active

tests/xdc/history_replication_signals_and_updates_test.go

@@ -208,7 +208,9 @@ func (t *hrsuTest) newHrsuTestCluster(ns string, name string, cluster *testcore.
 // TestAcceptedUpdateCanBeCompletedAfterFailoverAndFailback tests that an update can be accepted in one cluster, and completed in a
 // different cluster, after a failover.
 func (s *hrsuTestSuite) TestAcceptedUpdateCanBeCompletedAfterFailoverAndFailback() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	t, ctx, cancel := s.startHrsuTest()
 	defer cancel()
@@ -360,7 +362,9 @@ func (s *hrsuTestSuite) TestConflictResolutionReappliesUpdates() {
 // updates have the same update ID. The test confirms that when the conflict is resolved, we do not reapply the
 // UpdateAccepted event, since it has a conflicting ID.
 func (s *hrsuTestSuite) TestConflictResolutionDoesNotReapplyAcceptedUpdateWithConflictingId() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	t, ctx, cancel := s.startHrsuTest()
 	defer cancel()

tests/xdc/nexus_state_replication_test.go

@@ -30,6 +30,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"slices"
 	"testing"
 	"time"
@@ -100,7 +101,9 @@ func (s *NexusStateReplicationSuite) TearDownSuite() {
 // 9. Check that the operation completion triggers a workflow task when we poll on cluster1.
 // 10. Complete the workflow.
 func (s *NexusStateReplicationSuite) TestNexusOperationEventsReplicated() {
-	s.T().Skip("flaky test")
+	if os.Getenv("INCLUDE_FLAKY_TESTS") != "true" {
+		s.T().Skip("flaky test")
+	}
 
 	var callbackToken string
 	var publicCallbackUrl string