diff --git a/.azure-pipelines/steps/run-in-docker.yml b/.azure-pipelines/steps/run-in-docker.yml index 9e755b953c21..0711802f7a55 100644 --- a/.azure-pipelines/steps/run-in-docker.yml +++ b/.azure-pipelines/steps/run-in-docker.yml @@ -100,6 +100,7 @@ steps: --env DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH \ --env DD_LOGGER_DD_TAGS \ --env IS_SSI_RUN \ + --env RANDOM_SEED \ ${{ parameters.extraArgs }} \ dd-trace-dotnet/${{ parameters.baseImage }}-${{ parameters.target }}:$sdkVersion \ dotnet /build/bin/Debug/_build.dll ${{ parameters.command }} diff --git a/docker-compose.yml b/docker-compose.yml index a188920791e9..136e905405a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -454,6 +454,7 @@ services: - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH - DD_LOGGER_DD_TAGS - IS_SSI_RUN + - RANDOM_SEED hostname: integrationtests depends_on: - servicestackredis @@ -524,6 +525,7 @@ services: - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH - DD_LOGGER_DD_TAGS - IS_SSI_RUN + - RANDOM_SEED hostname: integrationtests IntegrationTests.Serverless: @@ -580,6 +582,7 @@ services: - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH - DD_LOGGER_DD_TAGS - IS_SSI_RUN + - RANDOM_SEED hostname: integrationtests ExplorationTests: @@ -722,6 +725,7 @@ services: - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH - DD_LOGGER_DD_TAGS - IS_SSI_RUN + - RANDOM_SEED depends_on: - servicestackredis_arm64 - stackexchangeredis_arm64 @@ -800,6 +804,7 @@ services: - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH - DD_LOGGER_DD_TAGS - IS_SSI_RUN + - RANDOM_SEED start-test-agent: image: andrewlock/wait-for-dependencies diff --git a/tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs b/tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs index 941ee0640c86..3f96151166e3 100644 --- a/tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs +++ b/tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs @@ -126,12 +126,26 @@ protected override async Task RunTestCollectionsAsync(IMessageBus me }) .ToList(); + if (Environment.GetEnvironmentVariable("RANDOM_SEED") is not { } environmentSeed + || !int.TryParse(environmentSeed, out var seed)) + { + seed = new Random().Next(); + } + + DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Using seed {seed} to randomize tests order")); + + var random = new Random(seed); + Shuffle(collections, random); + + foreach (var collection in collections) + { + Shuffle(collection.TestCases, random); + } + _runTestCollectionsCallback?.Invoke(DiagnosticMessageSink, collections.SelectMany(c => c.TestCases)); var summary = new RunSummary(); - using var runner = new ConcurrentRunner(); - var tasks = new List>(); foreach (var test in collections.Where(t => !t.DisableParallelization)) @@ -172,6 +186,20 @@ private static bool IsParallelizationDisabled(ITestCollection collection) return attr?.GetNamedArgument(nameof(CollectionDefinitionAttribute.DisableParallelization)) is true; } + + private static void Shuffle(IList list, Random rng) + { + int n = list.Count; + + while (n > 1) + { + n--; + int k = rng.Next(n + 1); + var value = list[k]; + list[k] = list[n]; + list[n] = value; + } + } } private class CustomTestCollectionRunner : XunitTestCollectionRunner