From c705b46617cd6074a1f3fa9d587c6a596246ce47 Mon Sep 17 00:00:00 2001 From: Daniel Romano <108014683+daniel-romano-DD@users.noreply.github.com> Date: Fri, 17 Jan 2025 16:18:34 +0100 Subject: [PATCH] Made tests work over head --- .../Datadog.Trace/Ci/ImpactedTestsModule.cs | 8 +- .../CI/TestingFrameworkImpactedTests.cs | 97 +++++++++++++------ .../CI/XUnitImpactedTests.cs | 13 ++- .../Samples.XUnitTests/TestSuite.cs | 3 - 4 files changed, 79 insertions(+), 42 deletions(-) diff --git a/tracer/src/Datadog.Trace/Ci/ImpactedTestsModule.cs b/tracer/src/Datadog.Trace/Ci/ImpactedTestsModule.cs index eb0445419a49..032047cb290b 100644 --- a/tracer/src/Datadog.Trace/Ci/ImpactedTestsModule.cs +++ b/tracer/src/Datadog.Trace/Ci/ImpactedTestsModule.cs @@ -44,9 +44,8 @@ public static void Analyze(Test test) { if (IsEnabled) { - Log.Debug("Impacted Tests Detection is enabled for {TestName}", test.Name); - var tags = test.GetTags(); + Log.Debug("Impacted Tests Detection is enabled for {TestName} - {FileName} {From}..{To} ", test.Name, tags.SourceFile, tags.SourceStart, tags.SourceEnd); var modified = false; var testFiles = GetTestCoverage(tags); var modifiedFiles = GetModifiedFiles(); @@ -127,14 +126,13 @@ private static FileCoverageInfo[] GetModifiedFiles() { var workspacePath = CIEnvironmentValues.Instance.WorkspacePath ?? string.Empty; var prBase = BaseCommit; - var commit = CurrentCommit; if (prBase is { Length: > 0 }) { - Log.Debug("PR detected. Retrieving diff lines from Git CLI for {Path} {BaseCommit}...{CurrentCommit}", workspacePath, prBase, commit); + Log.Debug("PR detected. Retrieving diff lines from Git CLI for {Path} from BaseCommit {BaseCommit}", workspacePath, prBase); // Milestone 1.5 : Retrieve diff files and lines from Git Diff CLI try { - modifiedFiles = GitCommandHelper.GetGitDiffFilesAndLines(workspacePath, prBase, commit); + modifiedFiles = GitCommandHelper.GetGitDiffFilesAndLines(workspacePath, prBase); } catch (Exception ex) { diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkImpactedTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkImpactedTests.cs index 1942a21c2b3d..e2e2046ab02d 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkImpactedTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkImpactedTests.cs @@ -27,13 +27,14 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI public abstract class TestingFrameworkImpactedTests : TestingFrameworkTest { #pragma warning disable SA1401 // FieldsMustBePrivate + protected const string ModifiedLine = "// Modified by TestingFrameworkImpactedTests.cs"; protected const int ExpectedTestCount = 16; - protected const string GitHubSha = "bf1257d80c03aba92ba4fe83f588a14f6ac48e32"; protected const string GitHubBaseSha = "a700b56e12ddcbad5d19a2fa8852a15518ab205b"; protected string buildDir = string.Empty; protected string repo = string.Empty; protected string branch = string.Empty; protected bool gitAvailable = false; + protected string repositoryRoot = string.Empty; #pragma warning restore SA1401 // FieldsMustBePrivate public TestingFrameworkImpactedTests(string sampleAppName, ITestOutputHelper output) @@ -126,32 +127,41 @@ protected void ProcessAgentRequest(MockTracerAgent.EvpProxyPayload request, List } } - protected async Task SubmitTests(string packageVersion, string scenario, int expectedTests, Func testFilter = null, Action> agentRequestProcessor = null) + protected async Task SubmitTests(string packageVersion, int expectedTests, Func testFilter = null, Action> agentRequestProcessor = null) { - var tests = new List(); - using var agent = GetAgent(tests, agentRequestProcessor); + try + { + ModifyFile(); - using var processResult = await RunDotnetTestSampleAndWaitForExit(agent, packageVersion: packageVersion, expectedExitCode: 1); - var deadline = DateTime.UtcNow.AddMilliseconds(5000); - testFilter ??= _ => true; // t => t.Meta.ContainsKey("is_modified") + var tests = new List(); + using var agent = GetAgent(tests, agentRequestProcessor); - List filteredTests = tests; - while (DateTime.UtcNow < deadline) - { - filteredTests = tests.Where(testFilter).ToList(); - if (tests.Count() >= ExpectedTestCount) + using var processResult = await RunDotnetTestSampleAndWaitForExit(agent, packageVersion: packageVersion, expectedExitCode: 1); + var deadline = DateTime.UtcNow.AddMilliseconds(5000); + testFilter ??= _ => true; // t => t.Meta.ContainsKey("is_modified") + + List filteredTests = tests; + while (DateTime.UtcNow < deadline) { - break; - } + filteredTests = tests.Where(testFilter).ToList(); + if (tests.Count() >= ExpectedTestCount) + { + break; + } - Thread.Sleep(500); - } + Thread.Sleep(500); + } - // Sort and aggregate - var results = filteredTests.Select(t => t.Resource).Distinct().OrderBy(t => t).ToList(); + // Sort and aggregate + var results = filteredTests.Select(t => t.Resource).Distinct().OrderBy(t => t).ToList(); - tests.Count().Should().BeGreaterOrEqualTo(ExpectedTestCount, "Expected test count not met"); - results.Count().Should().Be(expectedTests, "Expected filtered test count not met"); + tests.Count().Should().BeGreaterOrEqualTo(ExpectedTestCount, "Expected test count not met"); + results.Count().Should().Be(expectedTests, "Expected filtered test count not met"); + } + finally + { + RestoreFile(); + } } protected override Dictionary DefineCIEnvironmentValues(Dictionary values) @@ -180,7 +190,6 @@ protected void InjectGitHubActionsSession(bool setupPr = true, bool? enabled = t SetEnvironmentVariable(CIEnvironmentValues.Constants.GitHubRepository, repo); SetEnvironmentVariable(CIEnvironmentValues.Constants.GitHubBaseRef, branch); SetEnvironmentVariable(CIEnvironmentValues.Constants.GitHubWorkspace, buildDir); - SetEnvironmentVariable(CIEnvironmentValues.Constants.GitHubSha, GitHubSha); if (setupPr) { SetEnvironmentVariable(CIEnvironmentValues.Constants.GitHubEventPath, GetEventJsonFile()); @@ -196,9 +205,6 @@ static string GetEventJsonFile() string content = $$""" { "pull_request": { - "head": { - "sha": "{{GitHubSha}}" - }, "base": { "sha": "{{GitHubBaseSha}}" } @@ -231,22 +237,55 @@ private MockTracerAgent GetAgent(List receivedTests, Actio private void InitGit() { // Check git availability - var output = RunGitCommandAsync("branch --show-current"); + var output = RunGitCommand("branch --show-current"); if (output.ExitCode < 0) { // Try to fix the git path - RunGitCommandAsync("config --global --add safe.directory '*'"); - output = RunGitCommandAsync("branch --show-current"); + RunGitCommand("config --global --add safe.directory '*'"); + output = RunGitCommand("branch --show-current"); } if (output.ExitCode == 0) { - gitAvailable = true; + // Retrieve WS root directory + output = RunGitCommand("rev-parse --show-toplevel"); + if (output.ExitCode == 0) + { + gitAvailable = true; + repositoryRoot = output.Output.Trim(); + Output.WriteLine($"Git available. Repository: {repositoryRoot}"); + } + } + + if (output.ExitCode < 0) + { Output.WriteLine($"Git NOT available. ExitCode: {output.ExitCode} Error: {output.Error}"); } } - private ProcessHelpers.CommandOutput RunGitCommandAsync(string arguments) + private string GetTestFile() + { + return Path.Combine(repositoryRoot, "tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs"); + } + + private void ModifyFile() + { + var path = GetTestFile(); + var lines = File.ReadAllLines(path).ToList(); + lines.Insert(33, ModifiedLine); + lines.Insert(63, ModifiedLine); + lines.Insert(64, ModifiedLine); + File.WriteAllLines(path, lines); + } + + private void RestoreFile() + { + var path = GetTestFile(); + var lines = File.ReadAllLines(path).Where(l => l != ModifiedLine).ToList(); + File.WriteAllLines(path, lines); + } + + private ProcessHelpers.CommandOutput RunGitCommand(string arguments) { try { diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitImpactedTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitImpactedTests.cs index f8cadec097f6..939b46431c2e 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitImpactedTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitImpactedTests.cs @@ -21,6 +21,7 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI public class XUnitImpactedTests : TestingFrameworkImpactedTests { private const int ExpectedSpanCount = 41; + private const string IsModifiedTag = "test.is_modified"; public XUnitImpactedTests(ITestOutputHelper output) : base("XUnitTests", output) @@ -36,7 +37,7 @@ public XUnitImpactedTests(ITestOutputHelper output) public Task BaseShaFromPr(string packageVersion) { InjectGitHubActionsSession(); - return SubmitTests(packageVersion, $"baseShaFromPr", 2, (t) => t.Meta.ContainsKey("test.is_modified") && t.Meta["test.is_modified"] == "true"); + return SubmitTests(packageVersion, 2, TestIsModified); } [SkippableTheory] @@ -46,7 +47,7 @@ public Task BaseShaFromPr(string packageVersion) public Task BaseShaFromBackend(string packageVersion) { InjectGitHubActionsSession(false); - return SubmitTests(packageVersion, $"baseShaFromPr", 2, (t) => t.Meta.ContainsKey("test.is_modified") && t.Meta["test.is_modified"] == "true"); + return SubmitTests(packageVersion, 2, TestIsModified); } [SkippableTheory] @@ -66,7 +67,7 @@ public Task FilesFromBackend(string packageVersion) ProcessAgentRequest(request, receivedTests); }; - return SubmitTests(packageVersion, $"baseShaFromPr", 12, (t) => t.Meta.ContainsKey("test.is_modified") && t.Meta["test.is_modified"] == "true", agentRequestProcessor); + return SubmitTests(packageVersion, 12, TestIsModified, agentRequestProcessor); } [SkippableTheory] @@ -76,7 +77,7 @@ public Task FilesFromBackend(string packageVersion) public Task DisabledByEnvVar(string packageVersion) { InjectGitHubActionsSession(true, false); - return SubmitTests(packageVersion, $"baseShaFromPr", 0, (t) => t.Meta.ContainsKey("is_modified")); + return SubmitTests(packageVersion, 0, TestIsModified); } [SkippableTheory] @@ -86,7 +87,9 @@ public Task DisabledByEnvVar(string packageVersion) public Task EnabledBySettings(string packageVersion) { InjectGitHubActionsSession(true, null); - return SubmitTests(packageVersion, $"baseShaFromPr", 0, (t) => t.Meta.ContainsKey("is_modified")); + return SubmitTests(packageVersion, 2, TestIsModified); } + + private static bool TestIsModified(MockCIVisibilityTest t) => t.Meta.ContainsKey(IsModifiedTag) && t.Meta[IsModifiedTag] == "true"; } } diff --git a/tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs b/tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs index 5e525617a273..63da2fdf018b 100644 --- a/tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs +++ b/tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs @@ -32,7 +32,6 @@ public void SimpleErrorTest() { _output.WriteLine("Test:SimpleErrorTest"); int i = 0; - int z = 0 / i; } @@ -63,8 +62,6 @@ public void TraitErrorTest() { _output.WriteLine("Test:TraitErrorTest"); int i = 0; - - int z = 0 / i; }