Skip to content

Commit

Permalink
Made tests work over head
Browse files Browse the repository at this point in the history
  • Loading branch information
@daniel-romano-DD
daniel-romano-DD committed Jan 17, 2025
1 parent 7590886 commit c705b46
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 42 deletions.
8 changes: 3 additions & 5 deletions tracer/src/Datadog.Trace/Ci/ImpactedTestsModule.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,8 @@ public static void Analyze(Test test)
{
if (IsEnabled)
{
Log.Debug("Impacted Tests Detection is enabled for {TestName}", test.Name);

var tags = test.GetTags();
Log.Debug("Impacted Tests Detection is enabled for {TestName} - {FileName} {From}..{To} ", test.Name, tags.SourceFile, tags.SourceStart, tags.SourceEnd);
var modified = false;
var testFiles = GetTestCoverage(tags);
var modifiedFiles = GetModifiedFiles();
Expand Down Expand Up @@ -127,14 +126,13 @@ private static FileCoverageInfo[] GetModifiedFiles()
{
var workspacePath = CIEnvironmentValues.Instance.WorkspacePath ?? string.Empty;
var prBase = BaseCommit;
var commit = CurrentCommit;
if (prBase is { Length: > 0 })
{
Log.Debug("PR detected. Retrieving diff lines from Git CLI for {Path} {BaseCommit}...{CurrentCommit}", workspacePath, prBase, commit);
Log.Debug("PR detected. Retrieving diff lines from Git CLI for {Path} from BaseCommit {BaseCommit}", workspacePath, prBase);
// Milestone 1.5 : Retrieve diff files and lines from Git Diff CLI
try
{
modifiedFiles = GitCommandHelper.GetGitDiffFilesAndLines(workspacePath, prBase, commit);
modifiedFiles = GitCommandHelper.GetGitDiffFilesAndLines(workspacePath, prBase);
}
catch (Exception ex)
{
Expand Down
97 changes: 68 additions & 29 deletions tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkImpactedTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,14 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI
public abstract class TestingFrameworkImpactedTests : TestingFrameworkTest
{
#pragma warning disable SA1401 // FieldsMustBePrivate
protected const string ModifiedLine = "// Modified by TestingFrameworkImpactedTests.cs";
protected const int ExpectedTestCount = 16;
protected const string GitHubSha = "bf1257d80c03aba92ba4fe83f588a14f6ac48e32";
protected const string GitHubBaseSha = "a700b56e12ddcbad5d19a2fa8852a15518ab205b";
protected string buildDir = string.Empty;
protected string repo = string.Empty;
protected string branch = string.Empty;
protected bool gitAvailable = false;
protected string repositoryRoot = string.Empty;
#pragma warning restore SA1401 // FieldsMustBePrivate

public TestingFrameworkImpactedTests(string sampleAppName, ITestOutputHelper output)
Expand Down Expand Up @@ -126,32 +127,41 @@ protected void ProcessAgentRequest(MockTracerAgent.EvpProxyPayload request, List
}
}

protected async Task SubmitTests(string packageVersion, string scenario, int expectedTests, Func<MockCIVisibilityTest, bool> testFilter = null, Action<MockTracerAgent.EvpProxyPayload, List<MockCIVisibilityTest>> agentRequestProcessor = null)
protected async Task SubmitTests(string packageVersion, int expectedTests, Func<MockCIVisibilityTest, bool> testFilter = null, Action<MockTracerAgent.EvpProxyPayload, List<MockCIVisibilityTest>> agentRequestProcessor = null)
{
var tests = new List<MockCIVisibilityTest>();
using var agent = GetAgent(tests, agentRequestProcessor);
try
{
ModifyFile();

using var processResult = await RunDotnetTestSampleAndWaitForExit(agent, packageVersion: packageVersion, expectedExitCode: 1);
var deadline = DateTime.UtcNow.AddMilliseconds(5000);
testFilter ??= _ => true; // t => t.Meta.ContainsKey("is_modified")
var tests = new List<MockCIVisibilityTest>();
using var agent = GetAgent(tests, agentRequestProcessor);

List<MockCIVisibilityTest> filteredTests = tests;
while (DateTime.UtcNow < deadline)
{
filteredTests = tests.Where(testFilter).ToList();
if (tests.Count() >= ExpectedTestCount)
using var processResult = await RunDotnetTestSampleAndWaitForExit(agent, packageVersion: packageVersion, expectedExitCode: 1);
var deadline = DateTime.UtcNow.AddMilliseconds(5000);
testFilter ??= _ => true; // t => t.Meta.ContainsKey("is_modified")

List<MockCIVisibilityTest> filteredTests = tests;
while (DateTime.UtcNow < deadline)
{
break;
}
filteredTests = tests.Where(testFilter).ToList();
if (tests.Count() >= ExpectedTestCount)
{
break;
}

Thread.Sleep(500);
}
Thread.Sleep(500);
}

// Sort and aggregate
var results = filteredTests.Select(t => t.Resource).Distinct().OrderBy(t => t).ToList();
// Sort and aggregate
var results = filteredTests.Select(t => t.Resource).Distinct().OrderBy(t => t).ToList();

tests.Count().Should().BeGreaterOrEqualTo(ExpectedTestCount, "Expected test count not met");
results.Count().Should().Be(expectedTests, "Expected filtered test count not met");
tests.Count().Should().BeGreaterOrEqualTo(ExpectedTestCount, "Expected test count not met");
results.Count().Should().Be(expectedTests, "Expected filtered test count not met");
}
finally
{
RestoreFile();
}
}

protected override Dictionary<string, string> DefineCIEnvironmentValues(Dictionary<string, string> values)
Expand Down Expand Up @@ -180,7 +190,6 @@ protected void InjectGitHubActionsSession(bool setupPr = true, bool? enabled = t
SetEnvironmentVariable(CIEnvironmentValues.Constants.GitHubRepository, repo);
SetEnvironmentVariable(CIEnvironmentValues.Constants.GitHubBaseRef, branch);
SetEnvironmentVariable(CIEnvironmentValues.Constants.GitHubWorkspace, buildDir);
SetEnvironmentVariable(CIEnvironmentValues.Constants.GitHubSha, GitHubSha);
if (setupPr)
{
SetEnvironmentVariable(CIEnvironmentValues.Constants.GitHubEventPath, GetEventJsonFile());
Expand All @@ -196,9 +205,6 @@ static string GetEventJsonFile()
string content = $$"""
{
"pull_request": {
"head": {
"sha": "{{GitHubSha}}"
},
"base": {
"sha": "{{GitHubBaseSha}}"
}
Expand Down Expand Up @@ -231,22 +237,55 @@ private MockTracerAgent GetAgent(List<MockCIVisibilityTest> receivedTests, Actio
private void InitGit()
{
// Check git availability
var output = RunGitCommandAsync("branch --show-current");
var output = RunGitCommand("branch --show-current");
if (output.ExitCode < 0)
{
// Try to fix the git path
RunGitCommandAsync("config --global --add safe.directory '*'");
output = RunGitCommandAsync("branch --show-current");
RunGitCommand("config --global --add safe.directory '*'");
output = RunGitCommand("branch --show-current");
}

if (output.ExitCode == 0)
{
gitAvailable = true;
// Retrieve WS root directory
output = RunGitCommand("rev-parse --show-toplevel");
if (output.ExitCode == 0)
{
gitAvailable = true;
repositoryRoot = output.Output.Trim();
Output.WriteLine($"Git available. Repository: {repositoryRoot}");
}
}

if (output.ExitCode < 0)
{
Output.WriteLine($"Git NOT available. ExitCode: {output.ExitCode} Error: {output.Error}");
}
}

private ProcessHelpers.CommandOutput RunGitCommandAsync(string arguments)
private string GetTestFile()
{
return Path.Combine(repositoryRoot, "tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs");
}

private void ModifyFile()
{
var path = GetTestFile();
var lines = File.ReadAllLines(path).ToList();
lines.Insert(33, ModifiedLine);
lines.Insert(63, ModifiedLine);
lines.Insert(64, ModifiedLine);
File.WriteAllLines(path, lines);
}

private void RestoreFile()
{
var path = GetTestFile();
var lines = File.ReadAllLines(path).Where(l => l != ModifiedLine).ToList();
File.WriteAllLines(path, lines);
}

private ProcessHelpers.CommandOutput RunGitCommand(string arguments)
{
try
{
Expand Down
13 changes: 8 additions & 5 deletions tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitImpactedTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI
public class XUnitImpactedTests : TestingFrameworkImpactedTests
{
private const int ExpectedSpanCount = 41;
private const string IsModifiedTag = "test.is_modified";

public XUnitImpactedTests(ITestOutputHelper output)
: base("XUnitTests", output)
Expand All @@ -36,7 +37,7 @@ public XUnitImpactedTests(ITestOutputHelper output)
public Task BaseShaFromPr(string packageVersion)
{
InjectGitHubActionsSession();
return SubmitTests(packageVersion, $"baseShaFromPr", 2, (t) => t.Meta.ContainsKey("test.is_modified") && t.Meta["test.is_modified"] == "true");
return SubmitTests(packageVersion, 2, TestIsModified);
}

[SkippableTheory]
Expand All @@ -46,7 +47,7 @@ public Task BaseShaFromPr(string packageVersion)
public Task BaseShaFromBackend(string packageVersion)
{
InjectGitHubActionsSession(false);
return SubmitTests(packageVersion, $"baseShaFromPr", 2, (t) => t.Meta.ContainsKey("test.is_modified") && t.Meta["test.is_modified"] == "true");
return SubmitTests(packageVersion, 2, TestIsModified);
}

[SkippableTheory]
Expand All @@ -66,7 +67,7 @@ public Task FilesFromBackend(string packageVersion)

ProcessAgentRequest(request, receivedTests);
};
return SubmitTests(packageVersion, $"baseShaFromPr", 12, (t) => t.Meta.ContainsKey("test.is_modified") && t.Meta["test.is_modified"] == "true", agentRequestProcessor);
return SubmitTests(packageVersion, 12, TestIsModified, agentRequestProcessor);
}

[SkippableTheory]
Expand All @@ -76,7 +77,7 @@ public Task FilesFromBackend(string packageVersion)
public Task DisabledByEnvVar(string packageVersion)
{
InjectGitHubActionsSession(true, false);
return SubmitTests(packageVersion, $"baseShaFromPr", 0, (t) => t.Meta.ContainsKey("is_modified"));
return SubmitTests(packageVersion, 0, TestIsModified);
}

[SkippableTheory]
Expand All @@ -86,7 +87,9 @@ public Task DisabledByEnvVar(string packageVersion)
public Task EnabledBySettings(string packageVersion)
{
InjectGitHubActionsSession(true, null);
return SubmitTests(packageVersion, $"baseShaFromPr", 0, (t) => t.Meta.ContainsKey("is_modified"));
return SubmitTests(packageVersion, 2, TestIsModified);
}

private static bool TestIsModified(MockCIVisibilityTest t) => t.Meta.ContainsKey(IsModifiedTag) && t.Meta[IsModifiedTag] == "true";
}
}
3 changes: 0 additions & 3 deletions tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ public void SimpleErrorTest()
{
_output.WriteLine("Test:SimpleErrorTest");
int i = 0;

int z = 0 / i;
}

Expand Down Expand Up @@ -63,8 +62,6 @@ public void TraitErrorTest()
{
_output.WriteLine("Test:TraitErrorTest");
int i = 0;


int z = 0 / i;
}

Expand Down

0 comments on commit c705b46

Please sign in to comment.