From e28a1dcc629c152e5ee07ce8d8b34a9364df2d8e Mon Sep 17 00:00:00 2001 From: Matan Green Date: Mon, 16 Dec 2024 11:58:49 +0200 Subject: [PATCH 1/2] WIP: Exception Replay Improvements --- Datadog.Trace.sln | 24 +- exploration-tests/protobuf | 1 + .../ExceptionCaseInstrumentationManager.cs | 53 +++- .../ExceptionDebuggingProbe.cs | 6 +- .../ExceptionNormalizer.cs | 58 +++++ .../ExceptionReplaySnapshotCreator.cs | 54 ++++ .../ExceptionTrackManager.cs | 7 +- .../MethodUniqueIdentifier.cs | 8 +- .../TestExceptionNormalizer.cs | 4 + .../TrackedExceptionCase.cs | 2 +- .../Debugger/Helpers/ProbeStatusPollerMock.cs | 54 ++++ .../Snapshots/DebuggerSnapshotCreator.cs | 236 +++++++++--------- .../AspNetCore5ExceptionReplay.cs | 2 +- .../MockTracerAgent.cs | 2 +- .../Properties/launchSettings.json | 7 +- .../Samples.Debugger.AspNetCore5/Startup.cs | 57 ++++- .../ExceptionCaughtAndRethrownAsInnerTest.cs | 24 ++ ...ExceptionPropagatesThroughAsyncFlowTest.cs | 124 +++++++++ .../ExceptionReplay/RethrowTest.cs | 220 ++++++++++++++++ .../ExceptionReplay/ThrowOnlyOnceTest.cs | 227 +++++++++++++++++ .../ExceptionReplayIntentionalException.cs | 28 +++ 21 files changed, 1046 insertions(+), 152 deletions(-) create mode 160000 exploration-tests/protobuf create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySnapshotCreator.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/Helpers/ProbeStatusPollerMock.cs create mode 100644 tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ExceptionCaughtAndRethrownAsInnerTest.cs create mode 100644 tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ExceptionPropagatesThroughAsyncFlowTest.cs create mode 100644 tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/RethrowTest.cs create mode 100644 tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ThrowOnlyOnceTest.cs create mode 100644 tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplayIntentionalException.cs diff --git a/Datadog.Trace.sln b/Datadog.Trace.sln index 01f229af0fe7..5791973d2eeb 100644 --- a/Datadog.Trace.sln +++ b/Datadog.Trace.sln @@ -612,14 +612,14 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Debug|Any CPU.ActiveCfg = Debug|Win32 - {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Debug|Any CPU.Build.0 = Debug|Win32 - {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Release|Any CPU.ActiveCfg = Release|Win32 - {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Release|Any CPU.Build.0 = Release|Win32 - {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Debug|Any CPU.ActiveCfg = Debug|Win32 - {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Debug|Any CPU.Build.0 = Debug|Win32 - {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Release|Any CPU.ActiveCfg = Release|Win32 - {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Release|Any CPU.Build.0 = Release|Win32 + {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Debug|Any CPU.ActiveCfg = Debug|x64 + {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Debug|Any CPU.Build.0 = Debug|x64 + {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Release|Any CPU.ActiveCfg = Release|x64 + {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Release|Any CPU.Build.0 = Release|x64 + {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Debug|Any CPU.ActiveCfg = Debug|x64 + {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Debug|Any CPU.Build.0 = Debug|x64 + {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Release|Any CPU.ActiveCfg = Release|x64 + {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Release|Any CPU.Build.0 = Release|x64 {5DFDF781-F24C-45B1-82EF-9125875A80A4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5DFDF781-F24C-45B1-82EF-9125875A80A4}.Debug|Any CPU.Build.0 = Debug|Any CPU {5DFDF781-F24C-45B1-82EF-9125875A80A4}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -648,10 +648,10 @@ Global {3C6DD42E-9214-4747-92BA-78DE29AACE59}.Debug|Any CPU.Build.0 = Debug|Any CPU {3C6DD42E-9214-4747-92BA-78DE29AACE59}.Release|Any CPU.ActiveCfg = Release|Any CPU {3C6DD42E-9214-4747-92BA-78DE29AACE59}.Release|Any CPU.Build.0 = Release|Any CPU - {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Debug|Any CPU.ActiveCfg = Debug|Win32 - {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Debug|Any CPU.Build.0 = Debug|Win32 - {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Release|Any CPU.ActiveCfg = Release|Win32 - {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Release|Any CPU.Build.0 = Release|Win32 + {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Debug|Any CPU.ActiveCfg = Debug|x64 + {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Debug|Any CPU.Build.0 = Debug|x64 + {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Release|Any CPU.ActiveCfg = Release|x64 + {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Release|Any CPU.Build.0 = Release|x64 {FDB5C8D0-018D-4FF9-9680-C6A5078F819B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {FDB5C8D0-018D-4FF9-9680-C6A5078F819B}.Debug|Any CPU.Build.0 = Debug|Any CPU {FDB5C8D0-018D-4FF9-9680-C6A5078F819B}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/exploration-tests/protobuf b/exploration-tests/protobuf new file mode 160000 index 000000000000..7c40b2df1fdf --- /dev/null +++ b/exploration-tests/protobuf @@ -0,0 +1 @@ +Subproject commit 7c40b2df1fdf6f414c1c18c789715a9c948a0725 diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCaseInstrumentationManager.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCaseInstrumentationManager.cs index f6258a2f1ad7..a05f9806be52 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCaseInstrumentationManager.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCaseInstrumentationManager.cs @@ -9,6 +9,7 @@ using System.Diagnostics; using System.Linq; using System.Reflection; +using System.Reflection.Emit; using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Debugger.Configurations.Models; @@ -33,11 +34,13 @@ internal class ExceptionCaseInstrumentationManager private static readonly ConcurrentDictionary MethodToProbe = new(); private static readonly int MaxFramesToCapture = ExceptionDebugging.Settings.MaximumFramesToCapture; - internal static ExceptionCase Instrument(ExceptionIdentifier exceptionId) + internal static ExceptionCase Instrument(ExceptionIdentifier exceptionId, string exceptionToString) { Log.Information("Instrumenting {ExceptionId}", exceptionId); - var participatingUserMethods = GetMethodsToRejit(exceptionId.StackTrace); + var parsedFramesFromExceptionToString = ExceptionNormalizer.Instance.ParseFrames(exceptionToString).ToArray(); + var stackTrace = exceptionId.StackTrace.Where(frame => parsedFramesFromExceptionToString.Any(f => f.Contains(frame.Method.Name))).ToArray(); + var participatingUserMethods = GetMethodsToRejit(stackTrace); var uniqueMethods = participatingUserMethods .Distinct(EqualityComparer.Default) @@ -65,7 +68,7 @@ internal static ExceptionCase Instrument(ExceptionIdentifier exceptionId) } } - var newCase = new ExceptionCase(exceptionId, probes); + var newCase = new ExceptionCase(exceptionId.ExceptionTypes, probes); foreach (var method in uniqueMethods) { @@ -81,6 +84,50 @@ bool ShouldInstrumentFrameAtIndex(int i) } } + private static bool ContainsExceptionDispatchInfoThrow(MethodBase method) + { + var methodBody = method.GetMethodBody(); + if (methodBody == null) + { + return false; + } + + byte[] ilBytes = methodBody.GetILAsByteArray(); + + if (ilBytes == null) + { + return false; + } + + for (int i = 0; i < ilBytes.Length; i++) + { + if (ilBytes[i] == (byte)OpCodes.Call.Value || ilBytes[i] == (byte)OpCodes.Callvirt.Value) + { + // The next 4 bytes after a call instruction contain the metadata token + if (i + 4 < ilBytes.Length) + { + int metadataToken = BitConverter.ToInt32(ilBytes, i + 1); + try + { + MethodInfo calledMethod = (MethodInfo)method.Module.ResolveMethod(metadataToken); + if (calledMethod.DeclaringType == typeof(System.Runtime.ExceptionServices.ExceptionDispatchInfo) && + calledMethod.Name == "Throw") + { + return true; + } + } + catch (ArgumentException) + { + // If we can't resolve the method, just continue + continue; + } + } + } + } + + return false; + } + private static List GetMethodsToRejit(ParticipatingFrame[] allFrames) { var methodsToRejit = new List(); diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProbe.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProbe.cs index 74474d86e1e5..342685ee39b1 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProbe.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProbe.cs @@ -6,6 +6,10 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; using System.Threading; using Datadog.Trace.Debugger.Expressions; using Datadog.Trace.Debugger.Helpers; @@ -93,7 +97,7 @@ private void ProcessCase(ExceptionCase @case) var parentProbes = probes.Take(index).ToArray(); var childProbes = probes.Skip(index + 1).ToArray(); - var processor = new ExceptionProbeProcessor(probe, @case.ExceptionId.ExceptionTypes, parentProbes: parentProbes, childProbes: childProbes); + var processor = new ExceptionProbeProcessor(probe, @case.ExceptionTypes, parentProbes: parentProbes, childProbes: childProbes); @case.Processors.TryAdd(processor, 0); ExceptionDebuggingProcessor?.AddProbeProcessor(processor); } diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionNormalizer.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionNormalizer.cs index 9e0ef891cf19..1869b3d84ecb 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionNormalizer.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionNormalizer.cs @@ -4,6 +4,7 @@ // using System; +using System.Collections.Generic; using System.Numerics; using System.Runtime.CompilerServices; using System.Text; @@ -16,6 +17,12 @@ namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation { internal class ExceptionNormalizer { + protected ExceptionNormalizer() + { + } + + public static ExceptionNormalizer Instance { get; } = new(); + /// /// Given the string representation of an exception alongside it's FQN of the outer and (potential) inner exception, /// this function cleanse the stack trace from error messages, customized information attached to the exception and PDB line info if present. @@ -95,5 +102,56 @@ protected virtual int HashLine(VendoredMicrosoftCode.System.ReadOnlySpan l return fnvHashCode; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal IEnumerable ParseFrames(string exceptionString) + { + if (string.IsNullOrEmpty(exceptionString)) + { + throw new ArgumentException(@"Exception string cannot be null or empty", nameof(exceptionString)); + } + + var exceptionSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(exceptionString); + var inSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(" in "); + var atSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("at "); + var lambdaSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("lambda_"); + var datadogSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("at Datadog."); + + while (!exceptionSpan.IsEmpty) + { + var lineEndIndex = exceptionSpan.IndexOfAny('\r', '\n'); + VendoredMicrosoftCode.System.ReadOnlySpan line; + + if (lineEndIndex >= 0) + { + line = exceptionSpan.Slice(0, lineEndIndex); + exceptionSpan = exceptionSpan.Slice(lineEndIndex + 1); + if (!exceptionSpan.IsEmpty && exceptionSpan[0] == '\n') + { + exceptionSpan = exceptionSpan.Slice(1); + } + } + else + { + line = exceptionSpan; + exceptionSpan = default; + } + + // Is frame line (starts with `in `). + if (VendoredMicrosoftCode.System.MemoryExtensions.StartsWith(line.TrimStart(), atSpan, StringComparison.Ordinal)) + { + var index = VendoredMicrosoftCode.System.MemoryExtensions.IndexOf(line, inSpan, StringComparison.Ordinal); + line = index > 0 ? line.Slice(0, index) : line; + + if (VendoredMicrosoftCode.System.MemoryExtensions.Contains(line, lambdaSpan, StringComparison.Ordinal) || + VendoredMicrosoftCode.System.MemoryExtensions.Contains(line, datadogSpan, StringComparison.Ordinal)) + { + continue; + } + + yield return line.ToString(); + } + } + } } } diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySnapshotCreator.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySnapshotCreator.cs new file mode 100644 index 000000000000..1c3eb75d693c --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySnapshotCreator.cs @@ -0,0 +1,54 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Debugger.Snapshots; +using ProbeLocation = Datadog.Trace.Debugger.Expressions.ProbeLocation; + +#nullable enable +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class ExceptionReplaySnapshotCreator : DebuggerSnapshotCreator + { + private readonly string _exceptionHash; + private readonly string _exceptionCaptureId; + private readonly int _frameIndex; + + public ExceptionReplaySnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool hasCondition, string[] tags, CaptureLimitInfo limitInfo, string exceptionHash, string exceptionCaptureId, int frameIndex) + : base(isFullSnapshot, location, hasCondition, tags, limitInfo) + { + _exceptionHash = exceptionHash; + _exceptionCaptureId = exceptionCaptureId; + _frameIndex = frameIndex; + } + + public ExceptionReplaySnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool hasCondition, string[] tags, MethodScopeMembers methodScopeMembers, CaptureLimitInfo limitInfo, string exceptionHash, string exceptionCaptureId, int frameIndex) + : base(isFullSnapshot, location, hasCondition, tags, methodScopeMembers, limitInfo) + { + _exceptionHash = exceptionHash; + _exceptionCaptureId = exceptionCaptureId; + _frameIndex = frameIndex; + } + + internal override DebuggerSnapshotCreator EndSnapshot() + { + JsonWriter.WritePropertyName("exception_hash"); + JsonWriter.WriteValue(_exceptionHash); + + JsonWriter.WritePropertyName("exception_capture_id"); + JsonWriter.WriteValue(_exceptionCaptureId); + + JsonWriter.WritePropertyName("frame_index"); + JsonWriter.WriteValue(_frameIndex); + + return base.EndSnapshot(); + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs index 159ca5a83b13..834fcb9807b4 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs @@ -37,7 +37,6 @@ internal class ExceptionTrackManager private static readonly SemaphoreSlim WorkAvailable = new(0, int.MaxValue); private static readonly CancellationTokenSource Cts = new(); private static readonly ExceptionCaseScheduler ExceptionsScheduler = new(); - private static readonly ExceptionNormalizer ExceptionNormalizer = new(); private static readonly int MaxFramesToCapture = ExceptionDebugging.Settings.MaximumFramesToCapture; private static readonly TimeSpan RateLimit = ExceptionDebugging.Settings.RateLimit; private static readonly BasicCircuitBreaker ReportingCircuitBreaker = new(ExceptionDebugging.Settings.MaxExceptionAnalysisLimit, TimeSpan.FromSeconds(1)); @@ -96,7 +95,7 @@ public static void Report(Span span, Exception? exception) private static void ReportInternal(Exception exception, ErrorOriginKind errorOrigin, Span rootSpan) { var exToString = exception.ToString(); - var normalizedExHash = ExceptionNormalizer.NormalizeAndHashException(exToString, exception.GetType().Name, exception.InnerException?.GetType().Name); + var normalizedExHash = ExceptionNormalizer.Instance.NormalizeAndHashException(exToString, exception.GetType().Name, exception.InnerException?.GetType().Name); if (CachedDoneExceptions.Contains(normalizedExHash)) { @@ -155,7 +154,7 @@ private static void ProcessException(Exception exception, int normalizedExHash, var allParticipatingFrames = GetAllExceptionRelatedStackFrames(exception); var allParticipatingFramesFlattened = allParticipatingFrames.GetAllFlattenedFrames().Reverse().ToArray(); - normalizedExHash = normalizedExHash != 0 ? normalizedExHash : ExceptionNormalizer.NormalizeAndHashException(exception.ToString(), exception.GetType().Name, exception.InnerException?.GetType().Name); + normalizedExHash = normalizedExHash != 0 ? normalizedExHash : ExceptionNormalizer.Instance.NormalizeAndHashException(exception.ToString(), exception.GetType().Name, exception.InnerException?.GetType().Name); if (allParticipatingFramesFlattened.Length == 0) { @@ -514,7 +513,7 @@ private static bool ShouldReportException(Exception ex, ParticipatingFrame[] fra return true; } - bool AtLeastOneFrameBelongToUserCode() => framesToRejit.All(f => !FrameFilter.IsUserCode(f)) == false; + bool AtLeastOneFrameBelongToUserCode() => framesToRejit.Any(f => FrameFilter.IsUserCode(f)); } private static bool IsSupportedExceptionType(Exception ex) => diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs index af3d84328c8c..409a9b56615e 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs @@ -24,14 +24,14 @@ public override int GetHashCode() { private readonly int _hashCode; - public ExceptionCase(ExceptionIdentifier exceptionId, ExceptionDebuggingProbe[] probes) + public ExceptionCase(HashSet exceptionTypes, ExceptionDebuggingProbe[] probes) { - ExceptionId = exceptionId; + ExceptionTypes = exceptionTypes; Probes = probes; _hashCode = ComputeHashCode(); } - public ExceptionIdentifier ExceptionId { get; } + public HashSet ExceptionTypes { get; } public ExceptionDebuggingProbe[] Probes { get; } @@ -68,7 +68,7 @@ public override string ToString() { var probesInfo = Probes == null ? "null" : $"{Probes.Length} probes"; var processorsCount = Processors?.Count ?? 0; - return $"ExceptionCase: ExceptionId={ExceptionId}, Probes=[{probesInfo}], Processors={processorsCount}"; + return $"ExceptionCase: Probes=[{probesInfo}], Processors={processorsCount}"; } } diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TestExceptionNormalizer.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TestExceptionNormalizer.cs index a62f8bc17102..2cede926118c 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TestExceptionNormalizer.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TestExceptionNormalizer.cs @@ -19,6 +19,10 @@ internal class TestExceptionNormalizer : ExceptionNormalizer { private StringBuilder? _debug; + public TestExceptionNormalizer() + { + } + internal int NormalizeAndHashException(string exceptionString, string outerExceptionType, string? innerExceptionType, StringBuilder debug) { _debug = debug; diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedExceptionCase.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedExceptionCase.cs index 3668a55fb821..ade16dbbc8a0 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedExceptionCase.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedExceptionCase.cs @@ -76,7 +76,7 @@ public void Instrument() // else - If there is a concurrent initialization or tearing down, ignore this case if (Initialized()) { - var @case = ExceptionCaseInstrumentationManager.Instrument(ExceptionIdentifier); + var @case = ExceptionCaseInstrumentationManager.Instrument(ExceptionIdentifier, ExceptionToString); BeginCollect(@case); CachedDoneExceptions.Remove(NormalizedExceptionHash); } diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/ProbeStatusPollerMock.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/ProbeStatusPollerMock.cs new file mode 100644 index 000000000000..4001e22d5733 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Helpers/ProbeStatusPollerMock.cs @@ -0,0 +1,54 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Datadog.Trace.Debugger.ProbeStatuses; + +namespace Datadog.Trace.Debugger.Helpers +{ + internal class ProbeStatusPollerMock : IProbeStatusPoller + { + internal bool Called { get; private set; } + + public void StartPolling() + { + Called = true; + } + + public void AddProbes(FetchProbeStatus[] newProbes) + { + Called = true; + } + + public void RemoveProbes(string[] removeProbes) + { + Called = true; + } + + public void UpdateProbes(string[] probeIds, FetchProbeStatus[] newProbeStatuses) + { + Called = true; + } + + public void UpdateProbe(string probeId, FetchProbeStatus newProbeStatus) + { + Called = true; + } + + public string[] GetBoundedProbes(string[] candidateProbeIds) + { + Called = true; + return candidateProbeIds; + } + + public void Dispose() + { + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotCreator.cs b/tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotCreator.cs index 03669840de6a..65d0beb91ee5 100644 --- a/tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotCreator.cs +++ b/tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotCreator.cs @@ -28,7 +28,9 @@ internal class DebuggerSnapshotCreator : IDebuggerSnapshotCreator, IDisposable private const string DDSource = "dd_debugger"; private const string UnknownValue = "Unknown"; - private readonly JsonTextWriter _jsonWriter; +#pragma warning disable SA1401 + protected readonly JsonTextWriter JsonWriter; +#pragma warning restore SA1401 private readonly StringBuilder _jsonUnderlyingString; private readonly bool _isFullSnapshot; private readonly ProbeLocation _probeLocation; @@ -47,7 +49,7 @@ public DebuggerSnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool _isFullSnapshot = isFullSnapshot; _probeLocation = location; _jsonUnderlyingString = StringBuilderCache.Acquire(); - _jsonWriter = new JsonTextWriter(new StringWriter(_jsonUnderlyingString)); + JsonWriter = new JsonTextWriter(new StringWriter(_jsonUnderlyingString)); MethodScopeMembers = default; _captureBehaviour = CaptureBehaviour.Capture; _errors = null; @@ -228,7 +230,7 @@ internal void SetDuration() internal void Initialize() { - _jsonWriter.WriteStartObject(); + JsonWriter.WriteStartObject(); StartDebugger(); StartSnapshot(); if (_isFullSnapshot) @@ -239,35 +241,35 @@ internal void Initialize() internal void StartDebugger() { - _jsonWriter.WritePropertyName("debugger"); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("debugger"); + JsonWriter.WriteStartObject(); } internal void StartSnapshot() { - _jsonWriter.WritePropertyName("snapshot"); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("snapshot"); + JsonWriter.WriteStartObject(); } internal void StartCaptures() { - _jsonWriter.WritePropertyName("captures"); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("captures"); + JsonWriter.WriteStartObject(); } internal void StartEntry() { - _jsonWriter.WritePropertyName("entry"); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("entry"); + JsonWriter.WriteStartObject(); } internal void StartLines(int lineNumber) { - _jsonWriter.WritePropertyName("lines"); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("lines"); + JsonWriter.WriteStartObject(); - _jsonWriter.WritePropertyName(lineNumber.ToString()); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName(lineNumber.ToString()); + JsonWriter.WriteStartObject(); } internal void EndEntry(bool hasArgumentsOrLocals) @@ -275,11 +277,11 @@ internal void EndEntry(bool hasArgumentsOrLocals) if (hasArgumentsOrLocals) { // end arguments or locals - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); } // end entry - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); } internal void StartReturn() @@ -289,8 +291,8 @@ internal void StartReturn() StartCaptures(); } - _jsonWriter.WritePropertyName("return"); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("return"); + JsonWriter.WriteStartObject(); } internal void EndReturn(bool hasArgumentsOrLocals) @@ -298,15 +300,15 @@ internal void EndReturn(bool hasArgumentsOrLocals) if (hasArgumentsOrLocals) { // end arguments or locals - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); } // end line number or method return - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); if (_probeLocation == ProbeLocation.Line) { // end lines - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); } // end captures @@ -315,30 +317,30 @@ internal void EndReturn(bool hasArgumentsOrLocals) internal void EndCapture() { - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); } internal DebuggerSnapshotCreator EndDebugger() { - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); return this; } - internal DebuggerSnapshotCreator EndSnapshot() + internal virtual DebuggerSnapshotCreator EndSnapshot() { - _jsonWriter.WritePropertyName("id"); - _jsonWriter.WriteValue(SnapshotId); + JsonWriter.WritePropertyName("id"); + JsonWriter.WriteValue(SnapshotId); - _jsonWriter.WritePropertyName("timestamp"); - _jsonWriter.WriteValue(DateTimeOffset.Now.ToUnixTimeMilliseconds()); + JsonWriter.WritePropertyName("timestamp"); + JsonWriter.WriteValue(DateTimeOffset.Now.ToUnixTimeMilliseconds()); - _jsonWriter.WritePropertyName("duration"); - _jsonWriter.WriteValue(_accumulatedDuration.TotalMilliseconds); + JsonWriter.WritePropertyName("duration"); + JsonWriter.WriteValue(_accumulatedDuration.TotalMilliseconds); - _jsonWriter.WritePropertyName("language"); - _jsonWriter.WriteValue(TracerConstants.Language); + JsonWriter.WritePropertyName("language"); + JsonWriter.WriteValue(TracerConstants.Language); - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); return this; } @@ -356,11 +358,11 @@ public void CaptureStaticFields(ref CaptureInfo info) { if (info.IsAsyncCapture()) { - DebuggerSnapshotSerializer.SerializeStaticFields(info.AsyncCaptureInfo.KickoffInvocationTargetType, _jsonWriter, _limitInfo); + DebuggerSnapshotSerializer.SerializeStaticFields(info.AsyncCaptureInfo.KickoffInvocationTargetType, JsonWriter, _limitInfo); } else { - DebuggerSnapshotSerializer.SerializeStaticFields(info.InvocationTargetType, _jsonWriter, _limitInfo); + DebuggerSnapshotSerializer.SerializeStaticFields(info.InvocationTargetType, JsonWriter, _limitInfo); } } @@ -368,29 +370,29 @@ internal void CaptureArgument(TArg value, string name, Type type = null) { StartLocalsOrArgsIfNeeded("arguments"); // in case TArg is object and we have the concrete type, use it - DebuggerSnapshotSerializer.Serialize(value, type ?? typeof(TArg), name, _jsonWriter, _limitInfo); + DebuggerSnapshotSerializer.Serialize(value, type ?? typeof(TArg), name, JsonWriter, _limitInfo); } internal void CaptureLocal(TLocal value, string name, Type type = null) { StartLocalsOrArgsIfNeeded("locals"); // in case TLocal is object and we have the concrete type, use it - DebuggerSnapshotSerializer.Serialize(value, type ?? typeof(TLocal), name, _jsonWriter, _limitInfo); + DebuggerSnapshotSerializer.Serialize(value, type ?? typeof(TLocal), name, JsonWriter, _limitInfo); } internal void CaptureException(Exception ex) { - _jsonWriter.WritePropertyName("throwable"); - _jsonWriter.WriteStartObject(); - _jsonWriter.WritePropertyName("message"); - _jsonWriter.WriteValue(ex.Message); - _jsonWriter.WritePropertyName("type"); - _jsonWriter.WriteValue(ex.GetType().FullName); - _jsonWriter.WritePropertyName("stacktrace"); - _jsonWriter.WriteStartArray(); + JsonWriter.WritePropertyName("throwable"); + JsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("message"); + JsonWriter.WriteValue(ex.Message); + JsonWriter.WritePropertyName("type"); + JsonWriter.WriteValue(ex.GetType().FullName); + JsonWriter.WritePropertyName("stacktrace"); + JsonWriter.WriteStartArray(); AddFrames(new StackTrace(ex).GetFrames() ?? Array.Empty()); - _jsonWriter.WriteEndArray(); - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndArray(); + JsonWriter.WriteEndObject(); } internal void CaptureEntryMethodStartMarker(ref CaptureInfo info) @@ -641,7 +643,7 @@ internal void CaptureScopeMembers(ScopeMember[] members, ScopeMemberKind? kind = private void StartLocalsOrArgsIfNeeded(string newParent) { - var currentParent = _jsonWriter.Path.Split('.').LastOrDefault(p => p is "locals" or "arguments"); + var currentParent = JsonWriter.Path.Split('.').LastOrDefault(p => p is "locals" or "arguments"); if (currentParent == newParent) { // We're already there! @@ -653,11 +655,11 @@ private void StartLocalsOrArgsIfNeeded(string newParent) (currentParent == "arguments" && newParent == "locals")) { // We need to close the previous node first. - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); } - _jsonWriter.WritePropertyName(newParent); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName(newParent); + JsonWriter.WriteStartObject(); } // Finalize snapshot @@ -740,57 +742,57 @@ internal DebuggerSnapshotCreator AddEvaluationErrors() return this; } - _jsonWriter.WritePropertyName("evaluationErrors"); - _jsonWriter.WriteStartArray(); + JsonWriter.WritePropertyName("evaluationErrors"); + JsonWriter.WriteStartArray(); foreach (var error in _errors) { - _jsonWriter.WriteStartObject(); - _jsonWriter.WritePropertyName("expr"); - _jsonWriter.WriteValue(error.Expression); - _jsonWriter.WritePropertyName("message"); - _jsonWriter.WriteValue(error.Message); - _jsonWriter.WriteEndObject(); + JsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("expr"); + JsonWriter.WriteValue(error.Expression); + JsonWriter.WritePropertyName("message"); + JsonWriter.WriteValue(error.Message); + JsonWriter.WriteEndObject(); } - _jsonWriter.WriteEndArray(); + JsonWriter.WriteEndArray(); return this; } internal DebuggerSnapshotCreator AddProbeInfo(string probeId, int probeVersion, T methodNameOrLineNumber, string typeFullNameOrFilePath) { - _jsonWriter.WritePropertyName("probe"); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("probe"); + JsonWriter.WriteStartObject(); - _jsonWriter.WritePropertyName("id"); - _jsonWriter.WriteValue(probeId); + JsonWriter.WritePropertyName("id"); + JsonWriter.WriteValue(probeId); - _jsonWriter.WritePropertyName("version"); - _jsonWriter.WriteValue(probeVersion); + JsonWriter.WritePropertyName("version"); + JsonWriter.WriteValue(probeVersion); - _jsonWriter.WritePropertyName("location"); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("location"); + JsonWriter.WriteStartObject(); if (_probeLocation == ProbeLocation.Method) { - _jsonWriter.WritePropertyName("method"); - _jsonWriter.WriteValue(methodNameOrLineNumber); + JsonWriter.WritePropertyName("method"); + JsonWriter.WriteValue(methodNameOrLineNumber); - _jsonWriter.WritePropertyName("type"); - _jsonWriter.WriteValue(typeFullNameOrFilePath ?? UnknownValue); + JsonWriter.WritePropertyName("type"); + JsonWriter.WriteValue(typeFullNameOrFilePath ?? UnknownValue); } else { - _jsonWriter.WritePropertyName("file"); - _jsonWriter.WriteValue(SanitizePath(typeFullNameOrFilePath)); + JsonWriter.WritePropertyName("file"); + JsonWriter.WriteValue(SanitizePath(typeFullNameOrFilePath)); - _jsonWriter.WritePropertyName("lines"); - _jsonWriter.WriteStartArray(); - _jsonWriter.WriteValue(methodNameOrLineNumber.ToString()); - _jsonWriter.WriteEndArray(); + JsonWriter.WritePropertyName("lines"); + JsonWriter.WriteStartArray(); + JsonWriter.WriteValue(methodNameOrLineNumber.ToString()); + JsonWriter.WriteEndArray(); } - _jsonWriter.WriteEndObject(); - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); return this; } @@ -810,10 +812,10 @@ private DebuggerSnapshotCreator AddStackInfo() var stackFrames = (new StackTrace(true).GetFrames() ?? Array.Empty()) .SkipWhile(frame => frame?.GetMethod()?.DeclaringType?.Namespace?.StartsWith("Datadog") == true).ToArray(); - _jsonWriter.WritePropertyName("stack"); - _jsonWriter.WriteStartArray(); + JsonWriter.WritePropertyName("stack"); + JsonWriter.WriteStartArray(); AddFrames(stackFrames); - _jsonWriter.WriteEndArray(); + JsonWriter.WriteEndArray(); return this; } @@ -822,77 +824,77 @@ private void AddFrames(StackFrame[] frames) { foreach (var frame in frames) { - _jsonWriter.WriteStartObject(); - _jsonWriter.WritePropertyName("function"); + JsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("function"); var frameMethod = frame.GetMethod(); - _jsonWriter.WriteValue($"{frameMethod?.DeclaringType?.FullName ?? UnknownValue}.{frameMethod?.Name ?? UnknownValue}"); + JsonWriter.WriteValue($"{frameMethod?.DeclaringType?.FullName ?? UnknownValue}.{frameMethod?.Name ?? UnknownValue}"); var fileName = frame.GetFileName(); if (fileName != null) { - _jsonWriter.WritePropertyName("fileName"); - _jsonWriter.WriteValue(frame.GetFileName()); + JsonWriter.WritePropertyName("fileName"); + JsonWriter.WriteValue(frame.GetFileName()); } - _jsonWriter.WritePropertyName("lineNumber"); - _jsonWriter.WriteValue(frame.GetFileLineNumber()); - _jsonWriter.WriteEndObject(); + JsonWriter.WritePropertyName("lineNumber"); + JsonWriter.WriteValue(frame.GetFileLineNumber()); + JsonWriter.WriteEndObject(); } } internal DebuggerSnapshotCreator AddLoggerInfo(string methodName, string typeFullName, string probeFilePath) { - _jsonWriter.WritePropertyName("logger"); - _jsonWriter.WriteStartObject(); + JsonWriter.WritePropertyName("logger"); + JsonWriter.WriteStartObject(); var thread = Thread.CurrentThread; - _jsonWriter.WritePropertyName("thread_id"); - _jsonWriter.WriteValue(thread.ManagedThreadId); + JsonWriter.WritePropertyName("thread_id"); + JsonWriter.WriteValue(thread.ManagedThreadId); - _jsonWriter.WritePropertyName("thread_name"); - _jsonWriter.WriteValue(thread.Name); + JsonWriter.WritePropertyName("thread_name"); + JsonWriter.WriteValue(thread.Name); - _jsonWriter.WritePropertyName("version"); - _jsonWriter.WriteValue(LoggerVersion); + JsonWriter.WritePropertyName("version"); + JsonWriter.WriteValue(LoggerVersion); - _jsonWriter.WritePropertyName("name"); - _jsonWriter.WriteValue(typeFullName ?? SanitizePath(probeFilePath)); + JsonWriter.WritePropertyName("name"); + JsonWriter.WriteValue(typeFullName ?? SanitizePath(probeFilePath)); - _jsonWriter.WritePropertyName("method"); - _jsonWriter.WriteValue(methodName); + JsonWriter.WritePropertyName("method"); + JsonWriter.WriteValue(methodName); - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); return this; } internal DebuggerSnapshotCreator AddGeneralInfo(string service, string traceId, string spanId) { - _jsonWriter.WritePropertyName("service"); - _jsonWriter.WriteValue(service ?? UnknownValue); + JsonWriter.WritePropertyName("service"); + JsonWriter.WriteValue(service ?? UnknownValue); - _jsonWriter.WritePropertyName("ddsource"); - _jsonWriter.WriteValue(DDSource); + JsonWriter.WritePropertyName("ddsource"); + JsonWriter.WriteValue(DDSource); - _jsonWriter.WritePropertyName("dd.trace_id"); - _jsonWriter.WriteValue(traceId); + JsonWriter.WritePropertyName("dd.trace_id"); + JsonWriter.WriteValue(traceId); - _jsonWriter.WritePropertyName("dd.span_id"); - _jsonWriter.WriteValue(spanId); + JsonWriter.WritePropertyName("dd.span_id"); + JsonWriter.WriteValue(spanId); return this; } public DebuggerSnapshotCreator AddMessage() { - _jsonWriter.WritePropertyName("message"); - _jsonWriter.WriteValue(_message); + JsonWriter.WritePropertyName("message"); + JsonWriter.WriteValue(_message); return this; } public DebuggerSnapshotCreator Complete() { - _jsonWriter.WriteEndObject(); + JsonWriter.WriteEndObject(); return this; } @@ -907,7 +909,7 @@ public void Dispose() { Stop(); _scopeMembersPool.Return(MethodScopeMembers); - _jsonWriter?.Close(); + JsonWriter?.Close(); } catch { diff --git a/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ExceptionReplay/AspNetCore5ExceptionReplay.cs b/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ExceptionReplay/AspNetCore5ExceptionReplay.cs index 518922673964..1df74c2a53a5 100644 --- a/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ExceptionReplay/AspNetCore5ExceptionReplay.cs +++ b/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ExceptionReplay/AspNetCore5ExceptionReplay.cs @@ -66,7 +66,7 @@ public abstract class AspNetCore5ExceptionReplay : AspNetBase, IClassFixtureThe list of spans. public IImmutableList WaitForSpans( int count, - int timeoutInMilliseconds = 20000, + int timeoutInMilliseconds = int.MaxValue, string operationName = null, DateTimeOffset? minDateTime = null, bool returnAllOperations = false, diff --git a/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Properties/launchSettings.json b/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Properties/launchSettings.json index 913e904792e9..19bc64ecc860 100644 --- a/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Properties/launchSettings.json +++ b/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Properties/launchSettings.json @@ -39,8 +39,8 @@ "DD_DEBUGGER_DIAGNOSTICS_INTERVAL": "5", "DD_SERVICE": "exception_replay_service", "DD_ENV": "staging", - "DD_TRACE_DEBUG": "3", - "DD_TRACE_ENABLED": "true", + "DD_TRACE_DEBUG": "1", + "DD_TRACE_ENABLED": "1", "DD_VERSION": "1", "DD_IAST_ENABLED": "false", "DD_DEBUGGER_MAX_DEPTH_TO_SERIALIZE": "2", @@ -49,7 +49,8 @@ "DD_DYNAMIC_INSTRUMENTATION_INTERAL_FORCE_SYMBOL_DATABASE_UPLOAD": "true", "DD_DYNAMIC_INSTRUMENTATION_SYMBOL_DATABASE_UPLOAD_ENABLED": "true", "DD_SYMBOL_DATABASE_UPLOAD_ENABLED": "true", - "DD_EXCEPTION_DEBUGGING_RATE_LIMIT_SECONDS": "5" + "DD_EXCEPTION_DEBUGGING_RATE_LIMIT_SECONDS": "5", + "COMPLUS_ForceEnc": "0" }, "Logging": { "LogLevel": { diff --git a/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Startup.cs b/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Startup.cs index f53f20c54145..00caeb6a0dab 100644 --- a/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Startup.cs +++ b/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Startup.cs @@ -1,3 +1,4 @@ +using System; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -20,12 +21,15 @@ public Startup(IConfiguration configuration) // This method gets called by the runtime. Use this method to add services to the container. public void ConfigureServices(IServiceCollection services) { + services.AddTransient(); services.AddControllers(); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { + app.UseMiddleware(); + if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); @@ -60,11 +64,10 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) }); }); - app.Use( - async (context, next) => - { - await next.Invoke(); - }); + app.Use(async (context, next) => + { + await next(); + }); app.UseEndpoints(endpoints => @@ -73,4 +76,48 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) }); } } + + + public class FirstLastMiddleware + { + private readonly RequestDelegate _next; + + public FirstLastMiddleware(RequestDelegate next) + { + _next = next; + } + + public async Task InvokeAsync(HttpContext context) + { + try + { + // Code to run before the next middleware in the pipeline + Console.WriteLine("FirstLastMiddleware: Entering request pipeline"); + + // Call the next middleware in the pipeline + await _next(context); + + // Code to run after the next middleware in the pipeline + Console.WriteLine("FirstLastMiddleware: Exiting request pipeline normally"); + } + catch (Exception ex) + { + // Exception handling code + Console.WriteLine($"FirstLastMiddleware caught an exception: {ex.Message}"); + throw; // Re-throw the exception to be handled by the global exception handler + } + } + } + + public class CustomStartupFilter : IStartupFilter + { + public Action Configure(Action next) + { + return app => + { + app.UseMiddleware(); + next(app); + }; + } + } } diff --git a/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ExceptionCaughtAndRethrownAsInnerTest.cs b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ExceptionCaughtAndRethrownAsInnerTest.cs new file mode 100644 index 000000000000..dc9547fe148a --- /dev/null +++ b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ExceptionCaughtAndRethrownAsInnerTest.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Samples.Probes.TestRuns.ExceptionReplay +{ + [ExceptionReplayTestData(expectedNumberOfSnapshotsDefault: 4, expectedNumberOfSnaphotsFull: 4)] + internal class ExceptionCaughtAndRethrownAsInnerTest : IRun + { + public void Run() + { + try + { + throw new Exception("My future is unknown"); + } + catch (Exception e) + { + throw new ExceptionReplayIntentionalException(e.Message, innerException: e); + } + } + } +} diff --git a/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ExceptionPropagatesThroughAsyncFlowTest.cs b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ExceptionPropagatesThroughAsyncFlowTest.cs new file mode 100644 index 000000000000..b3ba106e9315 --- /dev/null +++ b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ExceptionPropagatesThroughAsyncFlowTest.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Samples.Probes.TestRuns.ExceptionReplay +{ + [ExceptionReplayTestData(expectedNumberOfSnapshotsDefault: 5, expectedNumberOfSnaphotsFull: 5)] + internal class DeterministicComplexExceptionPropagationTest : IAsyncRun + { + public async Task RunAsync() + { + try + { + await Task.Yield(); // Ensure we're running on a thread pool thread + await InitiateComplexExceptionChain(); + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + throw; + } + } + + private async Task InitiateComplexExceptionChain() + { + try + { + await SimulateComplexAsyncOperations(5); + } + catch (Exception ex) + { + throw new ApplicationException("Top-level exception in InitiateComplexExceptionChain", ex); + } + } + + private async Task SimulateComplexAsyncOperations(int depth) + { + if (depth <= 0) + { + throw new InvalidOperationException("Reached maximum depth"); + } + + await Task.Yield(); // Force continuation on a different thread + + try + { + var tasks = new List + { + SimulateAsyncOperation($"Operation-{depth}-A", depth % 2 == 0), + SimulateAsyncOperation($"Operation-{depth}-B", depth % 3 == 0), + SimulateAsyncOperation($"Operation-{depth}-C", depth % 5 == 0) + }; + + await Task.WhenAll(tasks); + + await SimulateComplexAsyncOperations(depth - 1); + } + catch (Exception ex) + { + throw new CustomAggregateException($"Multiple exceptions at depth {depth}", GenerateInnerExceptions(ex, 3)); + } + } + + private async Task SimulateAsyncOperation(string operationName, bool shouldThrow) + { + await Task.Yield(); // Force continuation on a different thread + + // Simulate some work + await Task.Delay(30); + + if (shouldThrow) + { + throw new TimeoutException($"Operation {operationName} timed out"); + } + + await SimulateNestedAsyncOperation(operationName, !shouldThrow); + } + + private async Task SimulateNestedAsyncOperation(string parentOperation, bool shouldThrow) + { + await Task.Yield(); // Force continuation on a different thread + + // Simulate some work + await Task.Delay(20); + + if (shouldThrow) + { + throw new InvalidOperationException($"Nested operation for {parentOperation} failed"); + } + + // Simulate a CPU-bound operation + await Task.Run(() => + { + if (parentOperation.Contains("A")) + { + throw new ArithmeticException("Error in CPU-bound calculation for Operation A"); + } + }); + } + + private IEnumerable GenerateInnerExceptions(Exception originalException, int count) + { + yield return originalException; + + for (int i = 0; i < count - 1; i++) + { + yield return new Exception($"Additional inner exception {i + 1}", + new InvalidOperationException($"Nested invalid operation {i + 1}")); + } + } + } + + public class CustomAggregateException : AggregateException + { + public CustomAggregateException(string message, IEnumerable innerExceptions) + : base(message, innerExceptions) { } + + public override string ToString() + { + return $"{Message}\n{string.Join("\n", InnerExceptions.Select(ex => ex.ToString()))}"; + } + } +} diff --git a/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/RethrowTest.cs b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/RethrowTest.cs new file mode 100644 index 000000000000..1a13d3885e0a --- /dev/null +++ b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/RethrowTest.cs @@ -0,0 +1,220 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Text; +using System.Threading.Tasks; + +#if !NET461_OR_GREATER && !NETCOREAPP2_1 + +namespace Samples.Probes.TestRuns.ExceptionReplay +{ + [ExceptionReplayTestData(expectedNumberOfSnapshotsDefault: 4, expectedNumberOfSnaphotsFull: 4)] + internal class RethrowTest : IAsyncRun + { + public async Task RunAsync() + { + await Task.Yield(); + + try + { + try + { + await Task.Yield(); + throw new Exception("Bdika"); + } + catch (Exception e) + { + await Task.Yield(); + await Task.Yield(); + await Task.Yield(); + await RunAsync2(); + throw; + } + } + finally + { + await Task.Yield(); + } + + } + + public async Task RunAsync2() + { + await Task.Yield(); + + try + { + try + { + await Task.Yield(); + await InTheMiddle(); + } + catch (Exception e) + { + //await Task.Yield(); + //await Task.Yield(); + //await Task.Yield(); + throw; + await Task.Yield(); + } + } + catch (Exception ex) + { + throw; + await Task.Yield(); + } + + } + + public async Task InTheMiddle() + { + int num = 3; + try + { + await Task.Yield(); + RelationalDataReader _ = await AwaitUsingFunc().ConfigureAwait(false); + Exception obj = null; + try + { + await InTheMiddle2(); + } + catch (Exception obj2) + { + obj = obj2; + } + + if (_ != null) + { + await ((IAsyncDisposable)_).DisposeAsync(); + } + Exception obj3 = obj; + if (obj3 != null) + { + Exception obj4 = obj3 as Exception; + if (obj4 == null) + { + throw obj3; + } + + ExceptionDispatchInfo.Capture(obj4).Throw(); + } + } + catch (Exception ex) when (!(ex is NotImplementedException)) + { + throw new NotImplementedException("Outer", ex); + } + } + + public async Task InTheMiddle2() + { + try + { + await Task.Yield(); + await using var _ = await AwaitUsingFunc().ConfigureAwait(false); + await Foo(); + } + catch (Exception e) when (e is not NotImplementedException) + { + throw new NotImplementedException("Outer", inner: e); + } + } + + public class RelationalDataReader : IAsyncDisposable, IDisposable + { + private bool _disposed = false; + private bool _disposedAsync = false; + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + if (disposing) + { + // Dispose managed resources + } + + // Dispose unmanaged resources + _disposed = true; + } + } + + public async ValueTask DisposeAsync() + { + if (!_disposed) + { + // Perform async cleanup here + await DisposeAsyncCore().ConfigureAwait(false); + + _disposed = true; + GC.SuppressFinalize(this); + } + } + + protected virtual async ValueTask DisposeAsyncCore() + { + // Perform actual async disposal logic here + await Task.CompletedTask; // Replace with actual async cleanup if needed + } + + ~RelationalDataReader() + { + Dispose(false); + } + } + + private async ValueTask AwaitUsingFunc() + { + await Task.Yield(); + return new RelationalDataReader(); + } + + private async Task Foo() + { + await Task.Yield(); + + try + { + await Task.Yield(); + CaptureAndThrow(); + } + finally + { + await Task.Yield(); + } + } + + void CaptureAndThrow() + { + try + { + Bar(); + } + catch (Exception e) + { + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e).Throw(); + } + } + + private void Bar() + { + try + { + throw new NotImplementedException(); + } + catch (Exception ex) + { + Console.WriteLine(nameof(RunAsync)); + throw; + } + } + } +} + +#endif diff --git a/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ThrowOnlyOnceTest.cs b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ThrowOnlyOnceTest.cs new file mode 100644 index 000000000000..ae48502d3cd5 --- /dev/null +++ b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/ThrowOnlyOnceTest.cs @@ -0,0 +1,227 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Text; +using System.Threading.Tasks; + +#if !NET461_OR_GREATER && !NETCOREAPP2_1 + +namespace Samples.Probes.TestRuns.ExceptionReplay +{ + [ExceptionReplayTestData(expectedNumberOfSnapshotsDefault: 4, expectedNumberOfSnaphotsFull: 4)] + internal class ThrowOnlyOnceTest : IAsyncRun + { + private static bool _hasThrown; + + public async Task RunAsync() + { + if (_hasThrown) + { + return; + } + + await Task.Yield(); + + try + { + try + { + await Task.Yield(); + await RunAsync2(); + } + catch (Exception e) + { + await Task.Yield(); + await Task.Yield(); + await Task.Yield(); + throw; + } + } + finally + { + _hasThrown = true; + await Task.Yield(); + } + + } + + public async Task RunAsync2() + { + await Task.Yield(); + + try + { + try + { + await Task.Yield(); + await InTheMiddle(); + } + catch (Exception e) + { + //await Task.Yield(); + //await Task.Yield(); + //await Task.Yield(); + throw; + await Task.Yield(); + } + } + catch (Exception ex) + { + throw; + await Task.Yield(); + } + + } + + public async Task InTheMiddle() + { + int num = 3; + try + { + await Task.Yield(); + RelationalDataReader _ = await AwaitUsingFunc().ConfigureAwait(false); + Exception obj = null; + try + { + await InTheMiddle2(); + } + catch (Exception obj2) + { + obj = obj2; + } + + if (_ != null) + { + await ((IAsyncDisposable)_).DisposeAsync(); + } + Exception obj3 = obj; + if (obj3 != null) + { + Exception obj4 = obj3 as Exception; + if (obj4 == null) + { + throw obj3; + } + + ExceptionDispatchInfo.Capture(obj4).Throw(); + } + } + catch (Exception ex) when (!(ex is NotImplementedException)) + { + throw new NotImplementedException("Outer", ex); + } + } + + public async Task InTheMiddle2() + { + try + { + await Task.Yield(); + await using var _ = await AwaitUsingFunc().ConfigureAwait(false); + await Foo(); + } + catch (Exception e) when (e is not NotImplementedException) + { + throw new NotImplementedException("Outer", inner: e); + } + } + + public class RelationalDataReader : IAsyncDisposable, IDisposable + { + private bool _disposed = false; + private bool _disposedAsync = false; + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + if (disposing) + { + // Dispose managed resources + } + + // Dispose unmanaged resources + _disposed = true; + } + } + + public async ValueTask DisposeAsync() + { + if (!_disposed) + { + // Perform async cleanup here + await DisposeAsyncCore().ConfigureAwait(false); + + _disposed = true; + GC.SuppressFinalize(this); + } + } + + protected virtual async ValueTask DisposeAsyncCore() + { + // Perform actual async disposal logic here + await Task.CompletedTask; // Replace with actual async cleanup if needed + } + + ~RelationalDataReader() + { + Dispose(false); + } + } + + private async ValueTask AwaitUsingFunc() + { + await Task.Yield(); + return new RelationalDataReader(); + } + + private async Task Foo() + { + await Task.Yield(); + + try + { + await Task.Yield(); + CaptureAndThrow(); + } + finally + { + await Task.Yield(); + } + } + + void CaptureAndThrow() + { + try + { + Bar(); + } + catch (Exception e) + { + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e).Throw(); + } + } + + private void Bar() + { + try + { + throw new NotImplementedException(); + } + catch (Exception ex) + { + Console.WriteLine(nameof(RunAsync)); + throw; + } + } + } +} + +#endif diff --git a/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplayIntentionalException.cs b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplayIntentionalException.cs new file mode 100644 index 000000000000..832b144169da --- /dev/null +++ b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplayIntentionalException.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Samples.Probes.TestRuns +{ + public class ExceptionReplayIntentionalException : Exception + { + private const string Prefix = "Intentional exception that was thrown from an integration test"; + + public ExceptionReplayIntentionalException() + : base(Prefix) + { + } + + public ExceptionReplayIntentionalException(string message) + : base($"{Prefix}. {message}") + { + } + + public ExceptionReplayIntentionalException(string message, Exception innerException) + : base($"{Prefix}. {message}", innerException) + { + } + } +} From fa2c06d1d5eb567e9b8c94f51ee2fb70c8225c8f Mon Sep 17 00:00:00 2001 From: Matan Green Date: Wed, 15 Jan 2025 15:10:00 +0200 Subject: [PATCH 2/2] Improved Exception Replay frame matching algorithm + capturing exceptions that have duplicated frames due to await in finally blocks + added missing attributes to snapshot (exceptionHash, exceptionId, frameIndex) --- Datadog.Trace.sln | 24 +- exploration-tests/protobuf | 1 - .../ExceptionCaseInstrumentationManager.cs | 67 +-- .../ExceptionDebuggingProcessor.cs | 24 +- .../ExceptionNormalizer.cs | 81 +++- .../ExceptionProbeProcessor.cs | 14 +- .../ExceptionReplaySnapshotCreator.cs | 32 +- .../ExceptionTrackManager.cs | 200 +++++--- .../FakeTrackedStackFrameNode.cs | 47 ++ .../ILAnalyzer.cs | 117 +++++ .../MethodMatcher.cs | 454 ++++++++++++++++++ .../MethodUniqueIdentifier.cs | 13 + .../ShadowStackTree.cs | 6 + .../StackTraceProcessor.cs | 83 ++++ .../TrackedStackFrameNode.cs | 47 +- .../Debugger/Helpers/MethodExtensions.cs | 5 + .../Debugger/Helpers/ProbeStatusPollerMock.cs | 1 + .../debugger_constants.h | 7 +- ...ugger_probes_instrumentation_requester.cpp | 2 +- .../MockTracerAgent.cs | 2 +- .../Samples.Debugger.AspNetCore5/Startup.cs | 6 +- .../ExceptionReplay/RethrowTest.cs | 80 ++- 22 files changed, 1097 insertions(+), 216 deletions(-) delete mode 160000 exploration-tests/protobuf create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/FakeTrackedStackFrameNode.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ILAnalyzer.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodMatcher.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/StackTraceProcessor.cs diff --git a/Datadog.Trace.sln b/Datadog.Trace.sln index 5791973d2eeb..01f229af0fe7 100644 --- a/Datadog.Trace.sln +++ b/Datadog.Trace.sln @@ -612,14 +612,14 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Debug|Any CPU.ActiveCfg = Debug|x64 - {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Debug|Any CPU.Build.0 = Debug|x64 - {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Release|Any CPU.ActiveCfg = Release|x64 - {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Release|Any CPU.Build.0 = Release|x64 - {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Debug|Any CPU.ActiveCfg = Debug|x64 - {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Debug|Any CPU.Build.0 = Debug|x64 - {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Release|Any CPU.ActiveCfg = Release|x64 - {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Release|Any CPU.Build.0 = Release|x64 + {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Debug|Any CPU.ActiveCfg = Debug|Win32 + {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Debug|Any CPU.Build.0 = Debug|Win32 + {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Release|Any CPU.ActiveCfg = Release|Win32 + {91B6272F-5780-4C94-8071-DBBA7B4F67F3}.Release|Any CPU.Build.0 = Release|Win32 + {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Debug|Any CPU.ActiveCfg = Debug|Win32 + {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Debug|Any CPU.Build.0 = Debug|Win32 + {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Release|Any CPU.ActiveCfg = Release|Win32 + {C0C8D381-D6B9-4C76-9428-F40F2FA93A9A}.Release|Any CPU.Build.0 = Release|Win32 {5DFDF781-F24C-45B1-82EF-9125875A80A4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5DFDF781-F24C-45B1-82EF-9125875A80A4}.Debug|Any CPU.Build.0 = Debug|Any CPU {5DFDF781-F24C-45B1-82EF-9125875A80A4}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -648,10 +648,10 @@ Global {3C6DD42E-9214-4747-92BA-78DE29AACE59}.Debug|Any CPU.Build.0 = Debug|Any CPU {3C6DD42E-9214-4747-92BA-78DE29AACE59}.Release|Any CPU.ActiveCfg = Release|Any CPU {3C6DD42E-9214-4747-92BA-78DE29AACE59}.Release|Any CPU.Build.0 = Release|Any CPU - {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Debug|Any CPU.ActiveCfg = Debug|x64 - {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Debug|Any CPU.Build.0 = Debug|x64 - {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Release|Any CPU.ActiveCfg = Release|x64 - {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Release|Any CPU.Build.0 = Release|x64 + {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Debug|Any CPU.ActiveCfg = Debug|Win32 + {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Debug|Any CPU.Build.0 = Debug|Win32 + {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Release|Any CPU.ActiveCfg = Release|Win32 + {5728056A-51AA-4FF5-AD0C-E86E44E36102}.Release|Any CPU.Build.0 = Release|Win32 {FDB5C8D0-018D-4FF9-9680-C6A5078F819B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {FDB5C8D0-018D-4FF9-9680-C6A5078F819B}.Debug|Any CPU.Build.0 = Debug|Any CPU {FDB5C8D0-018D-4FF9-9680-C6A5078F819B}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/exploration-tests/protobuf b/exploration-tests/protobuf deleted file mode 160000 index 7c40b2df1fdf..000000000000 --- a/exploration-tests/protobuf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7c40b2df1fdf6f414c1c18c789715a9c948a0725 diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCaseInstrumentationManager.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCaseInstrumentationManager.cs index a05f9806be52..36f10fba3487 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCaseInstrumentationManager.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCaseInstrumentationManager.cs @@ -38,8 +38,8 @@ internal static ExceptionCase Instrument(ExceptionIdentifier exceptionId, string { Log.Information("Instrumenting {ExceptionId}", exceptionId); - var parsedFramesFromExceptionToString = ExceptionNormalizer.Instance.ParseFrames(exceptionToString).ToArray(); - var stackTrace = exceptionId.StackTrace.Where(frame => parsedFramesFromExceptionToString.Any(f => f.Contains(frame.Method.Name))).ToArray(); + var parsedFramesFromExceptionToString = StackTraceProcessor.ParseFrames(exceptionToString).ToArray(); + var stackTrace = exceptionId.StackTrace.Where(frame => parsedFramesFromExceptionToString.Any(f => MethodMatcher.IsMethodMatch(f, frame.Method))).ToArray(); var participatingUserMethods = GetMethodsToRejit(stackTrace); var uniqueMethods = participatingUserMethods @@ -84,53 +84,11 @@ bool ShouldInstrumentFrameAtIndex(int i) } } - private static bool ContainsExceptionDispatchInfoThrow(MethodBase method) - { - var methodBody = method.GetMethodBody(); - if (methodBody == null) - { - return false; - } - - byte[] ilBytes = methodBody.GetILAsByteArray(); - - if (ilBytes == null) - { - return false; - } - - for (int i = 0; i < ilBytes.Length; i++) - { - if (ilBytes[i] == (byte)OpCodes.Call.Value || ilBytes[i] == (byte)OpCodes.Callvirt.Value) - { - // The next 4 bytes after a call instruction contain the metadata token - if (i + 4 < ilBytes.Length) - { - int metadataToken = BitConverter.ToInt32(ilBytes, i + 1); - try - { - MethodInfo calledMethod = (MethodInfo)method.Module.ResolveMethod(metadataToken); - if (calledMethod.DeclaringType == typeof(System.Runtime.ExceptionServices.ExceptionDispatchInfo) && - calledMethod.Name == "Throw") - { - return true; - } - } - catch (ArgumentException) - { - // If we can't resolve the method, just continue - continue; - } - } - } - } - - return false; - } - private static List GetMethodsToRejit(ParticipatingFrame[] allFrames) { var methodsToRejit = new List(); + MethodUniqueIdentifier? lastMethod = null; + var wasLastMisleading = false; foreach (var frame in allFrames) { @@ -149,11 +107,24 @@ private static List GetMethodsToRejit(ParticipatingFrame continue; } - methodsToRejit.Add(frame.MethodIdentifier); + var currentMethod = frame.MethodIdentifier; + var isCurrentMisleading = currentMethod.IsMisleadMethod(); + + // Add the method if either: + // 1. It's not misleading (we keep all non-misleading methods) + // 2. It's misleading but different from the last misleading method we saw + // 3. It's the first misleading method after non-misleading methods + if (!isCurrentMisleading || currentMethod != lastMethod || !wasLastMisleading) + { + methodsToRejit.Add(currentMethod); + } + + lastMethod = currentMethod; + wasLastMisleading = isCurrentMisleading; } catch (Exception ex) { - Log.Error(ex, "Failed to instrument frame the frame: {FrameToRejit}", frame); + Log.Error(ex, "Failed to instrument the frame: {FrameToRejit}", frame); } } diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProcessor.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProcessor.cs index 2ef2cbd4ff13..93350c0e397a 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProcessor.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProcessor.cs @@ -19,6 +19,7 @@ internal class ExceptionDebuggingProcessor : IProbeProcessor private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ExceptionDebuggingProcessor)); private readonly object _lock = new(); private readonly int _maxFramesToCapture; + private readonly bool _isMisleadingMethod; private ExceptionProbeProcessor[] _processors; internal ExceptionDebuggingProcessor(string probeId, MethodUniqueIdentifier method) @@ -27,6 +28,7 @@ internal ExceptionDebuggingProcessor(string probeId, MethodUniqueIdentifier meth ProbeId = probeId; Method = method; _maxFramesToCapture = ExceptionDebugging.Settings.MaximumFramesToCapture; + _isMisleadingMethod = method.IsMisleadMethod(); } public string ProbeId { get; } @@ -68,7 +70,17 @@ public bool Process(ref CaptureInfo info, IDebuggerSnapshotC case MethodState.EntryStart: case MethodState.EntryAsync: shadowStack = ShadowStackHolder.EnsureShadowStackEnabled(); - snapshotCreator.EnterHash = shadowStack.CurrentStackFrameNode?.EnterSequenceHash ?? Fnv1aHash.FnvOffsetBias; + var currentFrame = shadowStack.CurrentStackFrameNode; + snapshotCreator.EnterHash = Fnv1aHash.Combine(info.Method.MetadataToken, shadowStack.CurrentStackFrameNode?.EnterSequenceHash ?? Fnv1aHash.FnvOffsetBias); + + if (currentFrame?.Method == info.Method && _isMisleadingMethod) + { + // Methods marked as `misleading` are methods we tolerate being in the shadow stack multiple times. + // We flatten those methods due to `ExceptionDispatchInfo.Capture(X).Throw()` API causing frames to appear twice + // while in reality they were involved only once. + snapshotCreator.TrackedStackFrameNode = shadowStack.EnterFake(info.Method); + return true; + } var shouldProcess = false; foreach (var processor in snapshotCreator.Processors) @@ -113,7 +125,15 @@ public bool Process(ref CaptureInfo info, IDebuggerSnapshotC var exception = info.Value as Exception; snapshotCreator.TrackedStackFrameNode.LeavingException = exception; - snapshotCreator.LeaveHash = shadowStack.CurrentStackFrameNode?.LeaveSequenceHash ?? Fnv1aHash.FnvOffsetBias; + snapshotCreator.LeaveHash = shadowStack.CurrentStackFrameNode!.LeaveSequenceHash; + + if (snapshotCreator.TrackedStackFrameNode is FakeTrackedStackFrameNode) + { + shadowStack.Leave(snapshotCreator.TrackedStackFrameNode, exception); + snapshotCreator.TrackedStackFrameNode.CapturingStrategy = SnapshotCapturingStrategy.FullSnapshot; + snapshotCreator.TrackedStackFrameNode.AddScopeMember(info.Name, info.Type, info.Value, info.MemberKind); + return true; + } var leavingExceptionType = info.Value.GetType(); diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionNormalizer.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionNormalizer.cs index 1869b3d84ecb..6a39312b15c4 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionNormalizer.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionNormalizer.cs @@ -104,54 +104,83 @@ protected virtual int HashLine(VendoredMicrosoftCode.System.ReadOnlySpan l } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal IEnumerable ParseFrames(string exceptionString) + internal List ParseFrames(string exceptionString) { if (string.IsNullOrEmpty(exceptionString)) { throw new ArgumentException(@"Exception string cannot be null or empty", nameof(exceptionString)); } - var exceptionSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(exceptionString); - var inSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(" in "); - var atSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("at "); - var lambdaSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("lambda_"); - var datadogSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("at Datadog."); + var results = new List(); + var currentSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(exceptionString); - while (!exceptionSpan.IsEmpty) + while (!currentSpan.IsEmpty) { - var lineEndIndex = exceptionSpan.IndexOfAny('\r', '\n'); + var lineEndIndex = currentSpan.IndexOfAny('\r', '\n'); VendoredMicrosoftCode.System.ReadOnlySpan line; if (lineEndIndex >= 0) { - line = exceptionSpan.Slice(0, lineEndIndex); - exceptionSpan = exceptionSpan.Slice(lineEndIndex + 1); - if (!exceptionSpan.IsEmpty && exceptionSpan[0] == '\n') + line = currentSpan.Slice(0, lineEndIndex); + currentSpan = currentSpan.Slice(lineEndIndex + 1); + if (!currentSpan.IsEmpty && currentSpan[0] == '\n') { - exceptionSpan = exceptionSpan.Slice(1); + currentSpan = currentSpan.Slice(1); } } else { - line = exceptionSpan; - exceptionSpan = default; + line = currentSpan; + currentSpan = default; } - // Is frame line (starts with `in `). - if (VendoredMicrosoftCode.System.MemoryExtensions.StartsWith(line.TrimStart(), atSpan, StringComparison.Ordinal)) - { - var index = VendoredMicrosoftCode.System.MemoryExtensions.IndexOf(line, inSpan, StringComparison.Ordinal); - line = index > 0 ? line.Slice(0, index) : line; + ProcessLine(line, results); + } - if (VendoredMicrosoftCode.System.MemoryExtensions.Contains(line, lambdaSpan, StringComparison.Ordinal) || - VendoredMicrosoftCode.System.MemoryExtensions.Contains(line, datadogSpan, StringComparison.Ordinal)) - { - continue; - } + return results; + } - yield return line.ToString(); - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ProcessLine(VendoredMicrosoftCode.System.ReadOnlySpan line, List results) + { + line = line.TrimStart(); + if (line.IsEmpty) + { + return; + } + + // Check if it's a stack frame line (starts with "at ") + if (!VendoredMicrosoftCode.System.MemoryExtensions.StartsWith(line, VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("at "), StringComparison.Ordinal)) + { + return; + } + + // Skip the "at " prefix + line = line.Slice(3); + + // Skip lambda and Datadog frames early + if (ContainsAny(line, "lambda_", "at Datadog.")) + { + return; + } + + // Find the " in " marker and truncate if found + var inIndex = VendoredMicrosoftCode.System.MemoryExtensions.IndexOf(line, VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(" in "), StringComparison.Ordinal); + + if (inIndex > 0) + { + line = line.Slice(0, inIndex); } + + // Only create a string when we're sure we want to keep this frame + results.Add(line.ToString()); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool ContainsAny(VendoredMicrosoftCode.System.ReadOnlySpan source, string first, string second) + { + return VendoredMicrosoftCode.System.MemoryExtensions.Contains(source, VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(first), StringComparison.Ordinal) || + VendoredMicrosoftCode.System.MemoryExtensions.Contains(source, VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(second), StringComparison.Ordinal); } } } diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbeProcessor.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbeProcessor.cs index b804dd13c525..0bffe86f733f 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbeProcessor.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbeProcessor.cs @@ -116,19 +116,17 @@ private bool EnsureLeaveHashComputed() Array.Reverse(installedProbes); } - if (!installedProbes.Any()) - { - return Fnv1aHash.FnvOffsetBias; - } - var hash = Fnv1aHash.FnvOffsetBias; - foreach (var probe in installedProbes) + if (installedProbes.Any()) { - hash = Fnv1aHash.Combine(probe.Method.MethodToken, hash); + foreach (var probe in installedProbes) + { + hash = Fnv1aHash.Combine(probe.Method.Method.MetadataToken, hash); + } } - return hash; + return Fnv1aHash.Combine(ExceptionDebuggingProcessor.Method.Method.MetadataToken, hash); } internal void InvalidateEnterLeave() diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySnapshotCreator.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySnapshotCreator.cs index 1c3eb75d693c..81eaceb6ccf8 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySnapshotCreator.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySnapshotCreator.cs @@ -17,36 +17,32 @@ namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation { internal class ExceptionReplaySnapshotCreator : DebuggerSnapshotCreator { - private readonly string _exceptionHash; - private readonly string _exceptionCaptureId; - private readonly int _frameIndex; - - public ExceptionReplaySnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool hasCondition, string[] tags, CaptureLimitInfo limitInfo, string exceptionHash, string exceptionCaptureId, int frameIndex) + public ExceptionReplaySnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool hasCondition, string[] tags, CaptureLimitInfo limitInfo) : base(isFullSnapshot, location, hasCondition, tags, limitInfo) { - _exceptionHash = exceptionHash; - _exceptionCaptureId = exceptionCaptureId; - _frameIndex = frameIndex; } - public ExceptionReplaySnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool hasCondition, string[] tags, MethodScopeMembers methodScopeMembers, CaptureLimitInfo limitInfo, string exceptionHash, string exceptionCaptureId, int frameIndex) + public ExceptionReplaySnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool hasCondition, string[] tags, MethodScopeMembers methodScopeMembers, CaptureLimitInfo limitInfo) : base(isFullSnapshot, location, hasCondition, tags, methodScopeMembers, limitInfo) { - _exceptionHash = exceptionHash; - _exceptionCaptureId = exceptionCaptureId; - _frameIndex = frameIndex; } + internal static string ExceptionHash { get; } = Guid.NewGuid().ToString(); + + internal static string ExceptionCaptureId { get; } = Guid.NewGuid().ToString(); + + internal static string FrameIndex { get; } = Guid.NewGuid().ToString(); + internal override DebuggerSnapshotCreator EndSnapshot() { - JsonWriter.WritePropertyName("exception_hash"); - JsonWriter.WriteValue(_exceptionHash); + JsonWriter.WritePropertyName("exceptionHash"); + JsonWriter.WriteValue(ExceptionHash); - JsonWriter.WritePropertyName("exception_capture_id"); - JsonWriter.WriteValue(_exceptionCaptureId); + JsonWriter.WritePropertyName("exceptionId"); + JsonWriter.WriteValue(ExceptionCaptureId); - JsonWriter.WritePropertyName("frame_index"); - JsonWriter.WriteValue(_frameIndex); + JsonWriter.WritePropertyName("frameIndex"); + JsonWriter.WriteValue(FrameIndex); return base.EndSnapshot(); } diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs index 834fcb9807b4..0cb95b3d279f 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs @@ -19,6 +19,7 @@ using Datadog.Trace.Debugger.Sink.Models; using Datadog.Trace.Debugger.Snapshots; using Datadog.Trace.Debugger.Symbols; +using Datadog.Trace.DuckTyping; using Datadog.Trace.Logging; using Datadog.Trace.Util; using Datadog.Trace.VendoredMicrosoftCode.System.Buffers; @@ -55,7 +56,16 @@ private static async Task StartExceptionProcessingAsync(CancellationToken cancel while (ExceptionProcessQueue.TryDequeue(out var exception)) { - ProcessException(exception, 0, ErrorOriginKind.HttpRequestFailure, rootSpan: null); + try + { + ProcessException(exception, 0, ErrorOriginKind.HttpRequestFailure, rootSpan: null); + } +#pragma warning disable DD0001 + catch (Exception ex) + { + Log.Error(ex, "An exception was thrown while processing an exception for tracking from background thread. Exception = {Exception}", exception.ToString()); + } +#pragma warning restore DD0001 } } } @@ -223,9 +233,9 @@ private static void ProcessException(Exception exception, int normalizedExHash, SetDiagnosticTag(rootSpan, ExceptionReplayDiagnosticTagNames.InvalidatedExceptionCase, normalizedExHash); - var allFrames = trackedExceptionCase.ExceptionCase.ExceptionId.StackTrace; + var allFrames = StackTraceProcessor.ParseFrames(exception.ToString()); var allProbes = trackedExceptionCase.ExceptionCase.Probes; - var frameIndex = allFrames.Length - 1; + var frameIndex = allFrames.Count - 1; var debugErrorPrefix = "_dd.debug.error"; var assignIndex = 0; @@ -237,11 +247,11 @@ private static void ProcessException(Exception exception, int normalizedExHash, while (frameIndex >= 0) { var participatingFrame = allFrames[frameIndex--]; - var noCaptureReason = GetNoCaptureReason(participatingFrame, allProbes.FirstOrDefault(p => p.Method.Equals(participatingFrame.MethodIdentifier))); + var noCaptureReason = GetNoCaptureReason(participatingFrame, allProbes.FirstOrDefault(p => MethodMatcher.IsMethodMatch(participatingFrame, p.Method.Method))); if (noCaptureReason != string.Empty) { - TagMissingFrame(rootSpan, $"{debugErrorPrefix}.{assignIndex}.", participatingFrame.Method, noCaptureReason); + TagMissingFrame(rootSpan, $"{debugErrorPrefix}.{assignIndex}.", participatingFrame, noCaptureReason); } assignIndex += 1; @@ -325,90 +335,142 @@ private static void ProcessException(Exception exception, int normalizedExHash, { if (rootSpan == null) { - Log.Error("The RootSpan is null in the branch of extracing snapshots. Should not happen. Exception: {Exception}", exception.ToString()); + Log.Error("The RootSpan is null in the branch of extracting snapshots. Should not happen. Exception: {Exception}", exception.ToString()); return; } + var exceptionCaptureId = Guid.NewGuid().ToString(); + // Attach tags to the root span var debugErrorPrefix = "_dd.debug.error"; rootSpan.Tags.SetTag("error.debug_info_captured", "true"); rootSpan.Tags.SetTag($"{debugErrorPrefix}.exception_hash", trackedExceptionCase.ErrorHash); - rootSpan.Tags.SetTag($"{debugErrorPrefix}.exception_id", Guid.NewGuid().ToString()); + rootSpan.Tags.SetTag($"{debugErrorPrefix}.exception_id", exceptionCaptureId); var @case = trackedExceptionCase.ExceptionCase; var capturedFrames = resultCallStackTree.Frames; - var allFrames = @case.ExceptionId.StackTrace; + var allFrames = StackTraceProcessor.ParseFrames(exception.ToString()); + var frameIndex = allFrames.Count - 1; + var uploadedHeadFrame = false; // Upload head frame - var frameIndex = 0; - - while (frameIndex < allFrames.Length && - !allFrames[frameIndex].Method.Equals(capturedFrames[0].MethodInfo.Method)) + if (capturedFrames[0].MethodInfo.Method.Equals(@case.Probes[0].Method.Method)) { - frameIndex += 1; - } - - var frame = capturedFrames[0]; - TagAndUpload(rootSpan, $"{debugErrorPrefix}.{allFrames.Length - frameIndex - 1}.", frame); - - // Upload tail frames - frameIndex = allFrames.Length - 1; - var capturedFrameIndex = capturedFrames.Count - 1; - var assignIndex = 0; + while (frameIndex >= 0 && !MethodMatcher.IsMethodMatch(allFrames[frameIndex], capturedFrames[0].MethodInfo.Method)) + { + // Just processing the frames until a match is found + frameIndex -= 1; + } - while (capturedFrameIndex >= 1 && frameIndex >= 0) + TagAndUpload(rootSpan, $"{debugErrorPrefix}.{frameIndex}.", capturedFrames[0], exceptionId: exceptionCaptureId, exceptionHash: trackedExceptionCase.ErrorHash, frameIndex: frameIndex); + uploadedHeadFrame = true; + } + else { - frame = capturedFrames[capturedFrameIndex]; - - var participatingFrame = allFrames[frameIndex--]; + // Missing head + var probe = @case.Probes[0]; + var noCaptureReason = GetNoCaptureReason(probe.Method.Method.Name, probe); - if (!participatingFrame.Method.Equals(frame.MethodInfo.Method)) + if (noCaptureReason != string.Empty) { - var noCaptureReason = GetNoCaptureReason(participatingFrame, @case.Probes.FirstOrDefault(p => p.Method.Equals(participatingFrame.MethodIdentifier))); - - if (noCaptureReason != string.Empty) + while (frameIndex >= 0 && !MethodMatcher.IsMethodMatch(allFrames[frameIndex], @case.Probes[0].Method.Method)) { - TagMissingFrame(rootSpan, $"{debugErrorPrefix}.{assignIndex}.", participatingFrame.Method, noCaptureReason); + // Just processing the frames until a match is found + frameIndex -= 1; } - assignIndex += 1; - continue; + TagMissingFrame(rootSpan, $"{debugErrorPrefix}.{frameIndex}.", probe.Method.Method.Name, noCaptureReason); } - - capturedFrameIndex -= 1; - - var prefix = $"{debugErrorPrefix}.{assignIndex++}."; - TagAndUpload(rootSpan, prefix, frame); } - // Upload missing frames - var maxFramesToCaptureIncludingHead = MaxFramesToCapture + 1; - if (capturedFrames.Count < maxFramesToCaptureIncludingHead && - allFrames.Length > capturedFrames.Count) + frameIndex = 0; + var capturedFrameIndex = capturedFrames.Count - 1; + var probeIndex = @case.Probes.Length - 1; + var capturedFrameIndexBound = uploadedHeadFrame ? 0 : -1; + var uploadFramesBound = MaxFramesToCapture; + var uploadedFrames = 0; + while (frameIndex < allFrames.Count && uploadedFrames < uploadFramesBound && probeIndex >= 0) { - frameIndex = allFrames.Length - 1; - var probesIndex = @case.Probes.Length - 1; - assignIndex = 0; - - while (frameIndex >= 0 && maxFramesToCaptureIncludingHead > 0) + if (capturedFrameIndex <= capturedFrameIndexBound) { - maxFramesToCaptureIncludingHead -= 1; - var participatingFrame = allFrames[frameIndex--]; + // No 'captured frames' left for matching + while (probeIndex >= 0) + { + var noCaptureReason = GetNoCaptureReason(@case.Probes[probeIndex].Method.Method.Name, @case.Probes[probeIndex]); - var noCaptureReason = GetNoCaptureReasonForFrame(participatingFrame); + if (noCaptureReason != string.Empty) + { + while (frameIndex < allFrames.Count && !MethodMatcher.IsMethodMatch(allFrames[frameIndex], @case.Probes[probeIndex].Method.Method)) + { + // Just processing the frames until a match is found + frameIndex += 1; + } + + if (frameIndex >= allFrames.Count) + { + // Nothing left to match + break; + } + + TagMissingFrame(rootSpan, $"{debugErrorPrefix}.{frameIndex}.", @case.Probes[probeIndex].Method.Method.Name, noCaptureReason); + } + + probeIndex -= 1; + } - if (noCaptureReason == string.Empty && probesIndex >= 0) + break; + } + + while (probeIndex >= 0 && !capturedFrames[capturedFrameIndex].MethodInfo.Method.Equals(@case.Probes[probeIndex].Method.Method)) + { + // Determine if the frame is misleading (e.g duplicated on the stack thanks for ExceptionCaptureInfo.Throw) + var prevIndex = probeIndex + 1; + if (prevIndex < @case.Probes.Length && capturedFrames[capturedFrameIndex].MethodInfo.Method.Equals(@case.Probes[prevIndex].Method.Method) && @case.Probes[prevIndex].Method.IsMisleadMethod()) { - noCaptureReason = GetNoCaptureReason(participatingFrame, @case.Probes[probesIndex--]); + // The current captured frame is marked as misleading. Consider the previous probe as the 'current' probe for a proper matching in this extremely rare case + Log.Warning("Encountered misleading frame that is also recursive with exception: {Exception}, Method: {MethodName}", exception.ToString(), capturedFrames[capturedFrameIndex].MethodInfo.Method.GetFullName()); + probeIndex += 1; + break; } + var noCaptureReason = GetNoCaptureReason(@case.Probes[probeIndex].Method.Method.Name, @case.Probes[probeIndex]); + if (noCaptureReason != string.Empty) { - TagMissingFrame(rootSpan, $"{debugErrorPrefix}.{assignIndex}.", participatingFrame.Method, noCaptureReason); + while (frameIndex < allFrames.Count && !MethodMatcher.IsMethodMatch(allFrames[frameIndex], @case.Probes[probeIndex].Method.Method)) + { + // Just processing the frames until a match is found + frameIndex += 1; + } + + TagMissingFrame(rootSpan, $"{debugErrorPrefix}.{frameIndex}.", @case.Probes[probeIndex].Method.Method.Name, noCaptureReason); } - assignIndex++; + probeIndex -= 1; } + + if (probeIndex < 0) + { + // We exhausted the probes array, nothing left to match + break; + } + + while (frameIndex < allFrames.Count && !MethodMatcher.IsMethodMatch(allFrames[frameIndex], capturedFrames[capturedFrameIndex].MethodInfo.Method)) + { + frameIndex += 1; + } + + if (frameIndex >= allFrames.Count) + { + // We exhausted the whole frames array, nothing left to match + break; + } + + TagAndUpload(rootSpan, $"{debugErrorPrefix}.{frameIndex}.", capturedFrames[capturedFrameIndex], exceptionId: exceptionCaptureId, exceptionHash: trackedExceptionCase.ErrorHash, frameIndex: frameIndex); + capturedFrameIndex -= 1; + probeIndex -= 1; + frameIndex += 1; + uploadedFrames += 1; } Log.Information("Reverting an exception case for exception: {Name}, Message: {Message}, StackTrace: {StackTrace}", exception.GetType().Name, exception.Message, exception.StackTrace); @@ -432,31 +494,30 @@ private static void ProcessException(Exception exception, int normalizedExHash, } } - private static string GetNoCaptureReason(ParticipatingFrame frame, ExceptionDebuggingProbe? probe) + private static string GetNoCaptureReason(string methodName, ExceptionDebuggingProbe? probe) { - var noCaptureReason = GetNoCaptureReasonForFrame(frame); - - if (noCaptureReason != string.Empty) - { - return noCaptureReason; - } + var noCaptureReason = string.Empty; if (probe != null) { if (probe.MayBeOmittedFromCallStack) { // The process is spawned with `COMPLUS_ForceEnc` & the module of the method is non-optimized. - noCaptureReason = $"The method {frame.Method.GetFullyQualifiedName()} could not be captured because the process is spawned with Edit and Continue feature turned on and the module is compiled as Debug. Set the environment variable `COMPLUS_ForceEnc` to `0`. For further info, visit: https://github.com/dotnet/runtime/issues/91963."; + noCaptureReason = $"The method {methodName} could not be captured because the process is spawned with Edit and Continue feature turned on and the module is compiled as Debug. Set the environment variable `COMPLUS_ForceEnc` to `0`. For further info, visit: https://github.com/dotnet/runtime/issues/91963."; } else if (probe.ProbeStatus == Status.ERROR) { // Frame is failed to instrument. - noCaptureReason = $"The method {frame.Method.GetFullyQualifiedName()} has failed in instrumentation. Failure reason: {probe.ErrorMessage}"; + noCaptureReason = $"The method {methodName} has failed in instrumentation. Failure reason: {probe.ErrorMessage}"; } else if (probe.ProbeStatus == Status.RECEIVED) { // Frame is failed to instrument. - noCaptureReason = $"The method {frame.Method.GetFullyQualifiedName()} could not be found."; + noCaptureReason = $"The method {methodName} could not be found."; + } + else if (probe.Method.IsMisleadMethod()) + { + noCaptureReason = $"This frame of {methodName} is a duplication, due to `ExceptionDispatchInfo.Throw()`."; } } @@ -480,7 +541,7 @@ private static string GetNoCaptureReasonForFrame(ParticipatingFrame frame) return noCaptureReason; } - private static void TagAndUpload(Span span, string tagPrefix, ExceptionStackNodeRecord record) + private static void TagAndUpload(Span span, string tagPrefix, ExceptionStackNodeRecord record, string exceptionId, string exceptionHash, int frameIndex) { var method = record.MethodInfo.Method; var snapshotId = record.SnapshotId; @@ -491,13 +552,16 @@ private static void TagAndUpload(Span span, string tagPrefix, ExceptionStackNode span.Tags.SetTag(tagPrefix + "frame_data.class_name", method.DeclaringType?.Name); span.Tags.SetTag(tagPrefix + "snapshot_id", snapshotId); + snapshot = snapshot + .Replace(ExceptionReplaySnapshotCreator.ExceptionCaptureId, exceptionId) + .Replace(ExceptionReplaySnapshotCreator.ExceptionHash, exceptionHash) + .Replace(ExceptionReplaySnapshotCreator.FrameIndex, frameIndex.ToString()); ExceptionDebugging.AddSnapshot(probeId, snapshot); } - private static void TagMissingFrame(Span span, string tagPrefix, MethodBase method, string reason) + private static void TagMissingFrame(Span span, string tagPrefix, string method, string reason) { - span.Tags.SetTag(tagPrefix + "frame_data.function", method.Name); - span.Tags.SetTag(tagPrefix + "frame_data.class_name", method.DeclaringType?.Name); + span.Tags.SetTag(tagPrefix + "frame_data.name", method); span.Tags.SetTag(tagPrefix + "no_capture_reason", reason); } diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/FakeTrackedStackFrameNode.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/FakeTrackedStackFrameNode.cs new file mode 100644 index 000000000000..b122752c57a2 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/FakeTrackedStackFrameNode.cs @@ -0,0 +1,47 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Debugger.Instrumentation.Collections; +using Datadog.Trace.Debugger.Snapshots; +using Fnv1aHash = Datadog.Trace.VendoredMicrosoftCode.System.Reflection.Internal.Hash; + +#nullable enable +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class FakeTrackedStackFrameNode(TrackedStackFrameNode? parent, MethodBase method) + : TrackedStackFrameNode(parent, method, isInvalidPath: false) + { + protected override int ComputeEnterSequenceHash() + { + return Parent!.EnterSequenceHash; + } + + protected override int ComputeLeaveSequenceHash() + { + lock (this) + { + ClearNonRelevantChildNodes(); + + if (ActiveChildNodes?.Any() == true) + { + var firstChild = ActiveChildNodes.First(); + return firstChild.LeaveSequenceHash; + } + + return Fnv1aHash.Combine(Method.MetadataToken, Fnv1aHash.FnvOffsetBias); + } + } + + public override string ToString() + { + return $"{nameof(FakeTrackedStackFrameNode)}(Child Count = {ActiveChildNodes?.Count})"; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ILAnalyzer.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ILAnalyzer.cs new file mode 100644 index 000000000000..b488a6f690be --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ILAnalyzer.cs @@ -0,0 +1,117 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; + +#nullable enable +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal static class ILAnalyzer + { + public static bool HasDirectCallTo(MethodBase method, Type targetType, string methodName) + { + try + { + var effectiveMethod = GetEffectiveMethod(method); + return AnalyzeMethodBody(effectiveMethod, targetType, methodName); + } + catch + { + // If we can't analyze the method for any reason, assume it might not contain the call + return false; + } + } + + private static MethodBase GetEffectiveMethod(MethodBase method) + { + // For async methods, analyze the MoveNext method + if (method.GetCustomAttribute() is AsyncStateMachineAttribute asyncAttribute) + { + var stateMachineType = asyncAttribute.StateMachineType; + return stateMachineType.GetMethod( + "MoveNext", + BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public) ?? method; + } + + return method; + } + + private static bool AnalyzeMethodBody(MethodBase methodToAnalyze, Type targetType, string methodName) + { + var methodBody = methodToAnalyze.GetMethodBody(); + if (methodBody == null) + { + return false; + } + + var il = methodBody.GetILAsByteArray(); + if (il == null) + { + return false; + } + + int position = 0; + while (position < il.Length) + { + // Read opcode + int opcode = il[position++]; + if (opcode == 0xFE && position < il.Length) + { + opcode = 0xFE00 | il[position++]; + } + + // Check for call instructions + if (opcode == OpCodes.Call.Value || opcode == OpCodes.Callvirt.Value) + { + // Ensure we have enough bytes for the token + if (position + 3 >= il.Length) + { + break; + } + + int token = il[position] | + (il[position + 1] << 8) | + (il[position + 2] << 16) | + (il[position + 3] << 24); + + position += 4; + + try + { + var calledMethod = methodToAnalyze?.Module?.ResolveMethod(token); + if (calledMethod?.DeclaringType == targetType && + calledMethod.Name == methodName) + { + return true; + } + } + catch + { + // If we can't resolve this specific method, continue checking others + continue; + } + } + else + { + // Skip operands for other opcodes + position += GetOperandSize(opcode); + } + } + + return false; + } + + private static int GetOperandSize(int opcode) => opcode switch + { + 0x28 or 0x6F or 0x70 or 0x11 or 0x20 => 4, // Various 4-byte operand instructions + 0x73 or 0x6E => 4, // Newobj, Ldtoken + 0x1B or 0x31 or 0x15 or 0x2A => 1, // Various 1-byte operand instructions + _ => 0 + }; + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodMatcher.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodMatcher.cs new file mode 100644 index 000000000000..5067077d6792 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodMatcher.cs @@ -0,0 +1,454 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; + +#nullable enable +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation; + +internal static class MethodMatcher +{ + private static readonly ConcurrentDictionary AsyncMethodCache = new(); + + /// + /// Attempts to match a method name from a stack trace with a MethodBase, + /// handling special cases like async methods, iterators, and nested methods. + /// + internal static bool IsMethodMatch(string stackTraceMethodText, MethodBase methodBase) + { + // First try to resolve special method types (async, iterator, etc.) + var resolvedMethod = ResolveSpecialMethod(methodBase); + + // Use resolved method if available, otherwise use original + methodBase = resolvedMethod ?? methodBase; + + // Normalize and parse both representations + var normalizedStackTrace = NormalizeMethodText(stackTraceMethodText); + var methodBaseName = BuildMethodSignature(methodBase); + + var stackTraceParts = ParseMethodName(normalizedStackTrace); + var methodBaseParts = ParseMethodName(methodBaseName); + + if (stackTraceParts == null || methodBaseParts == null) + { + return normalizedStackTrace.Equals(methodBaseName, StringComparison.Ordinal); + } + + return DoMethodPartsMatch(stackTraceParts, methodBaseParts); + } + + private static MethodBase? ResolveSpecialMethod(MethodBase method) + { + // Check cache first + if (AsyncMethodCache.TryGetValue(method, out var cachedMethod)) + { + return cachedMethod; + } + + MethodBase? resolvedMethod = null; + + if (IsAsyncStateMachine(method)) + { + resolvedMethod = ResolveAsyncMethod(method); + } + else if (IsIteratorStateMachine(method)) + { + resolvedMethod = ResolveIteratorMethod(method); + } + else if (IsLocalFunction(method)) + { + resolvedMethod = ResolveLocalFunction(method); + } + + // Cache the result (null is a valid cache result) + AsyncMethodCache.TryAdd(method, resolvedMethod); + return resolvedMethod; + } + + private static bool IsAsyncStateMachine(MethodBase method) + { + if (method.Name != "MoveNext") + { + return false; + } + + var declaringType = method.DeclaringType; + if (declaringType == null) + { + return false; + } + + // Check both the type name pattern and implemented interfaces + return declaringType.Name.Contains(">d__") && + declaringType.GetInterfaces() + .Any(i => i.FullName == "System.Runtime.CompilerServices.IAsyncStateMachine"); + } + + private static bool IsIteratorStateMachine(MethodBase method) + { + if (method.Name != "MoveNext") + { + return false; + } + + var declaringType = method.DeclaringType; + if (declaringType == null) + { + return false; + } + + return declaringType.GetInterfaces() + .Any(i => i.FullName == "System.Collections.IEnumerator" || + (i.FullName?.StartsWith("System.Collections.Generic.IEnumerator`") ?? false)); + } + + private static bool IsLocalFunction(MethodBase method) + { + // Local functions have a generated name starting with "<" + return method.Name.StartsWith("<") && method.Name.Contains(">"); + } + + private static MethodBase? ResolveAsyncMethod(MethodBase moveNextMethod) + { + var stateMachineType = moveNextMethod.DeclaringType; + if (stateMachineType == null) + { + return null; + } + + // Handle nested types and generic type parameters + var containingType = ResolveContainingType(stateMachineType); + if (containingType == null) + { + return null; + } + + var originalMethodName = ExtractMethodNameFromStateMachine(stateMachineType.Name); + if (originalMethodName == null) + { + return null; + } + + // Find all methods with matching name + var candidates = containingType.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | + BindingFlags.Static | BindingFlags.Instance | + BindingFlags.DeclaredOnly) + .Where(m => m.Name == originalMethodName) + .ToList(); + + // If we have multiple candidates, try to match by async method builder attribute + if (candidates.Count > 1) + { + var matchingMethod = candidates.FirstOrDefault(HasMatchingAsyncMethodBuilder); + if (matchingMethod != null) + { + return matchingMethod; + } + } + + return candidates.FirstOrDefault(); + } + + private static Type? ResolveContainingType(Type stateMachineType) + { + var containingType = stateMachineType.DeclaringType; + if (containingType == null) + { + return null; + } + + // Handle nested generic types + if (containingType.IsGenericType && !containingType.IsGenericTypeDefinition) + { + // Get the generic type definition + containingType = containingType.GetGenericTypeDefinition(); + } + + return containingType; + } + + private static string? ExtractMethodNameFromStateMachine(string typeName) + { + var methodNameStart = typeName.IndexOf('<'); + var methodNameEnd = typeName.IndexOf('>'); + + if (methodNameStart < 0 || methodNameEnd <= methodNameStart) + { + return null; + } + + return typeName.Substring(methodNameStart + 1, methodNameEnd - methodNameStart - 1); + } + + private static bool HasMatchingAsyncMethodBuilder(MethodInfo method) + { + return method.GetCustomAttributes() + .Any(attr => attr.GetType().Name.EndsWith("AsyncStateMachineAttribute")); + } + + private static MethodBase? ResolveIteratorMethod(MethodBase moveNextMethod) + { + // Similar to async method resolution but looking for iterator patterns + var stateMachineType = moveNextMethod.DeclaringType; + if (stateMachineType == null) + { + return null; + } + + var containingType = ResolveContainingType(stateMachineType); + if (containingType == null) + { + return null; + } + + var originalMethodName = ExtractMethodNameFromStateMachine(stateMachineType.Name); + if (originalMethodName == null) + { + return null; + } + + return containingType.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | + BindingFlags.Static | BindingFlags.Instance) + .FirstOrDefault(m => m.Name == originalMethodName && + IsIteratorMethod(m)); + } + + private static bool IsIteratorMethod(MethodInfo method) + { + var returnType = method.ReturnType; + return typeof(System.Collections.IEnumerable).IsAssignableFrom(returnType) || + (returnType.IsGenericType && returnType.GetGenericTypeDefinition() == typeof(IEnumerable<>)); + } + + private static MethodBase? ResolveLocalFunction(MethodBase method) + { + // Extract the original method name from the local function name + var localFunctionName = method.Name; + var parentMethodStart = localFunctionName.IndexOf('<'); + var parentMethodEnd = localFunctionName.IndexOf('>', parentMethodStart); + + if (parentMethodStart < 0 || parentMethodEnd <= parentMethodStart) + { + return null; + } + + var parentMethodName = localFunctionName.Substring(parentMethodStart + 1, parentMethodEnd - parentMethodStart - 1); + + var declaringType = method.DeclaringType?.DeclaringType; + if (declaringType == null) + { + return null; + } + + return declaringType.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | + BindingFlags.Static | BindingFlags.Instance) + .FirstOrDefault(m => m.Name == parentMethodName); + } + + private static MethodNameParts? ParseMethodName(string fullName) + { + var parts = fullName.Split('.'); + if (parts.Length < 2) + { + return null; + } + + // The last part is always the method name + var methodName = NormalizeMethodName(parts[parts.Length - 1]); + + // Find where the type name starts (looking for capital letter after namespace) + var typeStartIndex = -1; + for (var i = 0; i < parts.Length - 1; i++) + { + if (parts[i].Length > 0 && char.IsUpper(parts[i][0])) + { + typeStartIndex = i; + break; + } + } + + if (typeStartIndex == -1) + { + return null; + } + + var namespaceParts = new string[typeStartIndex]; + Array.Copy(parts, 0, namespaceParts, 0, typeStartIndex); + + var typeParts = new string[parts.Length - typeStartIndex - 1]; + Array.Copy(parts, typeStartIndex, typeParts, 0, parts.Length - typeStartIndex - 1); + + return new MethodNameParts(namespaceParts, typeParts, methodName); + } + + private static bool DoMethodPartsMatch(MethodNameParts stackTrace, MethodNameParts methodBase) + { + // Namespace must match exactly + if (!AreArraysEqual(stackTrace.NamespaceParts, methodBase.NamespaceParts)) + { + return false; + } + + // Type parts must match exactly + if (!AreArraysEqual(stackTrace.TypeParts, methodBase.TypeParts)) + { + return false; + } + + // For method names, we need special comparison due to compiler-generated names + return AreMethodNamesEquivalent(stackTrace.MethodName, methodBase.MethodName); + } + + private static bool AreArraysEqual(string[] arr1, string[] arr2) + { + if (arr1.Length != arr2.Length) + { + return false; + } + + for (var i = 0; i < arr1.Length; i++) + { + if (!arr1[i].Equals(arr2[i], StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private static bool AreMethodNamesEquivalent(string stackTraceMethod, string methodBaseMethod) + { + // If they're exactly equal, we're done + if (stackTraceMethod.Equals(methodBaseMethod, StringComparison.Ordinal)) + { + return true; + } + + // For compiler-generated async methods + if (stackTraceMethod == "MoveNext" && + (methodBaseMethod.Contains("Async") || methodBaseMethod.Contains("<"))) + { + return true; + } + + // For compiler-generated methods, we need exact matches of the base name + var baseStackTraceName = RemoveAsyncSuffix(RemoveCompilerGeneratedParts(stackTraceMethod)); + var baseMethodName = RemoveAsyncSuffix(RemoveCompilerGeneratedParts(methodBaseMethod)); + + return baseStackTraceName.Equals(baseMethodName, StringComparison.Ordinal); + } + + private static string RemoveCompilerGeneratedParts(string methodName) + { + // Remove everything between < and > including the brackets + var startBracket = methodName.IndexOf('<'); + var endBracket = methodName.LastIndexOf('>'); + + if (startBracket >= 0 && endBracket > startBracket) + { + methodName = methodName.Substring(0, startBracket) + + methodName.Substring(endBracket + 1); + } + + // Remove any remaining compiler-generated suffixes (like "b__1") + var index = methodName.IndexOf("b__", StringComparison.Ordinal); + if (index > 0) + { + methodName = methodName.Substring(0, index); + } + + return methodName; + } + + private static string RemoveAsyncSuffix(string methodName) + { + return methodName.EndsWith("Async", StringComparison.Ordinal) + ? methodName.Substring(0, methodName.Length - 5) + : methodName; + } + + private static string NormalizeMethodText(string methodText) + { + // Remove parameter list if present (content within parentheses) + var parenthesesIndex = methodText.IndexOf('('); + if (parenthesesIndex > 0) + { + methodText = methodText.Substring(0, parenthesesIndex); + } + + // Special handling for generic method type arguments in stack trace format [T1,T2] + var genericIndex = methodText.IndexOf('['); + if (genericIndex > 0) + { + var closeIndex = methodText.IndexOf(']', genericIndex); + if (closeIndex > genericIndex) + { + // Count the number of type parameters + var typeParams = methodText.Substring(genericIndex + 1, closeIndex - genericIndex - 1) + .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) + .Length; + + // Replace with a normalized form + methodText = methodText.Substring(0, genericIndex) + "`" + typeParams; + } + } + + return methodText; + } + + private static string NormalizeMethodName(string methodName) + { + // Remove generic arity if present + var backtickIndex = methodName.IndexOf('`'); + if (backtickIndex > 0) + { + methodName = methodName.Substring(0, backtickIndex); + } + + return methodName; + } + + private static string BuildMethodSignature(MethodBase method) + { + var sb = new StringBuilder(); + + // Add declaring type's full name + if (method.DeclaringType != null) + { + var typeFullName = method.DeclaringType.FullName ?? method.DeclaringType.Name; + sb.Append(typeFullName.Replace("+", ".")); + } + + sb.Append('.'); + sb.Append(method.Name); + + return sb.ToString(); + } + + /// + /// Represents the parts of a method's full name + /// + private class MethodNameParts + { + public MethodNameParts(string[] namespaceParts, string[] typeParts, string methodName) + { + NamespaceParts = namespaceParts; + TypeParts = typeParts; + MethodName = methodName; + } + + public string[] NamespaceParts { get; } + + public string[] TypeParts { get; } + + public string MethodName { get; } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs index 409a9b56615e..adf71905fded 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs @@ -8,16 +8,29 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; #nullable enable namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation { internal readonly record struct MethodUniqueIdentifier(Guid Mvid, int MethodToken, MethodBase Method) { + internal static readonly ConcurrentDictionary MightMisleadStacktrace = new(); + public override int GetHashCode() { return HashCode.Combine(Mvid, MethodToken); } + + public bool IsMisleadMethod() + { + return MightMisleadStacktrace.GetOrAdd( + this, + identifier => ILAnalyzer.HasDirectCallTo( + identifier.Method, + typeof(System.Runtime.ExceptionServices.ExceptionDispatchInfo), + "Throw")); + } } internal readonly struct ExceptionCase : IEquatable diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackTree.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackTree.cs index 349bc3c0cb32..5370933c5949 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackTree.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackTree.cs @@ -46,6 +46,12 @@ public TrackedStackFrameNode Enter(MethodBase method, bool isInvalidPath = false return _trackedStackFrameActiveNode.Value; } + public TrackedStackFrameNode EnterFake(MethodBase method) + { + _trackedStackFrameActiveNode.Value = new FakeTrackedStackFrameNode(_trackedStackFrameActiveNode.Value, method); + return _trackedStackFrameActiveNode.Value; + } + public bool Leave(TrackedStackFrameNode trackedStackFrameNode, Exception? exception) { var currentActiveNode = _trackedStackFrameActiveNode.Value; diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/StackTraceProcessor.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/StackTraceProcessor.cs new file mode 100644 index 000000000000..e8189bf98e8a --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/StackTraceProcessor.cs @@ -0,0 +1,83 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using Datadog.Trace.Debugger.Helpers; +using Datadog.Trace.VendoredMicrosoftCode.System; + +#nullable enable +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation; + +internal static class StackTraceProcessor +{ + internal static List ParseFrames(string exceptionString) + { + if (string.IsNullOrEmpty(exceptionString)) + { + throw new ArgumentException(@"Exception string cannot be null or empty", nameof(exceptionString)); + } + + var results = new List(); + var currentSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(exceptionString); + + while (!currentSpan.IsEmpty) + { + var lineEndIndex = currentSpan.IndexOfAny('\r', '\n'); + VendoredMicrosoftCode.System.ReadOnlySpan line; + + if (lineEndIndex >= 0) + { + line = currentSpan.Slice(0, lineEndIndex); + currentSpan = currentSpan.Slice(lineEndIndex + 1); + if (!currentSpan.IsEmpty && currentSpan[0] == '\n') + { + currentSpan = currentSpan.Slice(1); + } + } + else + { + line = currentSpan; + currentSpan = default; + } + + ProcessLine(line, results); + } + + return results; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ProcessLine(VendoredMicrosoftCode.System.ReadOnlySpan line, List results) + { + line = line.TrimStart(); + if (line.IsEmpty) + { + return; + } + + // Check if it's a stack frame line (starts with "at ") + if (!VendoredMicrosoftCode.System.MemoryExtensions.StartsWith(line, VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("at "), StringComparison.Ordinal)) + { + return; + } + + // Skip the "at " prefix + line = line.Slice(3); + + // Find the " in " marker and truncate if found + var inIndex = VendoredMicrosoftCode.System.MemoryExtensions.IndexOf(line, VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(" in "), StringComparison.Ordinal); + + if (inIndex > 0) + { + line = line.Slice(0, inIndex); + } + + results.Add(line.ToString()); + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs index 7afccd2d91ca..a3854ff49027 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs @@ -18,7 +18,6 @@ namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation internal class TrackedStackFrameNode { private TrackedStackFrameNode? _parent; - private List? _activeChildNodes; private bool _disposed; private int? _enterSequenceHash; private int? _leaveSequenceHash; @@ -59,6 +58,8 @@ public int SequenceHash } } + protected List? ActiveChildNodes { get; private set; } + public TrackedStackFrameNode? Parent => _parent; public Exception? LeavingException { get; set; } @@ -89,7 +90,7 @@ public string SnapshotId public bool IsInvalidPath { get; } - public IEnumerable ChildNodes => _activeChildNodes?.ToList() ?? Enumerable.Empty(); + public IEnumerable ChildNodes => ActiveChildNodes?.ToList() ?? Enumerable.Empty(); public MethodBase Method { get; } @@ -133,7 +134,7 @@ private string CreateSnapshot() MaxFieldCount: DebuggerSettings.DefaultMaxNumberOfFieldsToCopy, MaxLength: DebuggerSettings.DefaultMaxStringLength); - using var snapshotCreator = new DebuggerSnapshotCreator(isFullSnapshot: true, location: ProbeLocation.Method, hasCondition: false, Array.Empty(), members, limitInfo: limitInfo); + using var snapshotCreator = new ExceptionReplaySnapshotCreator(isFullSnapshot: true, location: ProbeLocation.Method, hasCondition: false, Array.Empty(), members, limitInfo: limitInfo); _snapshotId = snapshotCreator.SnapshotId; @@ -219,7 +220,7 @@ internal void AddScopeMember(string name, Type type, T value, ScopeMemberKind } } - private int ComputeEnterSequenceHash() + protected virtual int ComputeEnterSequenceHash() { return Fnv1aHash.Combine(Method.MetadataToken, _parent?.EnterSequenceHash ?? Fnv1aHash.FnvOffsetBias); } @@ -227,19 +228,19 @@ private int ComputeEnterSequenceHash() /// /// TODO take not only first child. /// - private int ComputeLeaveSequenceHash() + protected virtual int ComputeLeaveSequenceHash() { lock (this) { ClearNonRelevantChildNodes(); - if (_activeChildNodes?.Any() == true) + if (ActiveChildNodes?.Any() == true) { - var firstChild = _activeChildNodes.First(); - return Fnv1aHash.Combine(firstChild.Method.MetadataToken, firstChild.LeaveSequenceHash); + var firstChild = ActiveChildNodes.First(); + return Fnv1aHash.Combine(Method.MetadataToken, firstChild.LeaveSequenceHash); } - return Fnv1aHash.FnvOffsetBias; + return Fnv1aHash.Combine(Method.MetadataToken, Fnv1aHash.FnvOffsetBias); } } @@ -260,14 +261,14 @@ private int ComputeLeaveSequenceHash() lock (_parent) { - _parent._activeChildNodes ??= new List(); - _parent._activeChildNodes.Add(this); + _parent.ActiveChildNodes ??= new List(); + _parent.ActiveChildNodes.Add(this); } lock (this) { // TODO For AggregateException, first/default might no be the most suitable way to tackle it. - var childCapturingCount = _activeChildNodes?.FirstOrDefault()?.NumOfChildren ?? 0; + var childCapturingCount = ActiveChildNodes?.FirstOrDefault()?.NumOfChildren ?? 0; NumOfChildren = childCapturingCount + 1; } @@ -289,23 +290,23 @@ public void Dispose() } _parent = null; - if (_activeChildNodes != null) + if (ActiveChildNodes != null) { - foreach (var node in _activeChildNodes) + foreach (var node in ActiveChildNodes) { node.Dispose(); } } - _activeChildNodes = null; + ActiveChildNodes = null; MarkAsUnwound(); _disposed = true; } - private void ClearNonRelevantChildNodes() + protected void ClearNonRelevantChildNodes() { // ReSharper disable once InconsistentlySynchronizedField - if (_activeChildNodes == null || _childNodesAlreadyCleansed) + if (ActiveChildNodes == null || _childNodesAlreadyCleansed) { return; } @@ -317,18 +318,18 @@ private void ClearNonRelevantChildNodes() return; } - if (!_activeChildNodes.Any()) + if (!ActiveChildNodes.Any()) { - _activeChildNodes = null; + ActiveChildNodes = null; return; } - for (var i = _activeChildNodes.Count - 1; i >= 0; i--) + for (var i = ActiveChildNodes.Count - 1; i >= 0; i--) { - var frame = _activeChildNodes[i]; + var frame = ActiveChildNodes[i]; if (frame.LeavingException == null || !HasChildException(frame.LeavingException)) { - _activeChildNodes.RemoveAt(i); + ActiveChildNodes.RemoveAt(i); frame.Dispose(); } } @@ -355,7 +356,7 @@ public bool HasChildException(Exception? exception) public override string ToString() { - return $"{nameof(TrackedStackFrameNode)}(Child Count = {_activeChildNodes?.Count})"; + return $"{nameof(TrackedStackFrameNode)}(Child Count = {ActiveChildNodes?.Count})"; } } } diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/MethodExtensions.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/MethodExtensions.cs index c76121e2860f..ca05a232f98d 100644 --- a/tracer/src/Datadog.Trace/Debugger/Helpers/MethodExtensions.cs +++ b/tracer/src/Datadog.Trace/Debugger/Helpers/MethodExtensions.cs @@ -19,6 +19,11 @@ namespace Datadog.Trace.Debugger.Helpers { internal static class MethodExtensions { + internal static string GetFullName(this MethodBase mb) + { + return mb.DeclaringType?.FullName + "." + mb.Name; + } + /// /// Gets fully qualified name of a method with parameters and generics. For example SkyApm.Sample.ConsoleApp.Program.Main(String[] args). /// Code was copied from System.Diagnostics.StackTrace.ToString() - .NET Standard implementation, not .NET Framework diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/ProbeStatusPollerMock.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/ProbeStatusPollerMock.cs index 4001e22d5733..4314341cfd1a 100644 --- a/tracer/src/Datadog.Trace/Debugger/Helpers/ProbeStatusPollerMock.cs +++ b/tracer/src/Datadog.Trace/Debugger/Helpers/ProbeStatusPollerMock.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Datadog.Trace.Debugger.ProbeStatuses; +#nullable enable namespace Datadog.Trace.Debugger.Helpers { internal class ProbeStatusPollerMock : IProbeStatusPoller diff --git a/tracer/src/Datadog.Tracer.Native/debugger_constants.h b/tracer/src/Datadog.Tracer.Native/debugger_constants.h index 849de952309d..57bc9886e650 100644 --- a/tracer/src/Datadog.Tracer.Native/debugger_constants.h +++ b/tracer/src/Datadog.Tracer.Native/debugger_constants.h @@ -48,6 +48,11 @@ inline WSTRING GetGenericErrorMessageWithErrorCode(short errorCode) return general_error_message + WStr(" [Error Code: ") + shared::ToWSTRING(errorCode) + WStr("]"); } +inline WSTRING GetGenericErrorMessageWithHr(HRESULT hr) +{ + return general_error_message + WStr(" [hr: ") + shared::ToWSTRING(Hex(hr)) + WStr("]"); +} + const WSTRING invalid_probe_method_already_instrumented = WStr("Dynamic Instrumentation failed to install the probe because the corresponding method is already instrumented by another product."); const WSTRING invalid_method_probe_probe_is_not_supported = @@ -69,7 +74,7 @@ const WSTRING non_supported_compiled_bytecode = const WSTRING type_contains_invalid_symbol = WStr("The type is not supported."); const WSTRING async_method_could_not_load_this = WStr("Instrumentation of async method in a generic class is not yet supported."); -const WSTRING invalid_probe_failed_to_instrument_method_probe = +const WSTRING invalid_probe_failed_to_instrument_method_probe = GetGenericErrorMessageWithErrorCode(1); const WSTRING invalid_probe_failed_to_instrument_line_probe = GetGenericErrorMessageWithErrorCode(2); diff --git a/tracer/src/Datadog.Tracer.Native/debugger_probes_instrumentation_requester.cpp b/tracer/src/Datadog.Tracer.Native/debugger_probes_instrumentation_requester.cpp index 770cb1f085f4..19b7e6e3f33d 100644 --- a/tracer/src/Datadog.Tracer.Native/debugger_probes_instrumentation_requester.cpp +++ b/tracer/src/Datadog.Tracer.Native/debugger_probes_instrumentation_requester.cpp @@ -1038,7 +1038,7 @@ HRESULT DebuggerProbesInstrumentationRequester::NotifyReJITError(ModuleID module { Logger::Info("Marking ", probeId, " as Error."); ProbesMetadataTracker::Instance()->SetErrorProbeStatus(probeId, - invalid_probe_failed_to_instrument_method_probe); + GetGenericErrorMessageWithHr(hrStatus)); } } diff --git a/tracer/test/Datadog.Trace.TestHelpers/MockTracerAgent.cs b/tracer/test/Datadog.Trace.TestHelpers/MockTracerAgent.cs index c6ae3f777de4..cbe051949bca 100644 --- a/tracer/test/Datadog.Trace.TestHelpers/MockTracerAgent.cs +++ b/tracer/test/Datadog.Trace.TestHelpers/MockTracerAgent.cs @@ -135,7 +135,7 @@ public static TcpUdpAgent Create(ITestOutputHelper output, int? port = null, int /// The list of spans. public IImmutableList WaitForSpans( int count, - int timeoutInMilliseconds = int.MaxValue, + int timeoutInMilliseconds = 20000, string operationName = null, DateTimeOffset? minDateTime = null, bool returnAllOperations = false, diff --git a/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Startup.cs b/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Startup.cs index 00caeb6a0dab..6014ad423bcf 100644 --- a/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Startup.cs +++ b/tracer/test/test-applications/debugger/Samples.Debugger.AspNetCore5/Startup.cs @@ -91,20 +91,16 @@ public async Task InvokeAsync(HttpContext context) { try { - // Code to run before the next middleware in the pipeline Console.WriteLine("FirstLastMiddleware: Entering request pipeline"); - // Call the next middleware in the pipeline await _next(context); - // Code to run after the next middleware in the pipeline Console.WriteLine("FirstLastMiddleware: Exiting request pipeline normally"); } catch (Exception ex) { - // Exception handling code Console.WriteLine($"FirstLastMiddleware caught an exception: {ex.Message}"); - throw; // Re-throw the exception to be handled by the global exception handler + throw; } } } diff --git a/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/RethrowTest.cs b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/RethrowTest.cs index 1a13d3885e0a..a3bc6202a9f0 100644 --- a/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/RethrowTest.cs +++ b/tracer/test/test-applications/debugger/dependency-libs/Samples.Probes.TestRuns/ExceptionReplay/RethrowTest.cs @@ -12,8 +12,12 @@ namespace Samples.Probes.TestRuns.ExceptionReplay [ExceptionReplayTestData(expectedNumberOfSnapshotsDefault: 4, expectedNumberOfSnaphotsFull: 4)] internal class RethrowTest : IAsyncRun { + private string _tempMethodName; + public async Task RunAsync() { + var methodName = nameof(RunAsync); + await Task.Yield(); try @@ -37,10 +41,13 @@ public async Task RunAsync() await Task.Yield(); } + _tempMethodName = methodName; } public async Task RunAsync2() { + var methodName = nameof(RunAsync2); + await Task.Yield(); try @@ -65,10 +72,13 @@ public async Task RunAsync2() await Task.Yield(); } + _tempMethodName = methodName; } public async Task InTheMiddle() { + var methodName = nameof(InTheMiddle); + int num = 3; try { @@ -104,10 +114,14 @@ public async Task InTheMiddle() { throw new NotImplementedException("Outer", ex); } + + _tempMethodName = methodName; } public async Task InTheMiddle2() { + var methodName = nameof(InTheMiddle2); + try { await Task.Yield(); @@ -118,6 +132,8 @@ public async Task InTheMiddle2() { throw new NotImplementedException("Outer", inner: e); } + + _tempMethodName = methodName; } public class RelationalDataReader : IAsyncDisposable, IDisposable @@ -171,12 +187,16 @@ protected virtual async ValueTask DisposeAsyncCore() private async ValueTask AwaitUsingFunc() { + var methodName = nameof(AwaitUsingFunc); await Task.Yield(); + _tempMethodName = methodName; return new RelationalDataReader(); } private async Task Foo() { + var methodName = nameof(Foo); + await Task.Yield(); try @@ -188,22 +208,76 @@ private async Task Foo() { await Task.Yield(); } + + _tempMethodName = methodName; } void CaptureAndThrow() { + var methodName = nameof(CaptureAndThrow); + try { - Bar(); + RecursiveThrow(); } catch (Exception e) { System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e).Throw(); } - } + _tempMethodName = methodName; + } + + void RecursiveThrow(int depth = 5) + { + var methodName = nameof(RecursiveThrow); + + try + { + if (depth > 0) + { + RecursiveThrow(depth - 1); + } + else + { + RecursiveCaptureAndThrow(); + } + } + catch (Exception e) + { + throw; + } + + _tempMethodName = methodName; + } + + void RecursiveCaptureAndThrow(int depth = 5) + { + var methodName = nameof(RecursiveCaptureAndThrow); + + try + { + if (depth > 0) + { + RecursiveCaptureAndThrow(depth - 1); + } + else + { + Bar(); + } + } + catch (Exception e) + { + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e).Throw(); + } + + _tempMethodName = methodName; + } + private void Bar() { + var methodName = nameof(Bar); + try { throw new NotImplementedException(); @@ -213,6 +287,8 @@ private void Bar() Console.WriteLine(nameof(RunAsync)); throw; } + + _tempMethodName = methodName; } } }