Repository navigation
Reliable CLI telemetry via a persist-then-drain OpenTelemetry exporter #55211
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
+3,922
−10
Merged
Changes from all commits
Commits
Show all changes
11 commits
Select commit
Hold shift + click to select a range
723ae8c
Make CLI telemetry delivery reliable via persist-then-drain pipeline
baronfel 604abf4
Merge origin/main into baronfel-otel-azure-monitor-cli-shutdown
Copilot 5c84ea4
Address review feedback: bounded CI shutdown, cancellation flow, tests
baronfel c7591a9
Add opt-in real-endpoint telemetry E2E harness
baronfel 9b76b95
Make E2E harness run id pinnable + emit ingestion-verification query
baronfel ba44f94
Make E2E run token searchable by Message/EventName, not raw AI tables
baronfel 569ab6c
Reference Cli.Telemetry from dotnet-aot.Tests to fix AoT/FullFramewor…
baronfel 6f1f6b9
Create telemetry storage directory lazily so opt-out leaves no ~/.dot…
baronfel 2e762e3
Emit real dotnet/cli event shape + stamp run id as SessionId in E2E h…
baronfel fed4f53
Add differential test proving Breeze serializer matches Azure Monitor…
baronfel f986f53
Dont allow OTel to steal blobs of telemetry away and then never let u…
nagilson File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
74 changes: 74 additions & 0 deletions
74
src/Cli/Microsoft.DotNet.Cli.Telemetry/Implementation/AzureMonitorConnectionString.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| namespace Microsoft.DotNet.Cli.Telemetry.Implementation; | ||
|
|
||
| /// <summary> | ||
| /// A minimal parser for the Application Insights connection string. We only need the | ||
| /// instrumentation key (stamped into every telemetry envelope's <c>iKey</c>) and the | ||
| /// ingestion endpoint (the base of the Breeze <c>/v2.1/track</c> upload URL). | ||
| /// </summary> | ||
| internal sealed record AzureMonitorConnectionString(string InstrumentationKey, Uri IngestionEndpoint) | ||
| { | ||
| private const string InstrumentationKeyName = "InstrumentationKey"; | ||
| private const string IngestionEndpointName = "IngestionEndpoint"; | ||
| private const string DefaultIngestionEndpoint = "https://dc.services.visualstudio.com/"; | ||
|
|
||
| /// <summary> | ||
| /// The full URL telemetry is POSTed to, e.g. | ||
| /// <c>https://<region>.in.applicationinsights.azure.com/v2.1/track</c>. | ||
| /// </summary> | ||
| public Uri TrackUri { get; } = new Uri(IngestionEndpoint, "v2.1/track"); | ||
|
|
||
| /// <summary> | ||
| /// Parses <paramref name="connectionString"/> into its instrumentation key and ingestion | ||
| /// endpoint. Returns <see langword="null"/> when the string is missing an instrumentation | ||
| /// key, which is the only field without a safe default. | ||
| /// </summary> | ||
| public static AzureMonitorConnectionString? Parse(string? connectionString) | ||
| { | ||
| if (string.IsNullOrWhiteSpace(connectionString)) | ||
| { | ||
| return null; | ||
| } | ||
|
|
||
| string? instrumentationKey = null; | ||
| var ingestionEndpoint = DefaultIngestionEndpoint; | ||
|
|
||
| foreach (var part in connectionString.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) | ||
| { | ||
| var separatorIndex = part.IndexOf('='); | ||
| if (separatorIndex <= 0) | ||
| { | ||
| continue; | ||
| } | ||
|
|
||
| var key = part.Substring(0, separatorIndex).Trim(); | ||
| var value = part.Substring(separatorIndex + 1).Trim(); | ||
|
|
||
| if (key.Equals(InstrumentationKeyName, StringComparison.OrdinalIgnoreCase)) | ||
| { | ||
| instrumentationKey = value; | ||
| } | ||
| else if (key.Equals(IngestionEndpointName, StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(value)) | ||
| { | ||
| ingestionEndpoint = value; | ||
| } | ||
| } | ||
|
|
||
| if (string.IsNullOrWhiteSpace(instrumentationKey)) | ||
| { | ||
| return null; | ||
| } | ||
|
|
||
| if (!Uri.TryCreate(EnsureTrailingSlash(ingestionEndpoint), UriKind.Absolute, out var endpointUri)) | ||
| { | ||
| return null; | ||
| } | ||
|
|
||
| return new AzureMonitorConnectionString(instrumentationKey, endpointUri); | ||
| } | ||
|
|
||
| private static string EnsureTrailingSlash(string value) | ||
| => value.EndsWith('/') ? value : value + "/"; | ||
| } |
217 changes: 217 additions & 0 deletions
217
src/Cli/Microsoft.DotNet.Cli.Telemetry/Implementation/AzureMonitorLogSerializer.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,217 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System.Globalization; | ||
| using Microsoft.Extensions.Logging; | ||
| using OpenTelemetry; | ||
| using OpenTelemetry.Logs; | ||
|
|
||
| namespace Microsoft.DotNet.Cli.Telemetry.Implementation; | ||
|
|
||
| /// <summary> | ||
| /// Serializes a batch of OpenTelemetry <see cref="LogRecord"/> instances into the Application | ||
| /// Insights "Breeze" wire format (newline-delimited JSON of telemetry envelopes), exactly as | ||
| /// <c>Azure.Monitor.OpenTelemetry.Exporter</c> would produce via its internal | ||
| /// <c>LogsHelper.OtelToAzureMonitorLogs</c>: | ||
| /// <list type="bullet"> | ||
| /// <item><description>A log record with an <see cref="LogRecord.Exception"/> becomes an | ||
| /// <c>ExceptionData</c> envelope.</description></item> | ||
| /// <item><description>Any other log record becomes a <c>MessageData</c> envelope.</description></item> | ||
| /// </list> | ||
| /// Both carry the mapped Application Insights <c>severityLevel</c>, the log attributes / scopes | ||
| /// as properties (plus <c>CategoryName</c>, <c>EventId</c>, and <c>EventName</c>), and the | ||
| /// operation id / parent id from the record's trace context. Feeding the Application Insights | ||
| /// <c>traces</c> table this way is what the Azure Monitor log exporter does, so existing | ||
| /// log-based dashboards keep working. | ||
| /// | ||
| /// <para>This is a focused reimplementation covering the shapes a CLI actually emits; the more | ||
| /// advanced Azure-specific log conventions (custom events, availability results, and the | ||
| /// <c>microsoft.*</c> context attributes) are not special-cased and simply flow through as | ||
| /// ordinary message properties.</para> | ||
| /// </summary> | ||
| internal static class AzureMonitorLogSerializer | ||
| { | ||
| /// <summary> | ||
| /// Serializes <paramref name="batch"/> to NDJSON bytes. Returns <see langword="null"/> | ||
| /// when the batch produces no telemetry. | ||
| /// </summary> | ||
| public static byte[]? SerializeBatch(in Batch<LogRecord> batch, TelemetryResourceContext resource, string instrumentationKey) | ||
| { | ||
| using var stream = new MemoryStream(); | ||
| var wroteAny = false; | ||
|
|
||
| foreach (var logRecord in batch) | ||
| { | ||
| WriteLogRecord(stream, logRecord, resource, instrumentationKey); | ||
| wroteAny = true; | ||
| } | ||
|
|
||
| return wroteAny ? stream.ToArray() : null; | ||
| } | ||
|
|
||
| private static readonly Action<LogRecordScope, (Dictionary<string, string> Properties, HashSet<string> Seen)> AddScopeProperties = | ||
| static (scope, state) => | ||
| { | ||
| foreach (var scopeItem in scope) | ||
| { | ||
| if (string.IsNullOrEmpty(scopeItem.Key) || scopeItem.Key == BreezeSchema.OriginalFormatKey) | ||
| { | ||
| continue; | ||
| } | ||
|
|
||
| if (BreezeWriter.TryFormatProperty(scopeItem, state.Seen, out var key, out var value)) | ||
| { | ||
| state.Properties[key] = value; | ||
| } | ||
| } | ||
| }; | ||
|
|
||
| private static void WriteLogRecord(Stream stream, LogRecord logRecord, TelemetryResourceContext resource, string instrumentationKey) | ||
| { | ||
| var seenKeys = new HashSet<string>(StringComparer.Ordinal); | ||
| var properties = new Dictionary<string, string>(StringComparer.Ordinal); | ||
|
|
||
| // Mirrors LogsHelper: the exception message wins, then the formatted message, then the | ||
| // raw "{OriginalFormat}" template if nothing else produced a message. | ||
| var message = logRecord.Exception?.Message ?? logRecord.FormattedMessage; | ||
|
|
||
| foreach (var attribute in logRecord.Attributes ?? []) | ||
| { | ||
| if (attribute.Key == BreezeSchema.OriginalFormatKey) | ||
| { | ||
| if (logRecord.Exception?.Message is not null) | ||
| { | ||
| // Keep the template as a property when the exception supplied the message. | ||
| if (attribute.Value is not null && seenKeys.Add(BreezeSchema.OriginalFormatProperty)) | ||
| { | ||
| properties[BreezeSchema.OriginalFormatProperty] = | ||
| BreezeSchema.Truncate(attribute.Value.ToString(), BreezeSchema.KvpMaxValueLength) ?? "null"; | ||
| } | ||
| } | ||
| else if (message is null) | ||
| { | ||
| message = attribute.Value?.ToString(); | ||
| } | ||
|
|
||
| continue; | ||
| } | ||
|
|
||
| if (BreezeWriter.TryFormatProperty(attribute, seenKeys, out var key, out var value)) | ||
| { | ||
| properties[key] = value; | ||
| } | ||
| } | ||
|
|
||
| logRecord.ForEachScope(AddScopeProperties, (properties, seenKeys)); | ||
|
|
||
| if (!string.IsNullOrEmpty(logRecord.CategoryName) && seenKeys.Add(BreezeSchema.CategoryNameProperty)) | ||
| { | ||
| properties[BreezeSchema.CategoryNameProperty] = | ||
| BreezeSchema.Truncate(logRecord.CategoryName, BreezeSchema.KvpMaxValueLength)!; | ||
| } | ||
|
|
||
| if (logRecord.EventId.Id != 0 && seenKeys.Add(BreezeSchema.EventIdProperty)) | ||
| { | ||
| properties[BreezeSchema.EventIdProperty] = logRecord.EventId.Id.ToString(CultureInfo.InvariantCulture); | ||
| } | ||
|
|
||
| if (!string.IsNullOrEmpty(logRecord.EventId.Name) && seenKeys.Add(BreezeSchema.EventNameProperty)) | ||
| { | ||
| properties[BreezeSchema.EventNameProperty] = | ||
| BreezeSchema.Truncate(logRecord.EventId.Name, BreezeSchema.KvpMaxValueLength)!; | ||
| } | ||
|
|
||
| var tags = BuildTags(logRecord, resource); | ||
| var severity = GetSeverityLevel(logRecord.LogLevel); | ||
| var propertyBag = properties.Count > 0 ? properties : null; | ||
| var time = BreezeWriter.FormatTime(logRecord.Timestamp); | ||
|
|
||
| if (logRecord.Exception is not null) | ||
| { | ||
| var envelope = new TelemetryEnvelope<ExceptionData> | ||
| { | ||
| Name = BreezeSchema.ExceptionEnvelopeName, | ||
| Time = time, | ||
| InstrumentationKey = instrumentationKey, | ||
| Tags = tags, | ||
| Data = new TelemetryData<ExceptionData> | ||
| { | ||
| BaseType = BreezeSchema.ExceptionDataType, | ||
| BaseData = BuildExceptionData(logRecord.Exception, severity, propertyBag), | ||
| }, | ||
| }; | ||
| BreezeWriter.WriteEnvelope(stream, envelope, TelemetryJsonContext.Default.ExceptionEnvelope); | ||
| } | ||
| else | ||
| { | ||
| var envelope = new TelemetryEnvelope<MessageData> | ||
| { | ||
| Name = BreezeSchema.MessageEnvelopeName, | ||
| Time = time, | ||
| InstrumentationKey = instrumentationKey, | ||
| Tags = tags, | ||
| Data = new TelemetryData<MessageData> | ||
| { | ||
| BaseType = BreezeSchema.MessageDataType, | ||
| BaseData = new MessageData | ||
| { | ||
| Message = BreezeSchema.Truncate(message, BreezeSchema.MessageMaxLength), | ||
| SeverityLevel = severity, | ||
| Properties = propertyBag, | ||
| }, | ||
| }, | ||
| }; | ||
| BreezeWriter.WriteEnvelope(stream, envelope, TelemetryJsonContext.Default.MessageEnvelope); | ||
| } | ||
| } | ||
|
|
||
| private static ExceptionData BuildExceptionData(Exception exception, string severity, Dictionary<string, string>? properties) | ||
| { | ||
| var stack = exception.StackTrace; | ||
| var hasFullStack = stack is not null && stack.Length <= BreezeSchema.ExceptionStackMaxLength; | ||
|
|
||
| return new ExceptionData | ||
| { | ||
| SeverityLevel = severity, | ||
| Exceptions = | ||
| [ | ||
| new ExceptionDetails | ||
| { | ||
| TypeName = BreezeSchema.Truncate(exception.GetType().FullName, BreezeSchema.ExceptionTypeNameMaxLength), | ||
| Message = BreezeSchema.Truncate(exception.Message, BreezeSchema.ExceptionMessageMaxLength), | ||
| HasFullStack = hasFullStack, | ||
| Stack = BreezeSchema.Truncate(stack, BreezeSchema.ExceptionStackMaxLength), | ||
| }, | ||
| ], | ||
| Properties = properties ?? [], | ||
| }; | ||
| } | ||
|
|
||
| private static Dictionary<string, string> BuildTags(LogRecord logRecord, TelemetryResourceContext resource) | ||
| { | ||
| var tags = new Dictionary<string, string>(StringComparer.Ordinal); | ||
|
|
||
| // The OTel log record carries the ambient Activity's trace context so Azure Monitor can | ||
| // correlate the log row with the span it was emitted under. | ||
| if (logRecord.TraceId != default) | ||
| { | ||
| tags[BreezeSchema.OperationId] = logRecord.TraceId.ToHexString(); | ||
| } | ||
| if (logRecord.SpanId != default) | ||
| { | ||
| tags[BreezeSchema.OperationParentId] = logRecord.SpanId.ToHexString(); | ||
| } | ||
|
|
||
| BreezeWriter.AddResourceTags(tags, resource); | ||
| return tags; | ||
| } | ||
|
|
||
| private static string GetSeverityLevel(LogLevel logLevel) => logLevel switch | ||
| { | ||
| LogLevel.Critical => BreezeSchema.SeverityCritical, | ||
| LogLevel.Error => BreezeSchema.SeverityError, | ||
| LogLevel.Warning => BreezeSchema.SeverityWarning, | ||
| LogLevel.Information => BreezeSchema.SeverityInformation, | ||
| _ => BreezeSchema.SeverityVerbose, | ||
| }; | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.