Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
217 changes: 217 additions & 0 deletions src/Build.UnitTests/Telemetry/Telemetry_Tests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
using Xunit;
using static Microsoft.Build.Framework.Telemetry.BuildInsights;
using static Microsoft.Build.Framework.Telemetry.TelemetryDataUtils;
using static Microsoft.Build.TelemetryInfra.TasksDetailsTelemetry;

Comment thread
baronfel marked this conversation as resolved.
Resolved
namespace Microsoft.Build.Engine.UnitTests
{
Expand Down Expand Up @@ -520,6 +521,222 @@ public void TelemetryCollector_AccumulatesAndSendsOnFinalize()
allTelemetryEvents[1].WorkerNodeTelemetryData.TargetsExecutionData.ShouldNotContainKey(key, "Old data should not appear after reset");
}

[Fact]
public void GetTasksDetailsProperties_ReturnsNullForNullData()
{
IWorkerNodeTelemetryData? data = null;
data.GetTasksDetailsProperties().ShouldBeNull();
}

[Fact]
public void GetTasksDetailsProperties_ReturnsNullForEmptyData()
{
var data = new WorkerNodeTelemetryData([], []);
data.GetTasksDetailsProperties().ShouldBeNull();
}

/// <summary>
/// Standard three-task fixture used by the <c>GetTasksDetailsProperties_*</c> tests:
/// Copy (10 executions), Csc (5 executions), and a custom task (3 executions).
/// </summary>
private static Dictionary<string, string> BuildThreeTaskFixtureProperties()
{
var tasksData = new Dictionary<TaskOrTargetTelemetryKey, TaskExecutionStats>
{
{ new TaskOrTargetTelemetryKey("Microsoft.Build.Tasks.Copy", false, false), new TaskExecutionStats(TimeSpan.FromMilliseconds(500), 10, 2048, "AssemblyTaskFactory", null) },
{ new TaskOrTargetTelemetryKey("Microsoft.Build.Tasks.Csc", false, false), new TaskExecutionStats(TimeSpan.FromMilliseconds(3000), 5, 4096, "AssemblyTaskFactory", null) },
{ new TaskOrTargetTelemetryKey("MyCustomTask", true, false), new TaskExecutionStats(TimeSpan.FromMilliseconds(100), 3, 512, "MyCompany.Factory", null) },
};
var data = new WorkerNodeTelemetryData(tasksData, []);
Dictionary<string, string>? properties = data.GetTasksDetailsProperties();
properties.ShouldNotBeNull();
return properties!;
}

[Fact]
public void GetTasksDetailsProperties_ReportsTaskCounts()
{
Dictionary<string, string> properties = BuildThreeTaskFixtureProperties();

properties["TaskCount"].ShouldBe("3");
properties["TotalTaskCount"].ShouldBe("3");
}

[Fact]
public void GetTasksDetailsProperties_OrdersTasksByExecutionsCountDescending()
{
Dictionary<string, string> properties = BuildThreeTaskFixtureProperties();

using var doc = System.Text.Json.JsonDocument.Parse(properties["Tasks"]);
var tasks = doc.RootElement;
tasks.ValueKind.ShouldBe(System.Text.Json.JsonValueKind.Array);
tasks.GetArrayLength().ShouldBe(3);

tasks[0].GetProperty("Name").GetString().ShouldBe("Microsoft.Build.Tasks.Copy");
tasks[0].GetProperty("ExecutionsCount").GetInt32().ShouldBe(10);

tasks[1].GetProperty("Name").GetString().ShouldBe("Microsoft.Build.Tasks.Csc");
tasks[1].GetProperty("ExecutionsCount").GetInt32().ShouldBe(5);

tasks[2].GetProperty("ExecutionsCount").GetInt32().ShouldBe(3);
}

[Fact]
public void GetTasksDetailsProperties_HashesCustomTaskAndFactoryNames()
{
Dictionary<string, string> properties = BuildThreeTaskFixtureProperties();

using var doc = System.Text.Json.JsonDocument.Parse(properties["Tasks"]);
// Custom task is sorted last (lowest ExecutionsCount).
var customTask = doc.RootElement[2];

customTask.GetProperty("IsCustom").GetBoolean().ShouldBeTrue();
customTask.GetProperty("Name").GetString().ShouldBe(GetHashed("MyCustomTask"));
customTask.GetProperty("FactoryName").GetString().ShouldBe(GetHashed("MyCompany.Factory"));
}

[Fact]
public void GetTasksDetailsProperties_BoundsToTop100()
{
var tasksData = new Dictionary<TaskOrTargetTelemetryKey, TaskExecutionStats>();
for (int i = 0; i < 150; i++)
{
tasksData[new TaskOrTargetTelemetryKey($"Task{i}", false, false)] =
new TaskExecutionStats(TimeSpan.FromMilliseconds(i), i + 1, 0, "AssemblyTaskFactory", null);
}

var data = new WorkerNodeTelemetryData(tasksData, []);
Dictionary<string, string>? properties = data.GetTasksDetailsProperties();

properties.ShouldNotBeNull();
properties!["TaskCount"].ShouldBe("100");
properties["TotalTaskCount"].ShouldBe("150");

// Parse the JSON and verify array length
using var doc = System.Text.Json.JsonDocument.Parse(properties["Tasks"]);
doc.RootElement.GetArrayLength().ShouldBe(100);

// The top task by execution count should be Task149 (count=150)
var first = doc.RootElement[0];
first.GetProperty("Name").GetString().ShouldBe("Task149");
first.GetProperty("ExecutionsCount").GetInt32().ShouldBe(150);
}

[Fact]
public void GetTasksDetailsProperties_IncludesTaskHostRuntimeWhenNonNull()
{
var tasksData = new Dictionary<TaskOrTargetTelemetryKey, TaskExecutionStats>
{
{ new TaskOrTargetTelemetryKey("Microsoft.Build.Tasks.Copy", false, false), new TaskExecutionStats(TimeSpan.FromMilliseconds(100), 5, 1024, "AssemblyTaskFactory", "CLR4") },
};
var data = new WorkerNodeTelemetryData(tasksData, []);

Dictionary<string, string>? properties = data.GetTasksDetailsProperties();

properties.ShouldNotBeNull();
using var doc = System.Text.Json.JsonDocument.Parse(properties!["Tasks"]);
var task = doc.RootElement[0];
task.GetProperty("TaskHostRuntime").GetString().ShouldBe("CLR4");
}

[Fact]
public void GetTasksDetailsProperties_ProducesValidJson()
{
var tasksData = new Dictionary<TaskOrTargetTelemetryKey, TaskExecutionStats>
{
{ new TaskOrTargetTelemetryKey("Microsoft.Build.Tasks.Copy", false, false), new TaskExecutionStats(TimeSpan.FromMilliseconds(500), 10, 2048, "AssemblyTaskFactory", null) },
{ new TaskOrTargetTelemetryKey("MyCustomTask", true, true), new TaskExecutionStats(TimeSpan.FromMilliseconds(100), 3, 512, null, null) },
};
var data = new WorkerNodeTelemetryData(tasksData, []);

Dictionary<string, string>? properties = data.GetTasksDetailsProperties();

properties.ShouldNotBeNull();
string json = properties!["Tasks"];

// Should not throw - valid JSON
using var doc = System.Text.Json.JsonDocument.Parse(json);
doc.RootElement.ValueKind.ShouldBe(System.Text.Json.JsonValueKind.Array);

// Verify all expected properties are present
var first = doc.RootElement[0];
first.GetProperty("Name").ValueKind.ShouldBe(System.Text.Json.JsonValueKind.String);
first.GetProperty("ExecutionsCount").ValueKind.ShouldBe(System.Text.Json.JsonValueKind.Number);
first.GetProperty("TotalMilliseconds").ValueKind.ShouldBe(System.Text.Json.JsonValueKind.Number);
first.GetProperty("TotalMemoryBytes").ValueKind.ShouldBe(System.Text.Json.JsonValueKind.Number);
first.GetProperty("IsCustom").ValueKind.ShouldBe(System.Text.Json.JsonValueKind.False);
first.GetProperty("IsNuget").ValueKind.ShouldBe(System.Text.Json.JsonValueKind.False);
}

[Fact]
public void GetTasksDetailsProperties_OmitsNullFactoryNameAndTaskHostRuntime()
{
// Both FactoryName and TaskHostRuntime are null - they should be omitted from JSON,
// not emitted as null. This locks in JsonIgnoreCondition.WhenWritingNull behavior.
var tasksData = new Dictionary<TaskOrTargetTelemetryKey, TaskExecutionStats>
{
{ new TaskOrTargetTelemetryKey("Microsoft.Build.Tasks.Copy", false, false), new TaskExecutionStats(TimeSpan.FromMilliseconds(500), 10, 2048, null, null) },
};
var data = new WorkerNodeTelemetryData(tasksData, []);

Dictionary<string, string>? properties = data.GetTasksDetailsProperties();

properties.ShouldNotBeNull();
using var doc = System.Text.Json.JsonDocument.Parse(properties!["Tasks"]);
var task = doc.RootElement[0];

task.TryGetProperty("FactoryName", out _).ShouldBeFalse();
task.TryGetProperty("TaskHostRuntime", out _).ShouldBeFalse();
}

[Fact]
public void GetTasksDetailsProperties_EscapesSpecialCharactersInFactoryName()
{
// Custom factory names are hashed (so they don't contain special chars in practice),
// but a built-in factory name with embedded quotes or backslashes must still round-trip
// through JSON without producing invalid output.
var tasksData = new Dictionary<TaskOrTargetTelemetryKey, TaskExecutionStats>
{
{ new TaskOrTargetTelemetryKey("Microsoft.Build.Tasks.Copy", false, false), new TaskExecutionStats(TimeSpan.FromMilliseconds(1), 1, 0, "Factory\"with\\special\nchars", null) },
};
var data = new WorkerNodeTelemetryData(tasksData, []);

Dictionary<string, string>? properties = data.GetTasksDetailsProperties();

properties.ShouldNotBeNull();

// The hashed factory-name pipeline only hashes non-known factories, so this value gets hashed.
// What we care about here: the resulting JSON must parse cleanly regardless of input.
using var doc = System.Text.Json.JsonDocument.Parse(properties!["Tasks"]);
doc.RootElement.GetArrayLength().ShouldBe(1);
}

[Fact]
public void GetTasksDetailsProperties_EmitsNumericAndBooleanFieldsAsJsonPrimitives()
{
// Guards against regressions where numbers/bools could be accidentally serialized as strings
// (e.g. via a custom JsonConverter). The SDK consumer parses these as numbers in Kusto.
var tasksData = new Dictionary<TaskOrTargetTelemetryKey, TaskExecutionStats>
{
// Use ticks (1 ms = 10_000 ticks) so the fractional millisecond value round-trips on both
// net472 (where TimeSpan.FromMilliseconds(double) rounds to whole ms) and net10.
{ new TaskOrTargetTelemetryKey("Microsoft.Build.Tasks.Copy", false, true), new TaskExecutionStats(TimeSpan.FromTicks(1234000), 7, 8192, "AssemblyTaskFactory", null) },
};
var data = new WorkerNodeTelemetryData(tasksData, []);

Dictionary<string, string>? properties = data.GetTasksDetailsProperties();

properties.ShouldNotBeNull();
using var doc = System.Text.Json.JsonDocument.Parse(properties!["Tasks"]);
var task = doc.RootElement[0];

task.GetProperty("ExecutionsCount").GetInt32().ShouldBe(7);
task.GetProperty("TotalMemoryBytes").GetInt64().ShouldBe(8192);
task.GetProperty("TotalMilliseconds").GetDouble().ShouldBe(123.4);
task.GetProperty("IsCustom").GetBoolean().ShouldBeFalse();
task.GetProperty("IsNuget").GetBoolean().ShouldBeTrue();
}

/// <summary>
/// <see cref="MockLoggingService"/> that records all <see cref="ILoggingService.LogBuildEvent"/> calls
/// so tests can inspect emitted build events.
Expand Down
12 changes: 12 additions & 0 deletions src/Build/BackEnd/BuildManager/BuildManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1229,6 +1229,18 @@ public void EndBuild()

loggingService.LogTelemetry(buildEventContext: null, _buildTelemetry.EventName, _buildTelemetry.GetProperties());

// Emit per-task execution details as a separate "build/tasks/details" event.
// The SDK merges these into the aggregated build/tasks telemetry event,
// providing parity with the Activity-based path used by VS telemetry.
if (!Traits.Instance.ExcludeTasksDetailsFromTelemetry)
{
Dictionary<string, string>? tasksDetailsProperties = _telemetryConsumingLogger?.WorkerNodeTelemetryData.GetTasksDetailsProperties();
Comment thread
baronfel marked this conversation as resolved.
Resolved
if (tasksDetailsProperties is not null)
{
loggingService.LogTelemetry(buildEventContext: null, TasksDetailsTelemetry.TasksDetailsEventName, tasksDetailsProperties);
}
}

EndBuildTelemetry();

// Clean telemetry to make it ready for next build submission.
Expand Down
1 change: 1 addition & 0 deletions src/Build/Microsoft.Build.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@
<Compile Include="Logging\ReusableLogger.cs" />
<Compile Include="TelemetryInfra\InternalTelemetryConsumingLogger.cs" />
<Compile Include="TelemetryInfra\ITelemetryCollector.cs" />
<Compile Include="TelemetryInfra\TasksDetailsTelemetry.cs" />
<Compile Include="TelemetryInfra\TelemetryCollectorProvider.cs" />
<Compile Include="Utilities\AwaitExtensions.cs" />
<Compile Include="Utilities\EncodingStringWriter.cs" />
Expand Down
60 changes: 60 additions & 0 deletions src/Build/TelemetryInfra/TasksDetailsTelemetry.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Build.Framework.Telemetry;
using static Microsoft.Build.Framework.Telemetry.TelemetryDataUtils;

namespace Microsoft.Build.TelemetryInfra;

/// <summary>
/// Serializes per-task execution details into a TelemetryEventArgs-compatible
/// (<see cref="System.Collections.Generic.IDictionary{TKey, TValue}"/> of strings) shape for the CLI/SDK telemetry path.
///
/// Equivalent data is published to VS via <see cref="TelemetryDataUtils.AsActivityDataHolder"/>, where the
/// VS telemetry sink accepts arbitrary objects and serializes them itself. The CLI path requires strings
/// (see <see cref="Framework.TelemetryEventArgs"/>), so we serialize a JSON array here.
Comment thread
baronfel marked this conversation as resolved.
Resolved
/// </summary>
internal static class TasksDetailsTelemetry
{
/// <summary>
/// Event name for per-task execution details emitted via TelemetryEventArgs.
/// Uses a separate event name so per-task details are distinguishable from
/// the per-project aggregated "build/tasks" telemetry event.
/// </summary>
internal const string TasksDetailsEventName = "build/tasks/details";

/// <summary>
/// Maximum number of individual task details to include in TelemetryEventArgs-based telemetry.
/// </summary>
private const int MaxTaskDetailsForTelemetryEvent = 100;

internal static Dictionary<string, string>? GetTasksDetailsProperties(this IWorkerNodeTelemetryData? telemetryData)
{
if (telemetryData is null || telemetryData.TasksExecutionData.Count == 0)
{
return null;
}

List<TaskDetailInfo> allTasks = GetTasksDetails(telemetryData.TasksExecutionData);
List<TaskDetailInfo> topTasks = allTasks
.OrderByDescending(t => t.ExecutionsCount)
.Take(MaxTaskDetailsForTelemetryEvent)
.ToList();
Comment thread
baronfel marked this conversation as resolved.
Resolved

return new Dictionary<string, string>(3)
{
["TaskCount"] = topTasks.Count.ToString(CultureInfo.InvariantCulture),
["TotalTaskCount"] = allTasks.Count.ToString(CultureInfo.InvariantCulture),
["Tasks"] = JsonSerializer.Serialize(topTasks, TasksDetailsJsonContext.Default.ListTaskDetailInfo),
};
}
}

[JsonSourceGenerationOptions(DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
[JsonSerializable(typeof(List<TaskDetailInfo>))]
internal partial class TasksDetailsJsonContext : JsonSerializerContext;
2 changes: 1 addition & 1 deletion src/Framework/Telemetry/TelemetryDataUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ internal record TargetDetailInfo(string Name, bool WasExecuted, bool IsCustom, b
/// <summary>
/// Converts tasks details to a list of custom objects for telemetry.
/// </summary>
private static List<TaskDetailInfo> GetTasksDetails(
internal static List<TaskDetailInfo> GetTasksDetails(
Dictionary<TaskOrTargetTelemetryKey, TaskExecutionStats> tasksDetails)
{
var result = new List<TaskDetailInfo>();
Expand Down