diff --git a/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs b/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs index 43a151d77908..7b2c5ec0bdf6 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs @@ -886,10 +886,55 @@ private static void AppendAssemblySummary(TestProgressState assemblyRun, ITermin terminal.Append(' '); AppendAssemblyResult(terminal, assemblyRun); terminal.Append(' '); + AppendAssemblyTestCounts(terminal, assemblyRun); + terminal.Append(' '); AppendLongDuration(terminal, assemblyRun.Stopwatch.Elapsed); terminal.AppendLine(); } + /// + /// Renders a compact, per-assembly counts block that mirrors the in-progress indicator. + /// Full-ANSI terminals get "[✓P/xF/↓S]" (with optional "/rR"); simple terminals + /// (NoAnsi / SimpleAnsi / CI) get the ASCII "[+P/xF/?S]" form so logs stay font- and + /// encoding-friendly. Glyphs and colors are intentionally kept in sync with the rendering in + /// and . + /// + private static void AppendAssemblyTestCounts(ITerminal terminal, TestProgressState assemblyRun) + { + int failed = assemblyRun.FailedTests; + int passed = assemblyRun.PassedTests; + int skipped = assemblyRun.SkippedTests; + int retried = assemblyRun.RetriedFailedTests; + + bool unicode = terminal is AnsiTerminal; + char passedGlyph = unicode ? '✓' : '+'; + char skippedGlyph = unicode ? '↓' : '?'; + + terminal.Append('['); + + AppendGlyphCount(terminal, passedGlyph, passed, TerminalColor.DarkGreen); + terminal.Append('/'); + AppendGlyphCount(terminal, 'x', failed, TerminalColor.DarkRed); + terminal.Append('/'); + AppendGlyphCount(terminal, skippedGlyph, skipped, TerminalColor.DarkYellow); + + if (retried > 0) + { + terminal.Append('/'); + AppendGlyphCount(terminal, 'r', retried, TerminalColor.Gray); + } + + terminal.Append(']'); + } + + private static void AppendGlyphCount(ITerminal terminal, char glyph, int count, TerminalColor color) + { + terminal.SetColor(color); + terminal.Append(glyph); + terminal.Append(count.ToString(CultureInfo.CurrentCulture)); + terminal.ResetColor(); + } + /// /// Appends a long duration in human readable format such as 1h 23m 500ms. /// diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs index 8021e8155c58..d88db270fcdc 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs @@ -1,8 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.RegularExpressions; using Microsoft.DotNet.Cli.Commands; using Microsoft.DotNet.Cli.Commands.Test; +using Microsoft.DotNet.Cli.Utils; using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; using ExitCodes = Microsoft.NET.TestFramework.ExitCode; @@ -269,6 +271,16 @@ public void RunMultipleTestProjectsWithFailingTests_ShouldReturnExitCodeAtLeastO .And.Contain("succeeded: 2") .And.Contain("failed: 1") .And.Contain("skipped: 2"); + + // Issue #52128: per-assembly summary lines must show their own counts in the + // compact bracketed form that mirrors the in-progress indicator. The subprocess + // stdout is redirected, so the SDK renders the ASCII glyph form "[+P/xF/?S]". + Assert.Matches( + GeneratePerAssemblyCountsRegexPattern("TestProject", TestingConstants.Failed, configuration, passed: 1, failed: 1, skipped: 1), + result.StdOut); + Assert.Matches( + GeneratePerAssemblyCountsRegexPattern("OtherTestProject", TestingConstants.Passed, configuration, passed: 1, failed: 0, skipped: 1), + result.StdOut); } result.ExitCode.Should().Be(ExitCodes.AtLeastOneTestFailed); @@ -293,6 +305,20 @@ public void RunMultipleTestProjectsWithDifferentFailures_ShouldReturnExitCodeGen Assert.Matches(RegexPatternHelper.GenerateProjectRegexPattern("OtherTestProject", TestingConstants.Failed, true, configuration, "2"), result.StdOut); Assert.Matches(RegexPatternHelper.GenerateProjectRegexPattern("AnotherTestProject", TestingConstants.Passed, true, configuration), result.StdOut); + // Issue #52128: per-assembly summary lines must show their own counts in the + // compact bracketed form even when no tests ran in the assembly ("Zero tests ran") + // and when an assembly failed. The subprocess stdout is redirected, so the SDK + // renders the ASCII glyph form "[+P/xF/?S]". + Assert.Matches( + GeneratePerAssemblyCountsRegexPattern("TestProject", TestingConstants.ZeroTestsRan, configuration, passed: 0, failed: 0, skipped: 0), + result.StdOut); + Assert.Matches( + GeneratePerAssemblyCountsRegexPattern("OtherTestProject", TestingConstants.Failed, configuration, passed: 1, failed: 1, skipped: 1), + result.StdOut); + Assert.Matches( + GeneratePerAssemblyCountsRegexPattern("AnotherTestProject", TestingConstants.Passed, configuration, passed: 1, failed: 0, skipped: 0), + result.StdOut); + result.StdOut .Should().Contain("Test run summary: Failed!") .And.Contain("total: 4") @@ -612,5 +638,33 @@ public void DotnetTest_MTPChildProcessHangTestProject_ShouldNotHang(string confi result.ExitCode.Should().Be(ExitCodes.ZeroTests); } + + // Issue #52128: builds a regex that matches the per-assembly summary line — the line that + // names the test assembly with its TFM/architecture and the status, followed by the + // compact counts block (matching the in-progress indicator) that we render between the + // status and the duration. Acceptance tests run "dotnet test" in a subprocess whose stdout + // is redirected, so the SDK picks NonAnsiTerminal (extends SimpleTerminal) and emits the + // ASCII glyph form "[+P/xF/?S]" (full-ANSI terminals emit "[✓P/xF/↓S]"). + // Example shapes we match: + // ".../Debug/net11.0/TestProject.dll (net11.0|x64) passed [+1/x0/?1] (1.2s)" + // ".../Debug/net11.0/TestProject.dll (net11.0|x64) failed with 1 error(s) [+1/x1/?1] (1.5s)" + private static string GeneratePerAssemblyCountsRegexPattern( + string projectName, + string status, + string configuration, + int passed, + int failed, + int skipped) + { + string version = ToolsetInfo.CurrentTargetFramework; + string escapedVersion = Regex.Escape(version); + string escapedProject = Regex.Escape(projectName); + // PathUtility.GetDirectorySeparatorChar() already returns a regex-escaped separator. + string separator = PathUtility.GetDirectorySeparatorChar(); + // After the status name we may have an optional "with N error(s)" suffix (rendered when + // tests failed AND exitCode != 0), so we allow any non-bracket, non-newline characters + // between the status and the leading "[" that introduces the compact counts block. + return $@".+{configuration}{separator}{escapedVersion}{separator}{escapedProject}(\.dll|\.exe)?\s+\({escapedVersion}\|[A-Za-z0-9]+\)\s{status}[^\[\r\n]*\[\+{passed}/x{failed}/\?{skipped}\]\s+\("; + } } } diff --git a/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs b/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs index 0a1098e55367..759e7f9b9814 100644 --- a/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.RegularExpressions; using Microsoft.DotNet.Cli.Commands.Test.Terminal; using Moq; @@ -46,4 +47,187 @@ public void AssemblyRunCompleted_WhenExecutionIdUnknown_DoesNotThrowAndReportsHa act.Should().NotThrow(); reporter.HasHandshakeFailure.Should().BeTrue(); } + + /// + /// Regression test for https://github.com/dotnet/sdk/issues/52128: the mid-stream per-assembly + /// summary printed when an assembly completes (ShowAssembly + ShowAssemblyStartAndComplete) + /// must include the per-assembly counts in the same compact bracketed form used by the + /// in-progress indicator. Tests use which routes through + /// SimpleTerminal, so the expected glyphs are the ASCII variants + /// [+P/xF/?S] (mirroring SimpleTerminal.RenderProgress). The full-ANSI path + /// uses [✓P/xF/↓S] and is exercised end-to-end by acceptance tests. + /// + [Fact] + public void AssemblyRunCompleted_WithShowAssemblyStartAndComplete_PrintsPerAssemblyCounts() + { + var capturingConsole = new CapturingConsole(); + + var options = new TerminalTestReporterOptions + { + AnsiMode = AnsiMode.SimpleAnsi, + ShowProgress = false, + ShowAssembly = true, + ShowAssemblyStartAndComplete = true, + }; + + using var reporter = new TerminalTestReporter(capturingConsole, options); + + reporter.TestExecutionStarted(DateTimeOffset.UtcNow, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false); + + const string assembly = "/repo/bin/Debug/net9.0/MyTests.dll"; + const string executionId = "exec-1"; + + reporter.AssemblyRunStarted(assembly, targetFramework: "net9.0", architecture: "x64", executionId, instanceId: "inst-1"); + + ReportTest(reporter, assembly, executionId, instanceId: "inst-1", testUid: "t-pass-1", TestOutcome.Passed); + ReportTest(reporter, assembly, executionId, instanceId: "inst-1", testUid: "t-pass-2", TestOutcome.Passed); + ReportTest(reporter, assembly, executionId, instanceId: "inst-1", testUid: "t-pass-3", TestOutcome.Passed); + ReportTest(reporter, assembly, executionId, instanceId: "inst-1", testUid: "t-skip-1", TestOutcome.Skipped); + + reporter.AssemblyRunCompleted(executionId, exitCode: 0, outputData: null, errorData: null); + + string assemblyLine = GetAssemblySummaryLine(capturingConsole.GetOutput(), assembly); + assemblyLine.Should().Contain("[+3/x0/?1]"); + } + + /// + /// In the final test-run summary, when more than one assembly ran, each assembly entry + /// must include its own per-assembly counts in the compact bracketed form + /// (https://github.com/dotnet/sdk/issues/52128). See the note on + /// + /// for why the SimpleAnsi (ASCII) variant is asserted here. + /// + [Fact] + public void TestExecutionCompleted_WithMultipleAssemblies_PrintsPerAssemblyCountsInSummary() + { + var capturingConsole = new CapturingConsole(); + + var options = new TerminalTestReporterOptions + { + AnsiMode = AnsiMode.SimpleAnsi, + ShowProgress = false, + ShowAssembly = true, + // Suppress mid-stream per-assembly lines so we can assert against the final summary only. + ShowAssemblyStartAndComplete = false, + }; + + using var reporter = new TerminalTestReporter(capturingConsole, options); + + reporter.TestExecutionStarted(DateTimeOffset.UtcNow, workerCount: 2, isDiscovery: false, isHelp: false, isRetry: false); + + const string assemblyA = "/repo/bin/Debug/net9.0/A.Tests.dll"; + const string assemblyB = "/repo/bin/Debug/net9.0/B.Tests.dll"; + + reporter.AssemblyRunStarted(assemblyA, "net9.0", "x64", executionId: "exec-A", instanceId: "inst-A"); + reporter.AssemblyRunStarted(assemblyB, "net9.0", "x64", executionId: "exec-B", instanceId: "inst-B"); + + // Assembly A: 2 passed, 1 failed, 0 skipped. + ReportTest(reporter, assemblyA, executionId: "exec-A", instanceId: "inst-A", testUid: "a-1", TestOutcome.Passed); + ReportTest(reporter, assemblyA, executionId: "exec-A", instanceId: "inst-A", testUid: "a-2", TestOutcome.Passed); + ReportTest(reporter, assemblyA, executionId: "exec-A", instanceId: "inst-A", testUid: "a-3", TestOutcome.Fail); + + // Assembly B: 5 passed, 0 failed, 2 skipped. + ReportTest(reporter, assemblyB, executionId: "exec-B", instanceId: "inst-B", testUid: "b-1", TestOutcome.Passed); + ReportTest(reporter, assemblyB, executionId: "exec-B", instanceId: "inst-B", testUid: "b-2", TestOutcome.Passed); + ReportTest(reporter, assemblyB, executionId: "exec-B", instanceId: "inst-B", testUid: "b-3", TestOutcome.Passed); + ReportTest(reporter, assemblyB, executionId: "exec-B", instanceId: "inst-B", testUid: "b-4", TestOutcome.Passed); + ReportTest(reporter, assemblyB, executionId: "exec-B", instanceId: "inst-B", testUid: "b-5", TestOutcome.Passed); + ReportTest(reporter, assemblyB, executionId: "exec-B", instanceId: "inst-B", testUid: "b-6", TestOutcome.Skipped); + ReportTest(reporter, assemblyB, executionId: "exec-B", instanceId: "inst-B", testUid: "b-7", TestOutcome.Skipped); + + reporter.AssemblyRunCompleted(executionId: "exec-A", exitCode: 1, outputData: null, errorData: null); + reporter.AssemblyRunCompleted(executionId: "exec-B", exitCode: 0, outputData: null, errorData: null); + + reporter.TestExecutionCompleted(DateTimeOffset.UtcNow, exitCode: 1); + + string output = capturingConsole.GetOutput(); + + GetAssemblySummaryLine(output, assemblyA).Should().Contain("[+2/x1/?0]"); + GetAssemblySummaryLine(output, assemblyB).Should().Contain("[+5/x0/?2]"); + } + + /// + /// When an assembly's tests were retried, the per-assembly summary should append a + /// "/r{N}" segment to the compact counts block so users can tell the final counts came from retries. + /// + [Fact] + public void AssemblyRunCompleted_WhenTestsWereRetried_ShowsRetriedCount() + { + var capturingConsole = new CapturingConsole(); + + var options = new TerminalTestReporterOptions + { + AnsiMode = AnsiMode.SimpleAnsi, + ShowProgress = false, + ShowAssembly = true, + ShowAssemblyStartAndComplete = true, + }; + + using var reporter = new TerminalTestReporter(capturingConsole, options); + + reporter.TestExecutionStarted(DateTimeOffset.UtcNow, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: true); + + const string assembly = "/repo/bin/Debug/net9.0/Flaky.Tests.dll"; + const string executionId = "exec-flaky"; + + // Attempt 1: register the first instance and report a failure. + reporter.AssemblyRunStarted(assembly, "net9.0", "x64", executionId, instanceId: "inst-1"); + ReportTest(reporter, assembly, executionId, instanceId: "inst-1", testUid: "flaky-1", TestOutcome.Fail); + + // Attempt 2: a new instance id triggers a retry; the failing test now passes. + reporter.AssemblyRunStarted(assembly, "net9.0", "x64", executionId, instanceId: "inst-2"); + ReportTest(reporter, assembly, executionId, instanceId: "inst-2", testUid: "flaky-1", TestOutcome.Passed); + + reporter.AssemblyRunCompleted(executionId, exitCode: 0, outputData: null, errorData: null); + + string assemblyLine = GetAssemblySummaryLine(capturingConsole.GetOutput(), assembly); + assemblyLine.Should().Contain("[+1/x0/?0/r1]"); + } + + private static void ReportTest(TerminalTestReporter reporter, string assembly, string executionId, string instanceId, string testUid, TestOutcome outcome) + { + reporter.TestCompleted( + assembly: assembly, + targetFramework: "net9.0", + architecture: "x64", + executionId: executionId, + instanceId: instanceId, + testNodeUid: testUid, + displayName: testUid, + informativeMessage: null, + outcome: outcome, + duration: TimeSpan.FromMilliseconds(1), + exceptions: null, + expected: null, + actual: null, + standardOutput: null, + errorOutput: null); + } + + /// + /// Finds the per-assembly summary line for the given assembly. Multiple lines may mention the + /// assembly (e.g. the "Running tests from ..." banner and the summary line). The summary line + /// is the one that contains the compact counts block written by AppendAssemblyTestCounts. + /// ANSI color escape sequences are stripped so callers can use plain-text assertions like + /// Contain("[+3/x0/?1]") (SimpleAnsi/SimpleTerminal mode uses the ASCII glyph set). + /// + private static string GetAssemblySummaryLine(string output, string assemblyPath) + { + foreach (string line in output.Split('\n')) + { + string stripped = StripAnsi(line); + if (stripped.Contains(assemblyPath, StringComparison.Ordinal) + && stripped.Contains("[+", StringComparison.Ordinal)) + { + return stripped; + } + } + + throw new InvalidOperationException( + $"Expected output to contain a per-assembly summary line for '{assemblyPath}', but it did not. Full output:{Environment.NewLine}{output}"); + } + + private static string StripAnsi(string value) => s_ansiEscapeRegex.Replace(value, string.Empty); + + private static readonly Regex s_ansiEscapeRegex = new("\x1b\\[[0-9;]*[a-zA-Z]", RegexOptions.Compiled); }