[CI] Avoid generic build failures for reported test failures

diagnostics.yml

@@ -61,57 +61,58 @@ extends:
         #                          #
         ############################
 
-      - template: ${{ variables.sourceBuildTemplate }}
-        parameters:
-          enableInternalSources: true
-          platform:
-            name: Complete
-            buildScript: ./eng/common/build.sh
+      - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}:
+        - template: ${{ variables.sourceBuildTemplate }}
+          parameters:
+            enableInternalSources: true
+            platform:
+              name: Complete
+              buildScript: ./eng/common/build.sh
 
       ############################
       #                          #
       #        Build legs        #
       #                          #
       ############################
 
-      - template: /eng/pipelines/build.yml
-        parameters:
-          jobTemplate: ${{ variables.jobTemplate }}
-          name: Windows
-          osGroup: Windows_NT
-          buildOnly: true
-          publishTestArtifacts: true
-          buildConfigs:
-          - configuration: Debug
-            architecture: x64
-            artifactUploadPath: bin/Windows_NT.x64.Debug
-          - configuration: Release
-            architecture: x64
-            artifactUploadPath: bin
-          - configuration: Release
-            architecture: x86
-            artifactUploadPath: bin/Windows_NT.x86.Release
-          - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      # Keep only the build required by Alpine3_23_x64_Release in pull requests while
+      # validating Build Analysis.
+      - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}:
+        - template: /eng/pipelines/build.yml
+          parameters:
+            jobTemplate: ${{ variables.jobTemplate }}
+            name: Windows
+            osGroup: Windows_NT
+            buildOnly: true
+            publishTestArtifacts: true
+            buildConfigs:
+            - configuration: Debug
+              architecture: x64
+              artifactUploadPath: bin/Windows_NT.x64.Debug
             - configuration: Release
-              architecture: arm64
-              artifactUploadPath: bin/Windows_NT.arm64.Release
+              architecture: x64
+              artifactUploadPath: bin
+            - configuration: Release
+              architecture: x86
+              artifactUploadPath: bin/Windows_NT.x86.Release
+            - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+              - configuration: Release
+                architecture: arm64
+                runTests: false
+                artifactUploadPath: bin/Windows_NT.arm64.Release
 
-      - template: /eng/pipelines/build.yml
-        parameters:
-          jobTemplate: ${{ variables.jobTemplate }}
-          osGroup: Linux
-          container: linux_x64
-          crossBuild: true
-          buildOnly: true
-          publishTestArtifacts: true
-          buildConfigs:
-          - configuration: Release
-            architecture: x64
-            artifactUploadPath: bin/linux.x64.Release
-          - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
-            - configuration: Debug
+        - template: /eng/pipelines/build.yml
+          parameters:
+            jobTemplate: ${{ variables.jobTemplate }}
+            osGroup: Linux
+            container: linux_x64
+            crossBuild: true
+            buildOnly: true
+            publishTestArtifacts: true
+            buildConfigs:
+            - configuration: Release
               architecture: x64
-              artifactUploadPath: bin/linux.x64.Debug
+              artifactUploadPath: bin/linux.x64.Release
 
       - template: /eng/pipelines/build.yml
         parameters:
@@ -128,42 +129,32 @@ extends:
             architecture: x64
             artifactUploadPath: bin/linux.x64.Release
             artifactTargetPath: bin/linux-musl.x64.Release
-          - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
-            - configuration: Debug
-              architecture: x64
-              artifactUploadPath: bin/linux.x64.Debug
-              artifactTargetPath: bin/linux-musl.x64.Debug
 
-      - template: /eng/pipelines/build.yml
-        parameters:
-          jobTemplate: ${{ variables.jobTemplate }}
-          osGroup: MacOS
-          buildOnly: true
-          publishTestArtifacts: true
-          buildConfigs:
-          - configuration: Release
-            architecture: x64
-            artifactUploadPath: bin/osx.x64.Release
-          - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
-            - configuration: Debug
+      # Temporarily skip the slower macOS legs while validating Build Analysis on this PR.
+      - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}:
+        - template: /eng/pipelines/build.yml
+          parameters:
+            jobTemplate: ${{ variables.jobTemplate }}
+            osGroup: MacOS
+            buildOnly: true
+            publishTestArtifacts: true
+            buildConfigs:
+            - configuration: Release
               architecture: x64
-              artifactUploadPath: bin/osx.x64.Debug
+              artifactUploadPath: bin/osx.x64.Release
 
-      - template: /eng/pipelines/build.yml
-        parameters:
-          jobTemplate: ${{ variables.jobTemplate }}
-          osGroup: MacOS
-          crossBuild: true
-          buildOnly: true
-          buildConfigs:
-          - configuration: Release
-            architecture: arm64
-            artifactUploadPath: bin/osx.arm64.Release
-          - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
-            - configuration: Debug
+        - template: /eng/pipelines/build.yml
+          parameters:
+            jobTemplate: ${{ variables.jobTemplate }}
+            osGroup: MacOS
+            crossBuild: true
+            buildOnly: true
+            buildConfigs:
+            - configuration: Release
               architecture: arm64
+              artifactUploadPath: bin/osx.arm64.Release
 
-      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
         - template: /eng/pipelines/build.yml
           parameters:
             jobTemplate: ${{ variables.jobTemplate }}
@@ -230,48 +221,43 @@ extends:
         dependsOn: build
         jobs:
 
-        - template: /eng/pipelines/build.yml
-          parameters:
-            jobTemplate: ${{ variables.jobTemplate }}
-            name: Windows_Test
-            osGroup: Windows_NT
-            dependsOn: Windows
-            testOnly: true
-            buildConfigs:
-            - configuration: Debug
-              architecture: x64
-            - configuration: Release
-              architecture: x64
-            - configuration: Release
-              architecture: x86
-
-        - template: /eng/pipelines/build.yml
-          parameters:
-            jobTemplate: ${{ variables.jobTemplate }}
-            name: MacOS_Test
-            osGroup: MacOS
-            dependsOn: MacOS
-            testOnly: true
-            buildConfigs:
-            - configuration: Release
-              architecture: x64
-            - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
+        - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}:
+          - template: /eng/pipelines/build.yml
+            parameters:
+              jobTemplate: ${{ variables.jobTemplate }}
+              name: Windows_Test
+              osGroup: Windows_NT
+              dependsOn: Windows
+              testOnly: true
+              buildConfigs:
               - configuration: Debug
                 architecture: x64
+              - configuration: Release
+                architecture: x64
+              - configuration: Release
+                architecture: x86
+
+          - template: /eng/pipelines/build.yml
+            parameters:
+              jobTemplate: ${{ variables.jobTemplate }}
+              name: MacOS_Test
+              osGroup: MacOS
+              dependsOn: MacOS
+              testOnly: true
+              buildConfigs:
+              - configuration: Release
+                architecture: x64
 
-        - template: /eng/pipelines/build.yml
-          parameters:
-            jobTemplate: ${{ variables.jobTemplate }}
-            name: Ubuntu_22_04
-            osGroup: Linux
-            container: test_ubuntu_22_04
-            dependsOn: Linux
-            testOnly: true
-            buildConfigs:
-            - configuration: Release
-              architecture: x64
-            - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
-              - configuration: Debug
+          - template: /eng/pipelines/build.yml
+            parameters:
+              jobTemplate: ${{ variables.jobTemplate }}
+              name: Ubuntu_22_04
+              osGroup: Linux
+              container: test_ubuntu_22_04
+              dependsOn: Linux
+              testOnly: true
+              buildConfigs:
+              - configuration: Release
                 architecture: x64
 
         - template: /eng/pipelines/build.yml
@@ -287,9 +273,6 @@ extends:
             buildConfigs:
             - configuration: Release
               architecture: x64
-            - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
-              - configuration: Debug
-                architecture: x64
 
         # - ${{ if ne(variables['System.TeamProject'], 'public') }}:
         #   - template: /eng/pipelines/build.yml
@@ -397,13 +380,14 @@ extends:
           symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=false'
           publishInstallersAndChecksums: true
 
-    # This sets up the bits to do a Release.
-    - template: /eng/pipelines/prepare-release.yml
-      parameters:
-        isOfficialBuild: ${{ variables['isOfficialBuild'] }}
-        repoAllowList: https://github.com/dotnet/diagnostics https://dev.azure.com/dnceng/internal/_git/dotnet-diagnostics
-        productAllowList: diagnostics dotnet-diagnostics
-        ${{ if eq(variables['isOfficialBuild'], 'true') }}:
-          dependsOn: [ publish_using_darc ]
-        ${{ else }}:
-          dependsOn: [ build ]
+    - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}:
+      # This sets up the bits to do a Release.
+      - template: /eng/pipelines/prepare-release.yml
+        parameters:
+          isOfficialBuild: ${{ variables['isOfficialBuild'] }}
+          repoAllowList: https://github.com/dotnet/diagnostics https://dev.azure.com/dnceng/internal/_git/dotnet-diagnostics
+          productAllowList: diagnostics dotnet-diagnostics
+          ${{ if eq(variables['isOfficialBuild'], 'true') }}:
+            dependsOn: [ publish_using_darc ]
+          ${{ else }}:
+            dependsOn: [ build ]

eng/Tests.targets

@@ -0,0 +1,28 @@
+<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
+<Project>
+
+  <Target Name="ValidateSuppressedTestFailures"
+          AfterTargets="RunTests"
+          Condition="'$(_ErrorOnTestFailure)' == 'false'">
+    <!-- A missing result means the runner failed rather than merely reporting failed tests. -->
+    <Error Condition="!Exists('%(TestToRun.ResultsXmlPath)')"
+           Text="The test runner failed to produce results for %(TestToRun.Identity)." />
+
+    <XmlPeek XmlInputPath="%(TestToRun.ResultsXmlPath)"
+             Query="/assemblies/assembly[not(@errors)]"
+             Condition="Exists('%(TestToRun.ResultsXmlPath)')">
+      <Output TaskParameter="Result" ItemName="_TestAssembliesWithoutErrorCounts" />
+    </XmlPeek>
+    <Error Condition="'@(_TestAssembliesWithoutErrorCounts)' != ''"
+           Text="The test runner produced a result with missing assembly error counts in @(TestToRun->'%(ResultsXmlPath)', ', ')." />
+
+    <XmlPeek XmlInputPath="%(TestToRun.ResultsXmlPath)"
+             Query="/assemblies/assembly/@errors"
+             Condition="Exists('%(TestToRun.ResultsXmlPath)')">
+      <Output TaskParameter="Result" ItemName="_TestAssemblyErrors" />
+    </XmlPeek>
+    <Error Condition="'%(_TestAssemblyErrors.Identity)' != '0'"
+           Text="The test runner reported %(_TestAssemblyErrors.Identity) assembly-level error(s) in @(TestToRun->'%(ResultsXmlPath)', ', ')." />
+  </Target>
+
+</Project>

eng/pipelines/build.yml

@@ -40,7 +40,7 @@ parameters:
   default: {}
 
 # Build configs. An object with the following properties: configuration, architecture.
-# Optionally it can also contain artifactUploadPath, artifactTargetPath
+# Optionally it can also contain artifactUploadPath, artifactTargetPath, runTests
 - name: buildConfigs
   type: object
   default: {}
@@ -156,6 +156,9 @@ jobs:
       - ${{ if eq(parameters.buildOnly, 'true') }}:
         - _TestArgs: ''
 
+      - ${{ if eq(config.runTests, false) }}:
+        - _TestArgs: ''
+
       # For testing msrc's and service releases. The RuntimeSourceVersion is either "default" or the service release version to test
       - _InternalInstallArgs: ''
       - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
@@ -200,6 +203,11 @@ jobs:
         ${{ else }}:
           displayName: Build / Test
         condition: succeeded()
+        ${{ if and(ne(parameters.buildOnly, 'true'), ne(config.runTests, false)) }}:
+          # PublishTestResults reports individual failures to Build Analysis. Do not also fail
+          # MSBuild for an xUnit failure, which would add a generic build/script failure.
+          env:
+            _ErrorOnTestFailure: 'false'
       - ${{ if ne(config.artifactUploadPath, '') }}:
         - task: CopyFiles@2
           displayName: Gather binaries for publish
@@ -236,20 +244,35 @@ jobs:
               targetPath: '$(Build.ArtifactStagingDirectory)/testartifacts'
               artifactName: TestArtifacts_$(_PhaseName)
 
-      - ${{ if ne(parameters.buildOnly, 'true') }}:
+      - ${{ if and(ne(parameters.buildOnly, 'true'), ne(config.runTests, false)) }}:
         # Publish test results to Azure Pipelines
         - task: PublishTestResults@2
+          displayName: Publish partial test results
           inputs:
             testResultsFormat: xUnit
             testResultsFiles: '**/*.xml'
             searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults'
             testRunTitle: 'Tests $(_PhaseName)'
-            failTaskOnFailedTests: true
+            failTaskOnFailedTests: false
             publishRunAttachments: true
             mergeTestResults: true
             buildConfiguration: ${{ config.configuration }}
           continueOnError: true
-          condition: always()
+          condition: failed()
+
+        - task: PublishTestResults@2
+          inputs:
+            testResultsFormat: xUnit
+            testResultsFiles: '**/*.xml'
+            searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults'
+            testRunTitle: 'Tests $(_PhaseName)'
+            # Build Analysis evaluates the published test failures separately.
+            failTaskOnFailedTests: false
+            failTaskOnMissingResultsFile: true
+            publishRunAttachments: true
+            mergeTestResults: true
+            buildConfiguration: ${{ config.configuration }}
+          condition: succeeded()
 
       - task: CopyFiles@2
         displayName: Gather Logs

src/tests/Microsoft.Diagnostics.Monitoring.EventPipe/LogsPipelineUnitTests.cs

@@ -35,6 +35,10 @@ public LogsPipelineUnitTests(ITestOutputHelper output)
         [SkippableTheory, MemberData(nameof(Configurations))]
         public async Task TestLogsPipeline(TestConfiguration config)
         {
+            // Temporarily force the known failure from https://github.com/dotnet/diagnostics/issues/5659
+            // to validate that Build Analysis recognizes it without a generic build failure.
+            Assert.Equal(6, 2);
+
             // TODO: When distributed tracing support lands EventPipeTracee
             // gains the ability to start traces. When there is an active trace
             // on .NET9+ logs should automatically receive correlation