[automated] Merge branch 'release/9.0.1xx' => 'release/9.0.3xx'

src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml

@@ -51,7 +51,6 @@ extends:
     containers:
       licenseScanContainer:
         image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-source-build-test-amd64
-        options: '--memory=20g'
     pool:
       name: NetCore1ESPool-Svc-Internal
       image: build.azurelinux.3.amd64
@@ -79,20 +78,37 @@ extends:
               local name="$1"
               local repoPath="$2"
               local scanIgnorePatterns="${3:-}"
+              local scanProcessCount="${4:-}"
               if [ ! -z "$matrix" ]; then
                 matrix="$matrix,"
               fi
-              matrix="$matrix \"$name\": { \"repoPath\": \"$repoPath\", \"scanIgnorePatterns\": \"$scanIgnorePatterns\" }"
+              matrix="$matrix \"$name\": { \"repoPath\": \"$repoPath\", \"scanIgnorePatterns\": \"$scanIgnorePatterns\", \"scanProcessCount\": \"$scanProcessCount\" }"
             }
 
             # Repos whose src/ subdirectories should be scanned as separate jobs.
             # The repo root is also scanned with --ignore to skip subdirectories covered by sub-scans.
             splitRepos="source-build-assets"
 
+            # Repos that should scan with a reduced number of scancode processes. This is
+            # independent of splitRepos. Their large, text-heavy content deadlocks scancode's
+            # multiprocessing pool at the end of a scan, so they are scanned serially (0 disables
+            # multiprocessing). Other repos use the test's default. Flows through the matrix.
+            serialScanRepos="source-build-assets"
+            serialScanProcessCount=0
+
+            # Returns the scancode --processes value for a repo, or empty to use the test's default.
+            getScanProcessCount() {
+              local repoName="$1"
+              if echo "$serialScanRepos" | grep -qw "$repoName"; then
+                echo "$serialScanProcessCount"
+              fi
+            }
+
             # If the repo name is provided, only scan that repo.
             if [ ! -z "$specificRepoName" ]; then
               repoName="$specificRepoName"
               dir="$vmrSrcDir/$specificRepoName"
+              scanProcessCount=$(getScanProcessCount "$repoName")
 
               if echo "$splitRepos" | grep -qw "$repoName"; then
                 ignorePatterns=""
@@ -104,17 +120,18 @@ extends:
                       ignorePatterns="$ignorePatterns "
                     fi
                     ignorePatterns="$ignorePatterns$subdirName"
-                    addMatrixEntry "${repoName}_${subdirName}" "$subdir"
+                    addMatrixEntry "${repoName}_${subdirName}" "$subdir" "" "$scanProcessCount"
                   fi
                 done
-                addMatrixEntry "$repoName" "$dir" "$ignorePatterns"
+                addMatrixEntry "$repoName" "$dir" "$ignorePatterns" "$scanProcessCount"
               else
-                addMatrixEntry "$repoName" "$dir"
+                addMatrixEntry "$repoName" "$dir" "" "$scanProcessCount"
               fi
             else
               for dir in "$vmrSrcDir"/*/
               do
                 repoName=$(basename "$dir")
+                scanProcessCount=$(getScanProcessCount "$repoName")
 
                 if echo "$splitRepos" | grep -qw "$repoName"; then
                   # Root scan: skip subdirectories that are covered by sub-scans
@@ -129,12 +146,12 @@ extends:
                         ignorePatterns="$ignorePatterns "
                       fi
                       ignorePatterns="$ignorePatterns$subdirName"
-                      addMatrixEntry "${repoName}_${subdirName}" "$subdir"
+                      addMatrixEntry "${repoName}_${subdirName}" "$subdir" "" "$scanProcessCount"
                     fi
                   done
-                  addMatrixEntry "$repoName" "$dir" "$ignorePatterns"
+                  addMatrixEntry "$repoName" "$dir" "$ignorePatterns" "$scanProcessCount"
                 else
-                  addMatrixEntry "$repoName" "$dir"
+                  addMatrixEntry "$repoName" "$dir" "" "$scanProcessCount"
                 fi
               done
             fi
@@ -176,6 +193,7 @@ extends:
             -clp:v=m
             /p:SmokeTestsLicenseScanPath=$(repoPath)
             "/p:SourceBuildTestsLicenseScanIgnorePatterns=$(scanIgnorePatterns)"
+            /p:SourceBuildTestsLicenseScanProcessCount=$(scanProcessCount)
             /p:SmokeTestsWarnOnLicenseScanDiffs=false
             /p:TargetRid=linux-x64
             /p:PortableRid=linux-x64

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs

@@ -23,6 +23,8 @@ internal static class Config
     public static bool ExcludeOmniSharpTests => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(ExcludeOmniSharpTests))!, out bool excludeOmniSharpTests) && excludeOmniSharpTests;
     public static string? LicenseScanPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(LicenseScanPath))!;
     public static string? LicenseScanIgnorePatterns => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(LicenseScanIgnorePatterns))!;
+    public static int? LicenseScanProcessCount =>
+        int.TryParse((string?)AppContext.GetData(ConfigSwitchPrefix + nameof(LicenseScanProcessCount)), out int processCount) ? processCount : null;
     public static string? MsftSdkTarballPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(MsftSdkTarballPath))!;
     public static string? PoisonReportPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PoisonReportPath))!;
     public static string? PrereqsPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PrereqsPath))!;

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs

@@ -152,6 +152,12 @@ public void ScanForLicenses()
         // Indicates how long until a timeout occurs for scanning a given file
         const int FileScanTimeoutSeconds = 1800;
 
+        // Number of parallel processes scancode should use. Large, text-heavy repos can
+        // deadlock scancode's multiprocessing pool at the end of a scan, so the pipeline
+        // scans them serially by passing 0. Default to 4 when not specified by the pipeline.
+        const int DefaultProcessCount = 4;
+        int processCount = Config.LicenseScanProcessCount ?? DefaultProcessCount;
+
         string scancodeResultsPath = Path.Combine(Config.LogsDirectory, "scancode-results.json");
 
         // Combine default and additional ignore patterns
@@ -167,7 +173,7 @@ public void ScanForLicenses()
         string ignoreOptions = string.Join(" ", allIgnorePatterns.Select(pattern => $"--ignore {pattern}"));
         ExecuteHelper.ExecuteProcessValidateExitCode(
             "scancode",
-            $"--license --processes 4 --timeout {FileScanTimeoutSeconds} --strip-root --only-findings {ignoreOptions} --json-pp {scancodeResultsPath} {Config.LicenseScanPath}",
+            $"--license --processes {processCount} --timeout {FileScanTimeoutSeconds} --strip-root --only-findings {ignoreOptions} --json-pp {scancodeResultsPath} {Config.LicenseScanPath}",
             OutputHelper);
 
         JsonDocument doc = JsonDocument.Parse(File.ReadAllText(scancodeResultsPath));

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj

@@ -75,6 +75,10 @@
                                       Condition="'$(SourceBuildTestsLicenseScanIgnorePatterns)' != ''">
         <Value>$(SourceBuildTestsLicenseScanIgnorePatterns)</Value>
       </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).LicenseScanProcessCount"
+                                      Condition="'$(SourceBuildTestsLicenseScanProcessCount)' != ''">
+        <Value>$(SourceBuildTestsLicenseScanProcessCount)</Value>
+      </RuntimeHostConfigurationOption>
       <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).MsftSdkTarballPath">
         <Value>$(MsftSdkTarballPath)</Value>
       </RuntimeHostConfigurationOption>

test/Microsoft.NET.TestFramework/Commands/TestCommand.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Diagnostics;
@@ -20,6 +20,15 @@ public abstract class TestCommand
 
         public List<string> EnvironmentToRemove { get; } = new List<string>();
 
+        /// <summary>
+        /// When true, the full command output is echoed to <see cref="Log"/> instead of being capped by a
+        /// <see cref="TruncatingTestOutputHelper"/>. Leave false (the default) so a single very verbose
+        /// command (e.g. <c>dotnet test -v diag</c>) can't flood xUnit's IPC channel and trigger a
+        /// spurious hang-dump timeout. Set it (or call <see cref="WithoutTruncatedOutputLogging"/>) for
+        /// tests that need the complete output in the test log.
+        /// </summary>
+        public bool DisableTruncatedOutputLogging { get; set; }
+
         //  These only work via Execute(), not when using GetProcessStartInfo()
         public Action<string> CommandOutputHandler { get; set; }
         public Action<Process> ProcessStartedHandler { get; set; }
@@ -61,6 +70,16 @@ public TestCommand WithTraceOutput()
             return this;
         }
 
+        /// <summary>
+        /// Opts this command out of bounded output logging so the complete command output is echoed to
+        /// <see cref="Log"/>. See <see cref="DisableTruncatedOutputLogging"/>.
+        /// </summary>
+        public TestCommand WithoutTruncatedOutputLogging()
+        {
+            DisableTruncatedOutputLogging = true;
+            return this;
+        }
+
         private SdkCommandSpec CreateCommandSpec(IEnumerable<string> args)
         {
             var commandSpec = CreateCommand(args);
@@ -120,29 +139,32 @@ private static bool SuccessOrNotTransientRestoreError(CommandResult result)
 
         public virtual CommandResult Execute(IEnumerable<string> args)
         {
-            var spec = CreateCommandSpec(args);
-
-            var command = spec
+            var command = CreateCommandSpec(args)
                 .ToCommand(_doNotEscapeArguments)
                 .CaptureStdOut()
                 .CaptureStdErr();
 
-            command.OnOutputLine(line =>
+            if (CommandOutputHandler != null)
             {
-                Log.WriteLine($"》{line}");
-                CommandOutputHandler?.Invoke(line);
-            });
-
-            command.OnErrorLine(line =>
-            {
-                Log.WriteLine($"❌{line}");
-            });
-
-            var display = $"dotnet {string.Join(" ", spec.Arguments)}";
+                command.OnOutputLine(CommandOutputHandler);
+            }
 
-            Log.WriteLine($"Executing '{display}':");
             var result = ((Command)command).Execute(ProcessStartedHandler);
-            Log.WriteLine($"Command '{display}' exited with exit code {result.ExitCode}.");
+
+            // By default, cap how much of the (potentially enormous) command output is echoed to the test
+            // log so a single very verbose command (e.g. `dotnet test -v diag`) can't flush hundreds of
+            // megabytes over the test host's IPC channel and trigger a spurious hang-dump timeout. The full
+            // StdOut/StdErr remain on the returned CommandResult; only the log echo is bounded. Tests that
+            // need the complete output in the log can opt out with WithoutTruncatedOutputLogging().
+            if (DisableTruncatedOutputLogging)
+            {
+                LogCommandResult(Log, result);
+            }
+            else
+            {
+                using var truncatingLog = new TruncatingTestOutputHelper(Log);
+                LogCommandResult(truncatingLog, result);
+            }
 
             return result;
         }