Stabilize the SOS test harness across platforms

src/SOS/Strike/util.cpp

@@ -3148,7 +3148,7 @@ BOOL GetSOSVersion(VS_FIXEDFILEINFO *pFileInfo)
                 UINT uLen = 0;
                 if (VerQueryValueA(pVersionInfo, "\\", (LPVOID *) &pTmpFileInfo, &uLen))
                 {
-                    if (pFileInfo->dwFileVersionMS == (DWORD)-1) {
+                    if (pTmpFileInfo->dwFileVersionMS == (DWORD)-1) {
                         return FALSE;
                     }
                     *pFileInfo = *pTmpFileInfo; // Copy the info

src/tests/Debuggees.proj

@@ -19,6 +19,8 @@
     <SOSSingleFileDebuggee Include="SOS.UnitTests/Debuggees/SimpleThrow/SimpleThrow.csproj" />
     <SOSSingleFileDebuggee Include="SOS.UnitTests/Debuggees/ReflectionTest/ReflectionTest.csproj" />
     <SOSSingleFileDebuggee Include="SOS.UnitTests/Debuggees/SosHarnessScenarios/SosHarnessScenarios.csproj" />
+    <SOSFrameworkDebuggee Include="@(SOSSingleFileDebuggee)" />
+    <SOSFrameworkDebuggee Remove="SOS.UnitTests/Debuggees/DynamicMethod/DynamicMethod.csproj" />
   </ItemGroup>
 
   <Target Name="PublishSOSSingleFileDebuggees"
@@ -42,4 +44,16 @@
              Properties="Configuration=$(Configuration);BuildProjectFramework=%(_SOSSingleFileRuntime.TargetFramework);RuntimeIdentifier=$(TargetRid);RuntimeFrameworkVersion=%(_SOSSingleFileRuntime.Runtime);SelfContained=true;PublishSingleFile=true;SOSSingleFileRuntimeVersion=%(_SOSSingleFileRuntime.Runtime);ArtifactsObjDir=$(ArtifactsObjDir)SOSSingleFile/%(_SOSSingleFileRuntime.TargetFramework)/$(TargetRid)/" />
   </Target>
 
+  <Target Name="BuildSOSFrameworkDebuggees"
+          AfterTargets="Build"
+          Condition="'$(OS)' == 'Windows_NT' and '$(SkipSOSFrameworkDebuggees)' != 'true'">
+    <PropertyGroup>
+      <_SOSFrameworkPlatformProperty Condition="'$(TargetArch)' == 'x86'">PlatformTarget=x86;</_SOSFrameworkPlatformProperty>
+    </PropertyGroup>
+    <MSBuild Projects="@(SOSFrameworkDebuggee)"
+             Targets="Restore;Build"
+             BuildInParallel="true"
+             Properties="Configuration=$(Configuration);BuildProjectFramework=net462;$(_SOSFrameworkPlatformProperty)DebugType=full;DebugSymbols=true;ArtifactsObjDir=$(ArtifactsObjDir)SOSFramework/$(TargetArch)/%(Filename)/" />
+  </Target>
+
 </Project>

src/tests/SOS.TestHarness/BoundedProcess.cs

@@ -0,0 +1,139 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+
+namespace SOS.TestHarness;
+
+internal sealed record BoundedProcessResult(int ExitCode, string StandardOutput, string StandardError);
+
+internal static partial class BoundedProcess
+{
+    private const int KillSignal = 9;
+    private static readonly TimeSpan s_terminationTimeout = TimeSpan.FromSeconds(5);
+
+    public static BoundedProcessResult Run(
+        ProcessStartInfo startInfo,
+        TimeSpan timeout,
+        bool isolateLinuxProcessGroup = false,
+        TimeSpan? outputDrainTimeout = null)
+    {
+        if (!startInfo.RedirectStandardOutput || !startInfo.RedirectStandardError)
+        {
+            throw new ArgumentException("Standard output and standard error must both be redirected.", nameof(startInfo));
+        }
+
+        ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(timeout, TimeSpan.Zero);
+
+        string command = $"{startInfo.FileName} {string.Join(' ', startInfo.ArgumentList)}".Trim();
+        bool hasLinuxProcessGroup = isolateLinuxProcessGroup && OperatingSystem.IsLinux();
+        if (hasLinuxProcessGroup)
+        {
+            WrapWithSetSid(startInfo);
+        }
+
+        using Process process = Process.Start(startInfo)
+            ?? throw new InvalidOperationException($"Failed to start '{command}'.");
+        int processGroupId = process.Id;
+        Task<string> stdoutTask = process.StandardOutput.ReadToEndAsync();
+        Task<string> stderrTask = process.StandardError.ReadToEndAsync();
+
+        if (!process.WaitForExit(TimeoutMilliseconds(timeout)))
+        {
+            Terminate(process, hasLinuxProcessGroup, processGroupId);
+            BoundedProcessResult output = DrainOutput(
+                process,
+                stdoutTask,
+                stderrTask,
+                command,
+                s_terminationTimeout);
+            throw new TimeoutException(
+                $"'{command}' did not exit within {timeout}.{Environment.NewLine}" +
+                $"stdout:{Environment.NewLine}{output.StandardOutput}{Environment.NewLine}" +
+                $"stderr:{Environment.NewLine}{output.StandardError}");
+        }
+
+        // Linux single-file dump helpers can survive the target while retaining its redirected handles.
+        // End the isolated group before waiting for stream EOF so those descendants cannot wedge drainage.
+        if (hasLinuxProcessGroup)
+        {
+            KillProcessGroup(processGroupId);
+        }
+
+        return DrainOutput(
+            process,
+            stdoutTask,
+            stderrTask,
+            command,
+            outputDrainTimeout ?? s_terminationTimeout);
+    }
+
+    private static BoundedProcessResult DrainOutput(
+        Process process,
+        Task<string> stdoutTask,
+        Task<string> stderrTask,
+        string command,
+        TimeSpan timeout)
+    {
+        Task outputTask = Task.WhenAll(stdoutTask, stderrTask);
+        if (!outputTask.Wait(timeout))
+        {
+            throw new TimeoutException(
+                $"'{command}' exited with code {process.ExitCode}, but its redirected output did not close " +
+                $"within {timeout}.");
+        }
+
+        return new BoundedProcessResult(
+            process.ExitCode,
+            stdoutTask.GetAwaiter().GetResult(),
+            stderrTask.GetAwaiter().GetResult());
+    }
+
+    private static void Terminate(Process process, bool hasLinuxProcessGroup, int processGroupId)
+    {
+        if (hasLinuxProcessGroup)
+        {
+            KillProcessGroup(processGroupId);
+        }
+
+        if (!process.HasExited)
+        {
+            process.Kill(entireProcessTree: true);
+        }
+
+        process.WaitForExit(TimeoutMilliseconds(s_terminationTimeout));
+    }
+
+    private static void WrapWithSetSid(ProcessStartInfo startInfo)
+    {
+        string setSid = File.Exists("/usr/bin/setsid") ? "/usr/bin/setsid" :
+            File.Exists("/bin/setsid") ? "/bin/setsid" :
+            throw new FileNotFoundException("Could not locate setsid for Linux process-group isolation.");
+
+        string executable = startInfo.FileName;
+        string[] arguments = startInfo.ArgumentList.ToArray();
+        startInfo.FileName = setSid;
+        startInfo.ArgumentList.Clear();
+        startInfo.ArgumentList.Add(executable);
+        foreach (string argument in arguments)
+        {
+            startInfo.ArgumentList.Add(argument);
+        }
+    }
+
+    private static void KillProcessGroup(int processGroupId)
+    {
+        _ = KillUnix(-processGroupId, KillSignal);
+    }
+
+    private static int TimeoutMilliseconds(TimeSpan timeout) =>
+        (int)Math.Min(timeout.TotalMilliseconds, int.MaxValue);
+
+    [LibraryImport("libc", EntryPoint = "kill", SetLastError = true)]
+    private static partial int KillUnix(int processId, int signal);
+}

src/tests/SOS.TestHarness/ChildEngineClient.cs

@@ -23,6 +23,7 @@ public sealed class ChildEngineClient : ILiveDebuggerHost
     private readonly StreamWriter _stdin;
     private readonly BlockingCollection<string> _lines = new();
     private readonly Thread _reader;
+    private readonly Task<string> _stderr;
 
     public string Name { get; }
 
@@ -92,6 +93,7 @@ private ChildEngineClient(string name, string mode, IReadOnlyList<string> modeAr
 
         _process = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start EngineHost");
         _stdin = _process.StandardInput;
+        _stderr = _process.StandardError.ReadToEndAsync();
 
         _reader = new Thread(ReadLoop) { IsBackground = true, Name = $"enginehost-reader-{name}" };
         _reader.Start();
@@ -144,10 +146,10 @@ private void WaitForReady(TimeSpan timeout)
     {
         while (true)
         {
-            if (!_lines.TryTake(out string? line, (int)timeout.TotalMilliseconds, HarnessCancellation.Token))
-            {
-                throw new TimeoutException("EngineHost did not become ready in time.");
-            }
+            string line = ReadLine(
+                timeout,
+                "EngineHost did not become ready in time.",
+                "before becoming ready");
 
             if (line == EngineProtocol.Ready)
             {
@@ -161,10 +163,10 @@ private string DrainToEnd(TimeSpan timeout, string command)
         StringBuilder sb = new();
         while (true)
         {
-            if (!_lines.TryTake(out string? line, (int)timeout.TotalMilliseconds, HarnessCancellation.Token))
-            {
-                throw new TimeoutException($"EngineHost did not return output for '{command}' within {timeout}.");
-            }
+            string line = ReadLine(
+                timeout,
+                $"EngineHost did not return output for '{command}' within {timeout}.",
+                $"while running '{command}'");
 
             if (line == EngineProtocol.End)
             {
@@ -186,12 +188,48 @@ private string DrainToEnd(TimeSpan timeout, string command)
         return sb.ToString();
     }
 
+    private string ReadLine(TimeSpan timeout, string timeoutMessage, string exitContext)
+    {
+        if (_lines.TryTake(out string? line, (int)timeout.TotalMilliseconds, HarnessCancellation.Token))
+        {
+            return line;
+        }
+
+        if (_lines.IsCompleted || _process.HasExited)
+        {
+            throw CreateEngineHostExitException(exitContext);
+        }
+
+        throw new TimeoutException(timeoutMessage);
+    }
+
+    private InvalidOperationException CreateEngineHostExitException(string context)
+    {
+        bool exited = _process.HasExited || _process.WaitForExit(1000);
+        string exitDescription = exited
+            ? $"exited with code {_process.ExitCode}"
+            : "closed its standard output";
+        string stderr = exited && _stderr.Wait(TimeSpan.FromSeconds(2))
+            ? _stderr.GetAwaiter().GetResult().Trim()
+            : string.Empty;
+        string details = stderr.Length == 0 ? string.Empty : $"{Environment.NewLine}{stderr}";
+
+        return new InvalidOperationException($"EngineHost {exitDescription} {context}.{details}");
+    }
+
     private void ReadLoop()
     {
-        string? line;
-        while ((line = _process.StandardOutput.ReadLine()) is not null)
+        try
+        {
+            string? line;
+            while ((line = _process.StandardOutput.ReadLine()) is not null)
+            {
+                _lines.Add(line);
+            }
+        }
+        finally
         {
-            _lines.Add(line);
+            _lines.CompleteAdding();
         }
     }
 

src/tests/SOS.TestHarness/DbgEngCapturer.cs

@@ -71,7 +71,7 @@ string Run(string command)
                 }
                 else // Crash
                 {
-                    RunToBreak(control, "second-chance crash");
+                    RunToBreak(control, "second-chance crash", requireSecondChanceException: true);
                 }
 
                 Run($".dump /o {DbgEngDumpOption(dumpKind)} \"{dumpPath}\"");
@@ -102,7 +102,7 @@ string Run(string command)
         _ => throw new ArgumentOutOfRangeException(nameof(dumpKind), dumpKind, "Unsupported dump kind"),
     };
 
-    private static void RunToBreak(IDebugControl control, string what)
+    private static void RunToBreak(IDebugControl control, string what, bool requireSecondChanceException = false)
     {
         const int MaxResumes = 40;
         for (int i = 0; i < MaxResumes; i++)
@@ -111,7 +111,10 @@ private static void RunToBreak(IDebugControl control, string what)
             control.WaitForEvent(TimeSpan.FromSeconds(60));
             control.GetExecutionStatus(out DEBUG_STATUS status);
 
-            if (status == DEBUG_STATUS.BREAK)
+            if (status == DEBUG_STATUS.BREAK
+                && (!requireSecondChanceException
+                    || (control.GetLastEvent(out DEBUG_LAST_EVENT_INFO_EXCEPTION exception, out _, out _)
+                        && exception.FirstChance == 0)))
             {
                 return;
             }

src/tests/SOS.TestHarness/DbgEngLiveHost.cs

@@ -114,7 +114,9 @@ public SosOutput RunToCrash()
                 Control.WaitForEvent(TimeSpan.FromSeconds(60));
                 Control.GetExecutionStatus(out DEBUG_STATUS status);
 
-                if (status == DEBUG_STATUS.BREAK)
+                if (status == DEBUG_STATUS.BREAK
+                    && Control.GetLastEvent(out DEBUG_LAST_EVENT_INFO_EXCEPTION exception, out _, out _)
+                    && exception.FirstChance == 0)
                 {
                     return; // second-chance crash break
                 }

src/tests/SOS.TestHarness/DumpSession.cs

@@ -21,18 +21,20 @@ namespace SOS.TestHarness;
 ///   <item><b>dotnet-dump</b> children busy-wait on stdin at ~100% CPU, so keeping many alive would
 ///   saturate the machine. They route through a capacity-1 <see cref="HostSlot"/> (most-recently-used
 ///   stays open, reopened on demand).</item>
+///   <item><b>lldb</b> children retain their loaded core and hosted SOS runtime. They use a separate
+///   capacity-1 slot so memoized sessions cannot accumulate enough processes to exhaust memory.</item>
 /// </list>
 /// </summary>
 internal sealed class DumpSession : IPooledHost, IDisposable
 {
     private readonly Host _hostKind;
-    private readonly bool _pooled;       // dotnet-dump: route through the single slot
+    private readonly bool _pooled;
     private readonly HostSlot? _slot;
     private readonly object _gate = new(); // serializes concurrent commands on this shared child
     private IDebuggerHost? _host;        // kept-alive host for non-pooled (cdb child) targets
 
-    // One diagnostics collector for the life of this session (survives the pooled dotnet-dump host being
-    // closed and reopened), for the child-process hosts that support capture. Null for the cdb child host.
+    // One diagnostics collector for the life of this session (survives a pooled host being closed and
+    // reopened), for the child-process hosts that support capture. Null for the cdb child host.
     private readonly HostDiagnostics? _diagnostics;
 
     public Host Host { get; }
@@ -59,10 +61,10 @@ internal DumpSession(Host hostKind, string targetName, string stopName, Flavor f
         CoreVersion = coreVersion;
         Dac = dac;
 
-        // dotnet-dump children spin on stdin -> bound to one via the slot. cdb children block
-        // when idle -> keep alive concurrently (no slot), which is the subprocess-backend payoff.
-        _pooled = hostKind == Host.DotnetDump;
-        _slot = _pooled ? HostSlot.DotNetDump : null;
+        // Bound resource-heavy LLDB and dotnet-dump children independently. cdb children block when
+        // idle and remain cheap enough to keep per session.
+        _slot = HostSlotFor(hostKind);
+        _pooled = _slot is not null;
 
         // The child-process hosts (lldb, dotnet-dump) capture their stdout/stderr and crash dumps; the cdb
         // child host runs dbgeng out-of-process and is not wired for capture.
@@ -80,8 +82,8 @@ internal DumpSession(Host hostKind, string targetName, string stopName, Flavor f
     /// Run a SOS command against this target (host prefixing handled by the host). A shared target
     /// may be handed to several tests at once (it is memoized by host/target/stop/flavor), and the
     /// cdb backend is a single child process whose stdin/stdout pipe is not safe for concurrent
-    /// callers — so non-pooled commands are serialized on a per-target gate. The dotnet-dump path
-    /// serializes itself on the slot lock.
+    /// callers — so non-pooled commands are serialized on a per-target gate. Pooled paths serialize
+    /// themselves on their slot locks.
     /// </summary>
     public SosOutput Sos(string command) =>
         RunCommand("SOS", command, h => h.Sos(command));
@@ -121,7 +123,14 @@ private SosOutput RunGuarded(Func<IDebuggerHost, SosOutput> action)
         }
     }
 
-    // IPooledHost — used only for the pooled (dotnet-dump) path.
+    internal static HostSlot? HostSlotFor(Host hostKind) => hostKind switch
+    {
+        Host.Lldb => HostSlot.Lldb,
+        Host.DotnetDump => HostSlot.DotNetDump,
+        _ => null,
+    };
+
+    // IPooledHost — used only for the pooled LLDB and dotnet-dump paths.
 
     IDebuggerHost IPooledHost.Host => _host!;