Alternate signal stack allocated during thread start is leaked if the signal stack is reconfigured by application code #136853

// dotnet run -c Release -- control  1000   # threads do nothing with sigaltstack          -> no growth
// dotnet run -c Release -- disable  1000   # install own stack, SS_DISABLE, munmap         -> +2 maps / +16 KB per thread
// dotnet run -c Release -- dangling 1000   # install own stack, munmap only (Wasmtime <= 22) -> same leak
using System.Runtime.InteropServices;

const int SS_DISABLE = 2;
const int PROT_READ = 1, PROT_WRITE = 2;
const int MAP_PRIVATE = 0x02, MAP_ANONYMOUS = 0x20;
const nint OwnStackSize = 256 * 1024;

string mode = args.Length > 0 ? args[0] : "disable";
int threads = args.Length > 1 ? int.Parse(args[1]) : 1000;

static long Maps() => File.ReadLines("/proc/self/maps").LongCount();
static long StatusKb(string key) => long.Parse(File.ReadLines("/proc/self/status").First(l => l.StartsWith(key)).Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]);

void ThreadBody()
{
    if (mode == "control")
    {
        return;
    }

    // What a native library with its own trap handler does on a thread that runs its code.
    nint stack = Native.mmap(0, OwnStackSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
    var install = new stack_t { ss_sp = stack, ss_flags = 0, ss_size = (nuint)OwnStackSize };
    if (Native.sigaltstack(ref install, out _) != 0)
    {
        throw new InvalidOperationException($"sigaltstack(install) failed: errno {Marshal.GetLastWin32Error()}");
    }

    // ... and what it does when the thread exits:
    if (mode == "disable")
    {
        var off = new stack_t { ss_sp = 0, ss_flags = SS_DISABLE, ss_size = 8192 };
        Native.sigaltstack(ref off, out _);
    }

    Native.munmap(stack, OwnStackSize);
}

Console.WriteLine($"mode={mode} threads={threads} runtime={RuntimeInformation.FrameworkDescription} os={RuntimeInformation.OSDescription}");
long maps0 = Maps(), vmdata0 = StatusKb("VmData:");
Console.WriteLine($"before: maps={maps0} VmData={vmdata0} kB");

for (int i = 0; i < threads; i++)
{
    var t = new Thread(ThreadBody);
    t.Start();
    t.Join();
}

long maps1 = Maps(), vmdata1 = StatusKb("VmData:");
Console.WriteLine($"after:  maps={maps1} VmData={vmdata1} kB");
Console.WriteLine($"delta:  maps=+{maps1 - maps0} ({(maps1 - maps0) / (double)threads:0.00} per thread) VmData=+{vmdata1 - vmdata0} kB");

// The leaked PAL stacks are 16 KB mappings: a 4 KB PROT_NONE guard page and a 12 KB rw-p region.
int guards = 0, bodies = 0;
foreach (var line in File.ReadLines("/proc/self/maps"))
{
    var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
    var range = parts[0].Split('-');
    long size = Convert.ToInt64(range[1], 16) - Convert.ToInt64(range[0], 16);
    if (parts.Length == 5 && size == 4096 && parts[1] == "---p") guards++;
    if (parts.Length == 5 && size == 12 * 1024 && parts[1] == "rw-p") bodies++;
}
Console.WriteLine($"anonymous 4 KB ---p mappings: {guards}; anonymous 12 KB rw-p mappings: {bodies}");

[StructLayout(LayoutKind.Sequential)]
struct stack_t
{
    public nint ss_sp;
    public int ss_flags;
    public nuint ss_size;
}

static class Native
{
    [DllImport("libc", SetLastError = true)] public static extern int sigaltstack(ref stack_t ss, out stack_t oss);
    [DllImport("libc", SetLastError = true)] public static extern nint mmap(nint addr, nint length, int prot, int flags, int fd, nint offset);
    [DllImport("libc", SetLastError = true)] public static extern int munmap(nint addr, nint length);
}

Actual behavior

Every exited thread leaves exactly two anonymous mappings behind, a 4 KB ---p guard page and a 12 KB rw-p region, which is the PAL's alternate stack (SIGSTKSZ + ALIGN_UP(sizeof(SignalHandlerWorkerReturnPoint), 16) + page, rounded up to 16 KB, with the first page mprotected as a guard).:

┌──────────┬─────────┬─────────────────────────┬────────────┬─────────────────────────────────────────────────┐
│   Mode   │ Threads │ Mappings before → after │ Per thread │ 4 KB ---p / 12 KB rw-p anonymous mappings after │
├──────────┼─────────┼─────────────────────────┼────────────┼─────────────────────────────────────────────────┤
│ control  │   1,000 │               247 → 260 │       0.01 │                                          17 / 2 │
├──────────┼─────────┼─────────────────────────┼────────────┼─────────────────────────────────────────────────┤
│ disable  │   1,000 │             247 → 2,256 │       2.01 │                                   1,016 / 1,003 │
├──────────┼─────────┼─────────────────────────┼────────────┼─────────────────────────────────────────────────┤
│ dangling │   1,000 │             248 → 2,257 │       2.01 │                                   1,017 / 1,000 │
├──────────┼─────────┼─────────────────────────┼────────────┼─────────────────────────────────────────────────┤
│ disable  │   5,000 │            248 → 10,264 │       2.00 │                                   5,017 / 5,002 │
└──────────┴─────────┴─────────────────────────┴────────────┴─────────────────────────────────────────────────┘

void
CPalThread::FreeSignalAlternateStack()
{
#ifndef TARGET_WASM // WebAssembly does not support alternate signal stacks.
    void *altstack = m_alternateStack;
    m_alternateStack = nullptr;

     if (altstack != nullptr)
    {
        stack_t ss, oss;
        // The man page for sigaltstack says that when the ss.ss_flags is set to SS_DISABLE,
        // all other ss fields are ignored. However, MUSL implementation checks that the
        // ss_size is >= MINSIGSTKSZ even in this case.
        ss.ss_size = MINSIGSTKSZ;
        ss.ss_flags = SS_DISABLE;
        ss.ss_sp = NULL;
        int st = sigaltstack(&ss, &oss);
        if ((st == 0) && (oss.ss_flags != SS_DISABLE))
        {
            // Make sure this altstack is this PAL's before freeing.
            if (oss.ss_sp == altstack)
            {
                int st = munmap(oss.ss_sp, oss.ss_size);
                _ASSERTE(st == 0);
            }
        }
    }
#endif // !TARGET_WASM
}