[API Proposal]: low level TLS machine #128871

namespace System.Net.Security;

public enum TlsOperationStatus
{
    Complete = 0,                    // operation finished; check bytesWritten / bytesRead
    WantRead = 1,                    // need more ciphertext from peer
    WantWrite = 2,                   // output buffer too small; call DrainPendingOutput
    Closed = 3,                      // peer sent close_notify
    WantCredentials = 4,             // client must supply a client cert
    NeedsCertificateValidation = 5,  // peer cert is available; caller must accept/reject
    NeedsServerOptions = 6,          // ClientHello parsed; caller must call SetServerContext
}

// Reusable configuration; one context can spawn many sessions concurrently.
// Options are cloned into each session at Create time, so the context is
// safe to share across threads after creation.
[Experimental("SYSLIB5007", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")]
public sealed partial class TlsContext : IDisposable
{
    public bool IsServer { get; }
    public static TlsContext Create(SslServerAuthenticationOptions? options); // null = deferred (SNI flow)
    public static TlsContext Create(SslClientAuthenticationOptions options);
    public void Dispose();
}

// Per-connection TLS state machine. Two interchangeable modes:
//   1. Sans-I/O (default factory): caller moves bytes via ProcessHandshake /
//      Encrypt / Decrypt. The session never touches a transport.
//   2. Socket-bound (SafeSocketHandle factory): session owns a non-blocking
//      SafeSocketHandle and drives I/O itself; methods return WantRead /
//      WantWrite on EAGAIN so the caller can wait on its preferred readiness
//      primitive (epoll/kqueue/IOCP/Task).
[Experimental("SYSLIB5007", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")]
public sealed partial class TlsSession : IDisposable
{
    // -------- Factories --------
    public static TlsSession Create(TlsContext context);
    public static TlsSession Create(TlsContext context, SafeSocketHandle socket);

    // -------- State --------
    public bool IsHandshakeComplete { get; }
    public bool HasPendingOutput { get; }
    public string? TargetHostName { get; set; }            // SNI. Client: caller sets before handshake. Server: auto-populated from parsed ClientHello (see CaptureClientHello switch).
    public SslClientHelloInfo? ClientHelloInfo { get; }    // server-only, populated once the ClientHello is received; stays populated until Dispose
    // Raw ClientHello bytes (record header + handshake message) captured on every server session.
    // Returned span is valid until Dispose. Throws InvalidOperationException on client sessions,
    // before the ClientHello has been received, or on server sessions where the
    // System.Net.Security.CaptureClientHello AppContext switch is disabled AND options were
    // supplied at TlsContext creation. Callers who need to persist the bytes should .ToArray().
    public ReadOnlySpan<byte> GetClientHelloBytes();
    public SslProtocols NegotiatedProtocol { get; }
    [CLSCompliant(false)]
    public TlsCipherSuite NegotiatedCipherSuite { get; }
    public SslApplicationProtocol NegotiatedApplicationProtocol { get; }
    public X509Certificate2? LocalCertificate { get; }
    public SafeSocketHandle? Socket { get; }               // null in sans-I/O mode

    // -------- Sans-I/O surface (mode 1) --------
    public TlsOperationStatus ProcessHandshake(ReadOnlySpan<byte> input, Span<byte> output,
                                               out int bytesConsumed, out int bytesWritten);
    public TlsOperationStatus Encrypt(ReadOnlySpan<byte> plaintext, Span<byte> ciphertext,
                                      out int bytesConsumed, out int bytesWritten);
    public TlsOperationStatus Decrypt(ReadOnlySpan<byte> ciphertext, Span<byte> plaintext,
                                      out int bytesConsumed, out int bytesWritten);
    public TlsOperationStatus Shutdown(Span<byte> ciphertext, out int bytesWritten);
    public TlsOperationStatus DrainPendingOutput(Span<byte> ciphertext, out int bytesWritten);
    public TlsOperationStatus RequestClientCertificate(Span<byte> ciphertext, out int bytesWritten);

    // -------- Socket-bound surface (mode 2) --------
    public TlsOperationStatus Handshake();
    public TlsOperationStatus Read(Span<byte> buffer, out int bytesRead);
    public TlsOperationStatus Write(ReadOnlySpan<byte> buffer, out int bytesWritten);

    // -------- Suspension points --------
    public X509Certificate2? GetRemoteCertificate();                              // leaf
    public X509Certificate2Collection? GetRemoteCertificates();                   // intermediates
    public SslPolicyErrors AcceptWithDefaultValidation();                         // built-in chain build + user callback
    public void SetRemoteCertificateValidationResult(SslPolicyErrors errors);     // caller-supplied verdict
    public void SetServerContext(TlsContext serverContext);                       // resolve NeedsServerOptions with a pre-built context
    public void SetClientCertificateContext(SslStreamCertificateContext context); // resolve WantCredentials
    public IReadOnlyList<string>? GetAcceptableIssuers();                         // CA hints from CertificateRequest
    public ChannelBinding? GetChannelBinding(ChannelBindingKind kind);            // tls-unique / tls-server-end-point

    public void Dispose();
}

using TlsContext ctx = TlsContext.Create(new SslServerAuthenticationOptions
{
    ServerCertificate = cert,
    EnabledSslProtocols = SslProtocols.Tls13,
    ApplicationProtocols = [SslApplicationProtocol.Http2, SslApplicationProtocol.Http11],
});

async Task HandleAsync(IDuplexPipe pipe)
{
    using TlsSession s = TlsSession.Create(ctx);
    byte[] scratch = ArrayPool<byte>.Shared.Rent(16 * 1024);

    while (!s.IsHandshakeComplete)
    {
        ReadResult rr = await pipe.Input.ReadAsync();
        SequencePosition consumedTo = rr.Buffer.Start;

        foreach (ReadOnlyMemory<byte> seg in rr.Buffer)
        {
            var status = s.ProcessHandshake(seg.Span, scratch, out int consumed, out int produced);
            consumedTo = rr.Buffer.GetPosition(consumed, consumedTo);
            if (produced > 0)
                await pipe.Output.WriteAsync(scratch.AsMemory(0, produced));

            if (status == TlsOperationStatus.NeedsCertificateValidation)
                s.AcceptWithDefaultValidation();

            if (status is TlsOperationStatus.WantRead or TlsOperationStatus.Complete)
                break;
        }
        pipe.Input.AdvanceTo(consumedTo);
    }

    Console.WriteLine($"Negotiated {s.NegotiatedProtocol} / ALPN={s.NegotiatedApplicationProtocol}");
    // ... app data via s.Decrypt / s.Encrypt ...
}

// Long-lived per-tenant contexts (typically cached keyed by SNI + options fingerprint).
ConcurrentDictionary<string, TlsContext> contextsBySni = LoadContexts();

// Bootstrap context: no cert baked in. Sessions are created against it so the
// ClientHello can be peeked managed-side; SetServerContext then adopts the per-tenant one.
using TlsContext bootstrap = TlsContext.Create((SslServerAuthenticationOptions?)null);
using TlsSession s = TlsSession.Create(bootstrap);

var status = s.ProcessHandshake(clientHello, scratch, out _, out int produced);
if (status == TlsOperationStatus.NeedsServerOptions)
{
    string? host = s.ClientHelloInfo?.ServerName;
    TlsContext ctx = contextsBySni.GetValueOrDefault(host ?? "") ?? contextsBySni["default"];
    s.SetServerContext(ctx);
    // resume handshake on next loop iteration
}

var status = s.ProcessHandshake(input, output, out _, out int produced);
if (status == TlsOperationStatus.NeedsCertificateValidation)
{
    using X509Certificate2? peer = s.GetRemoteCertificate();
    if (peer is not null && _revocationDirectory.IsRevoked(peer))
        s.SetRemoteCertificateValidationResult(SslPolicyErrors.RemoteCertificateChainErrors);
    else
        s.AcceptWithDefaultValidation();        // runs default chain build + user callback
}

// During app data: escalate to require a client cert.
byte[] outBuf = ArrayPool<byte>.Shared.Rent(2048);
s.RequestClientCertificate(outBuf, out int n);
await transport.WriteAsync(outBuf.AsMemory(0, n));

// Subsequent s.Decrypt(...) call will return NeedsCertificateValidation
// when the client's Certificate / CertificateVerify arrives.

Socket client = await listener.AcceptAsync();
client.Blocking = false;

using TlsContext ctx = TlsContext.Create(serverOptions);
using TlsSession s = TlsSession.Create(ctx, client.SafeHandle); // takes ownership

while (true)
{
    var status = s.Handshake();
    if (status == TlsOperationStatus.Complete) break;
    if (status == TlsOperationStatus.NeedsCertificateValidation) { s.AcceptWithDefaultValidation(); continue; }
    if (status is TlsOperationStatus.WantRead or TlsOperationStatus.WantWrite)
        await WaitOnReadinessAsync(client, status);   // epoll/kqueue/Task — caller's choice
}

byte[] buf = ArrayPool<byte>.Shared.Rent(16 * 1024);
while (s.Read(buf, out int n) == TlsOperationStatus.Complete && n > 0)
    s.Write(buf.AsSpan(0, n), out _);