One ring per reactor, one reactor per thread - run one per core. Every await -
an HTTP read, a Postgres query, a file read - is a completion on that ring, resumed inline
on the same thread. No thread pool on the hot path. The ring is the I/O.
Every await is a request placed on this core's queue, and your code
resumes where it left off when the kernel answers.
Each await parks on a reusable IValueTaskSource. The completion arrives,
the reactor calls SetResult, your code runs - same thread, no queue.
A Postgres query, a Redis command, an outbound HTTP call or a file read rides the same ring that accepted the request.
All of it runs: Playground is a complete server per file, grouped by topic.
Pick an example to see the code
// dotnet add package ioxide
// curl -ks --resolve alpha.test:8443:127.0.0.1 https://alpha.test:8443/
// curl -s --cacert alpha.test.pem --resolve alpha.test:8443:127.0.0.1 \
// https://alpha.test:8443/ # and this proves WHICH certificate came back
using System.Text;
using ioxide;
using ioxide.tls;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 8 * 1024; // response size
// A real PEM pair for the DEFAULT certificate, or null to generate a self-signed localhost one.
string? certOverride = null;
string? keyOverride = null;
// The names this port answers for, beside the default. Add a line and it is served.
string[] hosts = ["alpha.test", "beta.test"];
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var byHost = new Dictionary<string, TlsCertificate>();
foreach (string host in hosts)
{
(string hostCert, string hostKey) = ($"{host}.pem", $"{host}.key");
byHost[host] = new TlsCertificate { CertificatePath = hostCert, KeyPath = hostKey };
}
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/Sni for the QUIC side
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // the DEFAULT: no name asked for, or a name not below
KeyPath = keyPath,
Alpn = ["http/1.1"], // protocols this port serves, most-preferred first
// The table. Each entry is one host name and the certificate to answer it with; a host may
// give PEM text instead of paths (CertificatePem/KeyPem) for material kept out of the
// filesystem. Two entries for the same name, in any casing, are refused at startup rather
// than one of them silently never being served.
CertificatesByHost = byHost,
KernelTx = false, // kernel TLS transmit - see Tls/Ktls
KernelRx = false, // kTLS receive (experimental; requires KernelTx)
};
byte[] body = new byte[bodyBytes];
ReadOnlySpan<byte> fill = "ioxide-sni-payload "u8;
for (int i = 0; i < bodyBytes; i++)
{
body[i] = fill[i % fill.Length];
}
byte[] response =
[
.. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {bodyBytes}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
// One TlsService per reactor: it owns the contexts - the default and one per name - and drives
// handshakes on this ring.
reactor.OnStart = r => TlsService.Start(r, tlsOptions);
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
try
{
// The certificate is chosen inside this handshake, from the name the client sent.
// Nothing after it changes: the session reads and writes the same either way.
tls = await r.GetService<TlsService>()!.AcceptAsync(conn);
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
if (snapshot.IsClosed)
{
return;
}
tls.Write(conn, response);
await conn.FlushAsync();
conn.ResetRead();
}
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { IsBackground = false, Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"tls-sni on https://127.0.0.1:{port}/ default + {string.Join(", ", hosts)}");
foreach (Thread t in threads)
{
t.Join();
}One port, several names, a certificate each - chosen inside the handshake, before a byte of your protocol exists. The certificate at the top stays the DEFAULT: a client that sends no name (anything connecting by IP) or asks for one that is not registered gets it rather than a dead connection. Each entry is one OpenSSL context built at startup, so choosing one per handshake is a lookup that allocates nothing - ten names cost what one does. The same feature on h2 is , and on QUIC .
// dotnet add package ioxide
// curl -k --cert client.pem --key client.key https://127.0.0.1:8443/
using System.Buffers;
using System.IO.Pipelines;
using System.Text;
using ioxide;
using ioxide.tls;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 8 * 1024; // TLS cost is per-byte, so a 2-byte "ok" would hide it
// The server's own certificate and key. Null generates a self-signed pair on first run.
string? certOverride = null;
string? keyOverride = null;
// The CA that CLIENT certificates are checked against - the switch that turns mTLS on.
string? clientCaPath = Environment.GetEnvironmentVariable("PLAYGROUND_CLIENT_CA");
// Refuse a client offering no certificate, during the handshake. Off by default: see the header.
bool requireClientCertificate = Environment.GetEnvironmentVariable("PLAYGROUND_REQUIRE_CLIENT_CERT") == "1";
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor.
bool incrementalBuffers = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
if (clientCaPath is null)
{
Console.Error.WriteLine(
"set PLAYGROUND_CLIENT_CA to a PEM bundle of the CA that signs your client certificates.");
return 1;
}
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null,
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/*
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory)
KeyPath = keyPath, // PEM key file (or KeyPem for in-memory)
Alpn = ["http/1.1"], // protocols this port serves, most-preferred first
ClientCaPath = clientCaPath, // trust anchors for CLIENT certificates (or ClientCaPem)
RequireClientCertificate = requireClientCertificate, // refuse a client with none, at the handshake
// KernelTx stays false (the default). OpenSSL encrypts and decrypts, so nothing here needs the
// 'tls' kernel module. Client verification is unaffected by that choice either way - the
// certificate is exchanged during the handshake, which OpenSSL always performs.
KernelTx = false,
KernelRx = false, // kTLS receive (experimental; requires KernelTx)
};
byte[] body = new byte[bodyBytes];
"ioxide-mtls "u8.CopyTo(body);
for (int i = "ioxide-mtls "u8.Length; i < bodyBytes; i++)
{
body[i] = (byte)('a' + (i % 26));
}
// What an unauthenticated peer gets. Only reachable with RequireClientCertificate off - with it on
// that client never completed a handshake, so nothing here ever runs for it.
const string denied = "client certificate required";
byte[] forbidden = Encoding.ASCII.GetBytes(
$"HTTP/1.1 403 Forbidden\r\nContent-Length: {denied.Length}\r\n\r\n{denied}");
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r => TlsService.Start(r, tlsOptions);
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
try
{
tls = await r.GetService<TlsService>().AcceptAsync(conn);
// Who connected. Null means the peer offered no certificate, which only happens when
// RequireClientCertificate is off - a certificate that failed to verify never reaches
// here, because that fails the handshake.
//
// This is the whole point of the feature: enforcing an identity is half of it, and a
// server that can only enforce cannot authorise.
byte[] response = tls.PeerSubject is { } subject
? [.. Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Length: {bodyBytes}\r\nX-Client: {subject}\r\n\r\n"),
.. body]
: forbidden;
await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);
while (true)
{
ReadResult read = await pipe.Input.ReadAsync();
// Answer per REQUEST, not per read. TLS hands back RECORDS, so one request split
// across two records would otherwise draw two responses.
int answered = 0;
SequencePosition consumed = read.Buffer.Start;
var reader = new SequenceReader<byte>(read.Buffer);
while (reader.TryReadTo(out ReadOnlySequence<byte> _, "\r\n\r\n"u8, advancePastDelimiter: true))
{
consumed = reader.Position;
answered++;
}
// Consumed only whole requests; examined everything, so a partial head parks
// until more arrives instead of spinning on the same bytes.
pipe.Input.AdvanceTo(consumed, read.Buffer.End);
for (int n = 0; n < answered; n++)
{
// The identity is a property of the CONNECTION, so it is the same for every
// request on it - decided once, above the loop, not per request.
pipe.Output.Write(response);
}
if (answered > 0)
{
await pipe.Output.FlushAsync();
}
if (read.IsCompleted || read.IsCanceled)
{
return;
}
}
}
catch (Exception e)
{
Console.Error.WriteLine($"[tls-mtls-openssl-pipes] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[tls-mtls-openssl-pipes] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"client CA {clientCaPath}, "
+ $"{(requireClientCertificate ? "certificate REQUIRED" : "certificate optional")}, tx=openssl");
foreach (Thread thread in threads)
{
thread.Join();
}
return 0;The client proves who it is too, and the handler is told which peer it got. Everything below the identity check is the ordinary pipe server.
// dotnet add package ioxide
// sudo modprobe tls
// curl -k --cert client.pem --key client.key https://127.0.0.1:8443/
using System.Buffers;
using System.IO.Pipelines;
using System.Text;
using ioxide;
using ioxide.tls;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 8 * 1024; // TLS cost is per-byte, so a 2-byte "ok" would hide it
// The server's own certificate and key. Null generates a self-signed pair on first run.
string? certOverride = null;
string? keyOverride = null;
// The CA that CLIENT certificates are checked against - the switch that turns mTLS on.
string? clientCaPath = Environment.GetEnvironmentVariable("PLAYGROUND_CLIENT_CA");
// Refuse a client offering no certificate, during the handshake. Off by default: see the header.
bool requireClientCertificate = Environment.GetEnvironmentVariable("PLAYGROUND_REQUIRE_CLIENT_CERT") == "1";
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor.
bool incrementalBuffers = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
if (clientCaPath is null)
{
Console.Error.WriteLine(
"set PLAYGROUND_CLIENT_CA to a PEM bundle of the CA that signs your client certificates.");
return 1;
}
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null,
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/*
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory)
KeyPath = keyPath, // PEM key file (or KeyPem for in-memory)
Alpn = ["http/1.1"], // protocols this port serves, most-preferred first
ClientCaPath = clientCaPath, // trust anchors for CLIENT certificates (or ClientCaPem)
RequireClientCertificate = requireClientCertificate, // refuse a client with none, at the handshake
// The one line that differs from Playground/Tls/MtlsOpenSslPipes. The kernel encrypts outbound
// records, which needs `sudo modprobe tls` and pins this port to TLS 1.3, one ciphersuite, and
// no session resumption.
//
// It does NOT change anything above it. The client certificate is exchanged and verified during
// the HANDSHAKE, which OpenSSL performs either way - the kernel only takes over record crypto
// afterwards - so ClientCaPath, RequireClientCertificate and PeerSubject behave identically.
KernelTx = true,
KernelRx = false, // kTLS receive (experimental; requires KernelTx)
};
byte[] body = new byte[bodyBytes];
"ioxide-mtls "u8.CopyTo(body);
for (int i = "ioxide-mtls "u8.Length; i < bodyBytes; i++)
{
body[i] = (byte)('a' + (i % 26));
}
// What an unauthenticated peer gets. Only reachable with RequireClientCertificate off - with it on
// that client never completed a handshake, so nothing here ever runs for it.
const string denied = "client certificate required";
byte[] forbidden = Encoding.ASCII.GetBytes(
$"HTTP/1.1 403 Forbidden\r\nContent-Length: {denied.Length}\r\n\r\n{denied}");
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r => TlsService.Start(r, tlsOptions);
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
try
{
tls = await r.GetService<TlsService>().AcceptAsync(conn);
// Who connected. Null means the peer offered no certificate, which only happens when
// RequireClientCertificate is off - a certificate that failed to verify never reaches
// here, because that fails the handshake.
//
// This is the whole point of the feature: enforcing an identity is half of it, and a
// server that can only enforce cannot authorise.
byte[] response = tls.PeerSubject is { } subject
? [.. Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Length: {bodyBytes}\r\nX-Client: {subject}\r\n\r\n"),
.. body]
: forbidden;
await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);
while (true)
{
ReadResult read = await pipe.Input.ReadAsync();
// Answer per REQUEST, not per read. TLS hands back RECORDS, so one request split
// across two records would otherwise draw two responses.
int answered = 0;
SequencePosition consumed = read.Buffer.Start;
var reader = new SequenceReader<byte>(read.Buffer);
while (reader.TryReadTo(out ReadOnlySequence<byte> _, "\r\n\r\n"u8, advancePastDelimiter: true))
{
consumed = reader.Position;
answered++;
}
// Consumed only whole requests; examined everything, so a partial head parks
// until more arrives instead of spinning on the same bytes.
pipe.Input.AdvanceTo(consumed, read.Buffer.End);
for (int n = 0; n < answered; n++)
{
// The identity is a property of the CONNECTION, so it is the same for every
// request on it - decided once, above the loop, not per request.
pipe.Output.Write(response);
}
if (answered > 0)
{
await pipe.Output.FlushAsync();
}
if (read.IsCompleted || read.IsCanceled)
{
return;
}
}
}
catch (Exception e)
{
Console.Error.WriteLine($"[tls-mtls-ktls-pipes] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[tls-mtls-ktls-pipes] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"client CA {clientCaPath}, "
+ $"{(requireClientCertificate ? "certificate REQUIRED" : "certificate optional")}, tx=kernel");
foreach (Thread thread in threads)
{
thread.Join();
}
return 0;The same mutual TLS with the kernel encrypting outbound records. Verifying the peer and choosing who does the crypto are independent choices.
// dotnet add package ioxide
// dotnet add package ioxide.http2
// curl -k --http2 --resolve alpha.test:8443:127.0.0.1 https://alpha.test:8443/
using ioxide;
using ioxide.http2;
using ioxide.tls;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// A real PEM pair for the DEFAULT certificate, or null to generate a self-signed localhost one.
string? certOverride = null;
string? keyOverride = null;
// The names this port answers for, beside the default. Add a line and it is served.
string[] hosts = ["alpha.test", "beta.test"];
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var byHost = new Dictionary<string, TlsCertificate>();
foreach (string host in hosts)
{
(string hostCert, string hostKey) = ($"{host}.pem", $"{host}.key");
byHost[host] = new TlsCertificate { CertificatePath = hostCert, KeyPath = hostKey };
}
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/Sni for that side
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // the DEFAULT: no name asked for, or a name not below
KeyPath = keyPath,
Alpn = ["h2"], // h2 only here; add "http/1.1" to serve both, as Http2/Tls does
// The table. One entry per host name, and the certificate to answer it with; an entry may
// give PEM text instead of paths (CertificatePem/KeyPem) for material kept off the filesystem.
// Two entries for the same name, in any casing, are refused at startup rather than one of them
// silently never being served. Names are ASCII host names in full - SNI carries no port, an IP
// address is not a legal value, and an international name belongs here in its xn-- form.
CertificatesByHost = byHost,
KernelTx = false, // kernel TLS transmit - see Tls/Ktls
KernelRx = false, // kTLS receive (experimental; requires KernelTx)
};
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
// One TlsService per reactor. It owns the contexts - the default and one per name - and picks
// between them during each handshake on this ring. Each entry costs one OpenSSL context, built
// once at startup; choosing one per handshake is a lookup that allocates nothing, so serving
// ten names costs what serving one does.
reactor.OnStart = r => TlsService.Start(r, tlsOptions);
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
try
{
// The certificate is chosen inside this call, from the name the client sent. Nothing
// after it changes: the HTTP/2 code below is what the h2c sample runs.
tls = await r.GetService<TlsService>()!.AcceptAsync(conn);
await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);
await new Http2Connection(pipe).RunBufferedAsync(request =>
{
// :authority is HTTP/2's Host header, and this is where site routing belongs -
// per request, not per connection. It is NOT proof of which certificate was
// served; pin one with --cacert for that.
string authority = System.Text.Encoding.ASCII.GetString(request.Authority.Span);
return Http2Response.Text(authority.Length == 0
? "no :authority\n"
: $"site: {authority}\n");
});
}
catch (Exception e)
{
Console.Error.WriteLine($"[http2-sni] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http2-sni] {config.ReactorCount} reactors on :{port}, ALPN h2, "
+ $"default {certPath} + {string.Join(", ", hosts)}");
foreach (Thread thread in threads)
{
thread.Join();
}Two different names, at two different layers. SNI in the handshake picks the CERTIFICATE; :authority on each request picks the SITE. Nothing makes them agree - HTTP/2 lets a client reuse one connection for any origin the certificate covers, so authorization belongs on :authority, which is per request, and never on the name that chose the certificate.
// dotnet add package ioxide
// dotnet add package ioxide.http2
// kill -HUP <pid> # what an ACME hook sends after rewriting the PEM
// curl -s --cacert alpha.test.pem --resolve alpha.test:8443:127.0.0.1 \
// https://alpha.test:8443/ # succeeds before the rotation, fails after
using ioxide;
using ioxide.http2;
using ioxide.tls;
using System.Runtime.InteropServices;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// Seconds between automatic rotations, so the sample shows its own feature. 0 = only on SIGHUP,
// which is the shape a real deployment has: the ACME hook writes the PEM, then signals.
int rotateEverySeconds = 10;
// The names this port answers for, beside the default. Every one of them rotates.
string[] hosts = ["alpha.test", "beta.test"];
// ─────────────────────────────────────────────────────────────────────────────────────────────
// Two generations of the same identity: same subject, same SAN, different key and serial. This is
// what a renewal produces. The originals are what the server starts on.
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
(string renewedCert, string renewedKey) = ("localhost-renewed.pem", "localhost-renewed.key");
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/Rotate for that side
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
// The two sets, resolved once. Each is a whole generation - the default certificate and every
// name - and a rotation swaps one for the other. Only the PATHS are held here: ReplaceCertificates
// re-reads the files, which is what makes "same path, new contents" the normal ACME shape.
var originalTable = new Dictionary<string, TlsCertificate>();
var renewedTable = new Dictionary<string, TlsCertificate>();
foreach (string host in hosts)
{
(string hostCert, string hostKey) = ($"{host}.pem", $"{host}.key");
(string hostRenewedCert, string hostRenewedKey) = ($"{host}-renewed.pem", $"{host}-renewed.key");
originalTable[host] = new TlsCertificate { CertificatePath = hostCert, KeyPath = hostKey };
renewedTable[host] = new TlsCertificate { CertificatePath = hostRenewedCert, KeyPath = hostRenewedKey };
}
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // the DEFAULT: no name asked for, or a name not in the table
KeyPath = keyPath,
Alpn = ["h2"], // what a rotation may NOT change - see below
CertificatesByHost = originalTable,
KernelTx = false, // kernel TLS transmit - see Tls/Ktls
KernelRx = false, // kTLS receive (experimental; requires KernelTx)
};
// One service per reactor, each published by that reactor as it starts. Rotation reads this array
// from another thread, so the writes are volatile and a slot that is still null is simply skipped:
// a reactor that has not started yet has not served a handshake either.
var services = new TlsService?[config.ReactorCount];
int generation = 0;
object rotationGate = new();
void Rotate(string trigger)
{
// SIGHUP and the timer can land together, and the pair of "which generation" and "publish it"
// has to be atomic between them. ReplaceCertificates takes its own lock; this one is about
// the decision, not the publish.
lock (rotationGate)
{
int next = Interlocked.Increment(ref generation);
bool renewed = next % 2 == 1;
var certificate = renewed
? new TlsCertificate { CertificatePath = renewedCert, KeyPath = renewedKey }
: new TlsCertificate { CertificatePath = certPath, KeyPath = keyPath };
Dictionary<string, TlsCertificate> table = renewed ? renewedTable : originalTable;
int rotated = 0;
for (int i = 0; i < services.Length; i++)
{
TlsService? service = Volatile.Read(ref services[i]);
if (service is null)
{
continue; // that reactor has not started; it will start on the current set
}
try
{
// The whole set, every time. Handing this the default alone would empty the table.
service.ReplaceCertificates(certificate, table);
rotated++;
}
catch (Exception e)
{
// This reactor kept the certificates it had, and is still serving. Say so and
// carry on - a renewal that half-failed must not take the server with it.
Console.Error.WriteLine($"[http2-rotate] reactor {i} kept its certificates: {e.Message}");
}
}
Console.WriteLine($"[http2-rotate] generation {next} ({trigger}): {rotated}/{services.Length} reactors "
+ $"now serving {(renewed ? "renewed" : "original")} - pin "
+ $"{table[hosts[0]].CertificatePath}");
}
}
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
int id = i;
var reactor = new Reactor(i, config);
reactor.OnStart = r => Volatile.Write(ref services[id], TlsService.Start(r, tlsOptions));
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
try
{
// Whichever generation is published at this instant. A connection that is already up
// keeps what it handshook with.
tls = await r.GetService<TlsService>()!.AcceptAsync(conn);
await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);
await new Http2Connection(pipe).RunBufferedAsync(_ =>
{
// What the SERVER believes it published. The client's view is the certificate it
// just validated, and only pinning with --cacert can tell you that.
int served = Volatile.Read(ref generation);
return Http2Response.Text($"generation {served} ({(served % 2 == 1 ? "renewed" : "original")})\n");
});
}
catch (Exception e)
{
Console.Error.WriteLine($"[http2-rotate] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
// SIGHUP is what an ACME hook sends after writing the PEM. Cancel = true keeps the default action
// - terminating the process - from running after the handler.
using var sighup = PosixSignalRegistration.Create(PosixSignal.SIGHUP, context =>
{
context.Cancel = true;
Rotate("SIGHUP");
});
using var timer = rotateEverySeconds > 0
? new Timer(_ => Rotate("timer"), null, rotateEverySeconds * 1000, rotateEverySeconds * 1000)
: null;
Console.WriteLine($"[http2-rotate] pid {Environment.ProcessId}, {config.ReactorCount} reactors on :{port}, ALPN h2, "
+ $"default + {string.Join(", ", hosts)}, "
+ $"rotating {(rotateEverySeconds > 0 ? $"every {rotateEverySeconds}s and " : "")}on SIGHUP");
foreach (Thread thread in threads)
{
thread.Join();
}Replacing a certificate without dropping a connection. Three things decide whether an automated renewal is safe: it replaces the WHOLE set, so passing only the default leaves every name answered by the default certificate; a TlsService belongs to ONE reactor, so all of them have to be rotated and there is no instant at which they flip together; and it builds before it publishes, so a half-written PEM throws and leaves the service serving what it was. Live connections never notice - their certificate was chosen at handshake. Compare , where one shared engine makes it a single call.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet add package ioxide.http3
// curl --http3-only -k --resolve alpha.test:8443:127.0.0.1 https://alpha.test:8443/
using System.Text;
using ioxide;
using ioxide.http3;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443; // https://127.0.0.1:8443/ over QUIC
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
if (ushort.TryParse(Environment.GetEnvironmentVariable("PLAYGROUND_QUIC_PORT"), out ushort p))
{
quicPort = p;
}
if (int.TryParse(Environment.GetEnvironmentVariable("PLAYGROUND_REACTORS"), out int r) && r > 0)
{
reactors = r;
}
// A real PEM pair for the DEFAULT certificate, or null to generate a self-signed localhost one.
string? certOverride = null;
string? keyOverride = null;
// The names this port answers for, beside the default. Add a line and it is served.
string[] hosts = ["alpha.test", "beta.test"];
// UDP receive slots per reactor: how many datagrams the ring can have outstanding at once.
int udpRecvSlots = 16;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"]);
// PEM paths, because ngtcp2 loads certificates by path.
foreach (string host in hosts)
{
(string hostCert, string hostKey) = ($"{host}.pem", $"{host}.key");
engine.AddHost(host, hostCert, hostKey);
}
var config = new ServerConfig
{
ReactorCount = reactors,
Tcp = null, // QUIC only
Udp = new UdpOptions { RecvSlots = udpRecvSlots },
Quic = new QuicOptions
{
Port = quicPort,
LocalCidLength = 8,
// From here the host table is live and closed to further additions.
ConnectionFactory = engine.CreateFactory(),
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.QuicHandle = (_, connection) =>
new Http3Connection(connection).RunAsync(_ =>
{
// The certificate was chosen during the handshake, before this ever runs. Routing a
// request to the right site is a separate job, and the :authority header is where you
// would do it - the name in the handshake picked the certificate, nothing more.
var response = new Http3Response { Body = Encoding.UTF8.GetBytes("ok\n") };
response.Headers.Add(("content-type"u8.ToArray(), "text/plain"u8.ToArray()));
return response;
});
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http3-sni] {config.ReactorCount} reactors on :{quicPort}, "
+ $"default + {string.Join(", ", hosts)}");
foreach (Thread thread in threads)
{
thread.Join();
}The QUIC side, where TLS lives INSIDE the transport: no TlsService, one shared QuicEngine, and names registered before it starts serving - AddHost refuses afterwards, because the table is read during handshakes on every reactor at once. Client verification carries over to every host, so a name cannot be a way around the mutual TLS the engine was built with.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet add package ioxide.http3
// kill -HUP <pid> # what an ACME hook sends after rewriting the PEM
// curl --http3-only -s --cacert alpha.test.pem \
// --resolve alpha.test:8443:127.0.0.1 https://alpha.test:8443/
using System.Text;
using ioxide;
using ioxide.http3;
using ioxide.ngtcp2;
using System.Runtime.InteropServices;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443; // https://127.0.0.1:8443/ over QUIC
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// Seconds between automatic rotations, so the sample shows its own feature. 0 = only on SIGHUP,
// which is the shape a real deployment has: the ACME hook writes the PEM, then signals.
int rotateEverySeconds = 10;
// The names this port answers for, beside the default. Every one of them rotates.
string[] hosts = ["alpha.test", "beta.test"];
// UDP receive slots per reactor: how many datagrams the ring can have outstanding at once.
int udpRecvSlots = 16;
// ─────────────────────────────────────────────────────────────────────────────────────────────
// Two generations of the same identity: same subject, same SAN, different key and serial. This is
// what a renewal produces. The originals are what the server starts on.
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
(string renewedCert, string renewedKey) = ("localhost-renewed.pem", "localhost-renewed.key");
// ONE engine, shared by every reactor - this is the whole difference from Http2/Rotate.
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"]);
// Names are registered before CreateFactory, as they must be: once the engine is serving, AddHost
// refuses. Rotation is the supported way to change the table afterwards, and it replaces rather
// than edits, which is why it can be safe while AddHost cannot.
var originalTable = new Dictionary<string, QuicCertificate>();
var renewedTable = new Dictionary<string, QuicCertificate>();
foreach (string host in hosts)
{
(string hostCert, string hostKey) = ($"{host}.pem", $"{host}.key");
(string hostRenewedCert, string hostRenewedKey) = ($"{host}-renewed.pem", $"{host}-renewed.key");
engine.AddHost(host, hostCert, hostKey);
// The two sets, resolved once. Only the PATHS are held: ReplaceCertificates re-reads the
// files, which is what makes "same path, new contents" the normal ACME shape.
originalTable[host] = new QuicCertificate(hostCert, hostKey);
renewedTable[host] = new QuicCertificate(hostRenewedCert, hostRenewedKey);
}
var config = new ServerConfig
{
ReactorCount = reactors,
Tcp = null, // QUIC only
Udp = new UdpOptions { RecvSlots = udpRecvSlots },
Quic = new QuicOptions
{
Port = quicPort,
LocalCidLength = 8,
// From here the host table is live, and only ReplaceCertificates may change it.
ConnectionFactory = engine.CreateFactory(),
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
int generation = 0;
object rotationGate = new();
void Rotate(string trigger)
{
// SIGHUP and the timer can land together, so the pair of "which generation" and "publish it"
// is taken atomically here. The engine takes its own lock around the publish.
lock (rotationGate)
{
int next = Interlocked.Increment(ref generation);
bool renewed = next % 2 == 1;
var certificate = renewed
? new QuicCertificate(renewedCert, renewedKey)
: new QuicCertificate(certPath, keyPath);
Dictionary<string, QuicCertificate> table = renewed ? renewedTable : originalTable;
try
{
// One call, every reactor. The table goes with it because the set is replaced whole.
engine.ReplaceCertificates(certificate, table);
}
catch (Exception e)
{
// The engine is still serving the previous set, on every reactor. Nothing to undo.
Console.Error.WriteLine($"[http3-rotate] generation {next} refused, still serving the previous set: {e.Message}");
return;
}
Console.WriteLine($"[http3-rotate] generation {next} ({trigger}): serving "
+ $"{(renewed ? "renewed" : "original")} - pin "
+ $"{table[hosts[0]].CertificatePath}");
}
}
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.QuicHandle = (_, connection) =>
new Http3Connection(connection).RunAsync(_ =>
{
// What the SERVER believes it published. The client's view is the certificate it just
// validated, and only pinning with --cacert can tell you that.
int served = Volatile.Read(ref generation);
var response = new Http3Response
{
Body = Encoding.UTF8.GetBytes($"generation {served} ({(served % 2 == 1 ? "renewed" : "original")})\n"),
};
response.Headers.Add(("content-type"u8.ToArray(), "text/plain"u8.ToArray()));
return response;
});
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
// SIGHUP is what an ACME hook sends after writing the PEM. Cancel = true keeps the default action
// - terminating the process - from running after the handler.
using var sighup = PosixSignalRegistration.Create(PosixSignal.SIGHUP, context =>
{
context.Cancel = true;
Rotate("SIGHUP");
});
using var timer = rotateEverySeconds > 0
? new Timer(_ => Rotate("timer"), null, rotateEverySeconds * 1000, rotateEverySeconds * 1000)
: null;
Console.WriteLine($"[http3-rotate] pid {Environment.ProcessId}, {config.ReactorCount} reactors on :{quicPort}, "
+ $"default + {string.Join(", ", hosts)}, "
+ $"rotating {(rotateEverySeconds > 0 ? $"every {rotateEverySeconds}s and " : "")}on SIGHUP");
foreach (Thread thread in threads)
{
thread.Join();
}The same operational need with a different shape under it: ONE engine shared by every reactor, so a rotation is a single call and a handshake sees either every old certificate or every new one - never a mixture. The engine also REFUSES to replace the default alone while it answers for names, because that hook (renew the default, forget the rest) silently published an empty table. What a rotation may never change on either stack: the client trust anchors, and whether a client certificate is required.
// dotnet add package ioxide
// dotnet add package ioxide.Kestrel
// dotnet run # TRANSPORT=ioxide is the default
// TRANSPORT=sockets dotnet run # the stock Kestrel transport, for comparison
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using ioxide.Kestrel;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.Logging;
// A minimal ASP.NET Core app that runs on either the ioxide io_uring transport or Kestrel's stock
// sockets transport. Pick with the TRANSPORT environment variable (default: ioxide):
//
// TRANSPORT=ioxide dotnet run # ioxide.Kestrel transport (io_uring, one reactor per core)
// TRANSPORT=sockets dotnet run # stock Kestrel sockets transport (the framework default)
// TRANSPORT=h3 dotnet run # stock Kestrel + HTTP/3 over msquic on udp :8443 - the
// # ASP.NET twin of the ioxide `quic-h3` example, same port,
// # for side-by-side h2load/h3x runs. Needs libmsquic
// # (sudo apt install libmsquic). TLS is mandatory for h3,
// # so the endpoint rides a self-signed localhost cert.
//
// Then: curl http://localhost:8080/ and curl http://localhost:8080/plaintext
// h3: curl --http3-only -k https://localhost:8443/plaintext
// h2load --alpn-list=h3 -n 100000 -c 32 -m 32 https://127.0.0.1:8443/plaintext
var builder = WebApplication.CreateBuilder(args);
builder.Logging.ClearProviders(); // turn off all logging (no per-request info: lines)
var transport = (Environment.GetEnvironmentVariable("TRANSPORT") ?? "ioxide").Trim().ToLowerInvariant();
builder.WebHost.ConfigureKestrel(o => o.ListenAnyIP(8080));
switch (transport)
{
case "ioxide":
builder.WebHost.UseIoxide(o => o.ReactorCount = 16); // io_uring transport, 16 reactors (one ring per thread)
break;
case "sockets":
case "kestrel":
// Stock Kestrel sockets transport — the framework default, nothing to wire up.
break;
case "h3":
// HTTP/3 rides Kestrel's msquic multiplexed transport, so this mode is necessarily the
// stock stack (the ioxide transport is TCP-only). :8443 serves h1+h2 over TCP and h3
// over UDP on the same port (Alt-Svc advertises the upgrade); :8080 stays plain h1.
// Don't run this alongside the ioxide quic-h3 example - both bind udp :8443.
if (!System.Net.Quic.QuicListener.IsSupported)
{
Console.Error.WriteLine("[Examples.AspNet] warning: QUIC is not supported on this box " +
"(libmsquic missing? try: sudo apt install libmsquic) - :8443 will serve h1/h2 only");
}
builder.WebHost.ConfigureKestrel(o => o.ListenAnyIP(8443, ep =>
{
ep.Protocols = HttpProtocols.Http1AndHttp2AndHttp3;
ep.UseHttps(H3Cert());
}));
break;
default:
Console.Error.WriteLine($"Unknown TRANSPORT '{transport}'. Use 'ioxide', 'sockets' or 'h3'.");
return;
}
var app = builder.Build();
app.MapGet("/", () => $"Hello from ioxide.Kestrel! transport={transport}");
app.MapGet("/plaintext", () => "Hello, World!");
Console.WriteLine(transport == "h3"
? "[Examples.AspNet] listening on http://localhost:8080 and https://localhost:8443 (h1+h2+h3, msquic)"
: $"[Examples.AspNet] listening on http://localhost:8080 (transport={transport})");
app.Run();
return;
// Self-signed localhost cert for the h3 endpoint. The PKCS#12 round-trip re-imports the ephemeral
// private key in the shape the TLS stacks (msquic included) accept.
static X509Certificate2 H3Cert()
{
using var rsa = RSA.Create(2048);
var request = new CertificateRequest("CN=localhost", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
using var cert = request.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddYears(1));
return X509CertificateLoader.LoadPkcs12(cert.Export(X509ContentType.Pkcs12), null);
}Not an ioxide server: an ordinary ASP.NET Core app with ioxide underneath Kestrel as its transport. UseIoxide is the whole integration, and TRANSPORT= switches back to stock sockets so the same app can be measured both ways.
// dotnet add package ioxide
// curl http://127.0.0.1:8080/
// curl -ks https://127.0.0.1:8081/
using System.Buffers;
using System.IO.Pipelines;
using System.Text;
using ioxide;
using ioxide.tls;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080; // http://127.0.0.1:8080/ - the plaintext door
ushort tlsPort = 8081; // https://127.0.0.1:8081/ - the TLS door
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [tlsPort], // every reactor listens on both; ListenerPort says which one accepted
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
// Default backend: OpenSSL in both directions. Only connections on the TLS door go near this.
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory)
KeyPath = keyPath, // PEM key file (or KeyPem for in-memory)
Alpn = ["http/1.1"], // protocols this port serves, most-preferred first
KernelTx = false, // kTLS transmit offload (kernel makes the records)
KernelRx = false, // kTLS receive (experimental; requires KernelTx)
};
const string bodyText = "multiport-ok\n";
byte[] response = Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {bodyText.Length}\r\n\r\n{bodyText}");
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r => TlsService.Start(r, tlsOptions);
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
try
{
// Which door? This branch is the entire difference between the two.
if (conn.ListenerPort == tlsPort)
{
tls = await r.GetService<TlsService>().AcceptAsync(conn);
await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);
await ServeAsync(pipe, response);
}
else
{
var pipe = new PlainPipe(new TcpConnectionPipeReader(conn), new TcpConnectionPipeWriter(conn));
try
{
await ServeAsync(pipe, response);
}
finally
{
pipe.Input.Complete();
pipe.Output.Complete();
}
}
}
catch (Exception e)
{
Console.Error.WriteLine($"[tls-multiport] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[tls-multiport] {config.ReactorCount} reactors, "
+ $"plaintext :{config.Tcp!.Port}, tls :{tlsPort} (openssl both ways), cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}
// One loop for both doors. It reads an IDuplexPipe and nothing else - whether the bytes were
// decrypted on the way in, or will be encrypted on the way out, is not visible from here.
static async Task ServeAsync(IDuplexPipe pipe, ReadOnlyMemory<byte> response)
{
while (true)
{
ReadResult read = await pipe.Input.ReadAsync();
// Answer per REQUEST, not per read - a request split across reads must not draw two
// responses, and two requests in one read must not draw one.
int answered = 0;
SequencePosition consumed = read.Buffer.Start;
var reader = new SequenceReader<byte>(read.Buffer);
while (reader.TryReadTo(out ReadOnlySequence<byte> _, "\r\n\r\n"u8, advancePastDelimiter: true))
{
consumed = reader.Position;
answered++;
}
// Consumed only whole requests; examined everything, so a partial head parks until more
// arrives instead of spinning on the same bytes.
pipe.Input.AdvanceTo(consumed, read.Buffer.End);
for (int n = 0; n < answered; n++)
{
pipe.Output.Write(response.Span);
}
if (answered > 0)
{
await pipe.Output.FlushAsync();
}
if (read.IsCompleted || read.IsCanceled)
{
return;
}
}
}
// The plaintext door's IDuplexPipe: the connection's own reader and writer, paired.
sealed record PlainPipe(PipeReader Input, PipeWriter Output) : IDuplexPipe;One server, two doors: plaintext on :8080, TLS on :8081, and ONE serve loop for both. The branch on ListenerPort is the entire difference - the TLS door builds a TlsConnectionDualPipe, the plaintext door pairs the connection's own reader and writer, and the loop reads an IDuplexPipe without knowing which it got. Ports come from multi-port; TLS is the default backend, OpenSSL both ways.
// dotnet add package ioxide
// curl -k https://127.0.0.1:8443/
using System.Net.Security;
using System.Security.Authentication;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using ioxide;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 8 * 1024; // same body as Tls.Ktls, so the two compare directly
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var certificate = X509Certificate2.CreateFromPemFile(certPath, keyPath);
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
int bodySize = bodyBytes;
byte[] body = new byte[bodySize];
ReadOnlySpan<byte> fill = "ioxide-sslstream-payload "u8;
for (int i = 0; i < bodySize; i++)
{
body[i] = fill[i % fill.Length];
}
byte[] response =
[
.. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {bodySize}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
SslStream? ssl = null;
try
{
// The stream adapter reads from the ring and writes into the slab; SslStream never
// knows it isn't a NetworkStream.
ssl = new SslStream(new TcpConnectionStream(conn), leaveInnerStreamOpen: false);
await ssl.AuthenticateAsServerAsync(new SslServerAuthenticationOptions
{
ServerCertificate = certificate,
ApplicationProtocols = [SslApplicationProtocol.Http11],
EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13,
});
var request = new byte[8192];
while (true)
{
int n = await ssl.ReadAsync(request);
if (n == 0) return; // peer closed
await ssl.WriteAsync(response); // encrypted in userspace
}
}
catch (Exception e)
{
Console.Error.WriteLine($"[tls-sslstream] connection failed: {e.Message}");
}
finally
{
ssl?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[tls-sslstream] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"{bodySize}-byte body, cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}The third TLS backend, and the portable one: the BCL's SslStream over TcpConnectionStream. Fully managed, full-featured, and the slowest of the three, because the bytes are copied through a Stream both ways. Reach for it for client certificates - the ring-native path does not do mTLS - or anything else OpenSSL-on-the-ring does not expose; for TLS 1.2 and resumption the default already has you covered: .
// dotnet add package ioxide
// curl -s http://127.0.0.1:8080/ | wc -c
using System.Text;
using ioxide;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080; // http://127.0.0.1:8080/
int reactors = Environment.ProcessorCount;
int bodyBytes = 100 * 1024; // well past the small-payload regime
// IORING_OP_SEND_ZC instead of plain SEND. FlushAsync then completes on the F_NOTIF (the kernel
// released the buffer), so the slab is never recycled while the kernel still owns it. It only
// pays off once the body is large enough that pinning beats copying.
bool zeroCopy = false;
// Drop the slab below bodyBytes (try 16 * 1024) to force the overflow path every request.
int slabSize = 256 * 1024;
// What overflow does: Grow reallocates the slab in place, Segmented chains extra segments and
// sends them as one SENDMSG. Same bytes on the wire either way - checksum both and see.
WriteOverflowStrategy overflow = WriteOverflowStrategy.Grow;
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = slabSize, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = overflow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = zeroCopy, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
int bodySize = bodyBytes;
byte[] head = Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {bodySize}\r\n\r\n");
byte[] response = new byte[head.Length + bodySize];
head.CopyTo(response, 0);
for (int i = 0; i < bodySize; i++)
{
response[head.Length + i] = (byte)(i % 251);
}
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
try
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
conn.ReturnBuffer(in item);
}
}
// With ZC on, this await is the buffer-release notification, not just the
// submit - which is exactly why the next Write may safely reuse the slab.
conn.Write(response);
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[big] {config.ReactorCount} reactors on :{config.Tcp!.Port}, {bodySize}-byte body, "
+ $"slab={slabSize}, overflow={overflow}, zc={zeroCopy}");
foreach (Thread thread in threads)
{
thread.Join();
}The write path under a payload that does not fit the slab, and the knobs that shape it: WriteSlabSize, WriteOverflow (Grow reallocates, Segmented chains slabs into one SENDMSG) and ZeroCopySend, which only pays off once the response is large enough for the pinning to be worth it.
// dotnet add package ioxide
// curl http://127.0.0.1:8080/
using ioxide;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080; // http://127.0.0.1:8080/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
byte[] response = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\r\nok"u8.ToArray();
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
try
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
// Off to the thread pool and back. Everything after this line runs on the reactor
// again, because the per-reactor SynchronizationContext posts it home - which is
// what makes it safe to touch the connection below.
await Task.Yield();
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer) conn.ReturnBuffer(in item);
}
conn.Write(response);
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[hop] {config.ReactorCount} reactors on :{config.Tcp.Port} (every request bounces off-reactor)");
foreach (Thread thread in threads)
{
thread.Join();
}The same server as , except every request deliberately bounces off the reactor thread and back. It is here as the counter-example: this is what ioxide spends its design avoiding, and having it runnable makes the cost measurable rather than asserted.
// dotnet add package ioxide
// curl http://127.0.0.1:8080/
using System.Text;
using System.Text.Json;
using ioxide;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080; // http://127.0.0.1:8080/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
int offReactorSeen = 0;
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
try
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer) conn.ReturnBuffer(in item);
}
// Ordinary thread-pool work, awaited from a reactor handler.
string json = await Task.Run(static () => JsonSerializer.Serialize("hello world"));
// We should be back on the reactor here. If not, say so once.
if (!r.OnReactorThread && Interlocked.Exchange(ref offReactorSeen, 1) == 0)
{
Console.WriteLine("[taskrun] continuation resumed OFF the reactor (no sync context)");
}
conn.Write("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 13\r\n\r\n"u8);
conn.Write(Encoding.UTF8.GetBytes(json));
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[taskrun] {config.ReactorCount} reactors on :{config.Tcp.Port} (each request awaits a Task.Run)");
foreach (Thread thread in threads)
{
thread.Join();
}Proof that normal .NET async works inside a handler - Task.Run, Task.Delay, the thread pool - and that you come back to your reactor afterwards. The per-reactor SynchronizationContext is what makes that true, so connection and pool state stay single-threaded without a lock even when a handler wanders.
// dotnet add package ioxide
// dotnet add package ioxide.http2
// curl -k --http2 https://127.0.0.1:8443/
using System.IO.Pipelines;
using System.Text;
using ioxide;
using ioxide.http2;
using ioxide.tls;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 2; // "ok" - this sample is about ALPN, not throughput
// Hand OUTBOUND encryption to the kernel: the handler writes plaintext and the kernel makes the
// records. Off by default - OpenSSL both ways is the portable path, and on loopback the kernel
// is not faster. Its real payoff is sendfile and NIC offload, which a benchmark here cannot see.
bool kernelTx = false;
// Hand INBOUND decryption to the kernel as well. Requires kernelTx - the RX handoff happens at
// the same moment as the TX one - and is experimental for a reason: a TLS 1.3 KeyUpdate cannot be
// read through IORING_OP_RECV, and roughly one first connection in twelve fails outright.
bool kernelRx = false;
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
// handler code is identical either way; this only changes how recv buffers are handed out.
bool incrementalBuffers = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
byte[] body = bodyBytes == 2 ? "ok"u8.ToArray() : [.. Enumerable.Repeat((byte)'x', bodyBytes)];
byte[] http11Response =
[
.. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Length: {body.Length}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r => TlsService.Start(r, new TlsOptions
{
CertificatePath = certPath, // PEM certificate chain file (leaf first)
CertificatePem = null, // in-memory PEM alternative - set one, not both
KeyPath = keyPath, // PEM private key file
KeyPem = null, // in-memory PEM alternative to KeyPath
Alpn = ["h2", "http/1.1"], // ORDERED, most preferred first - a client offering both gets h2
KernelTx = kernelTx, // kTLS encrypt: kernel makes the records (off = OpenSSL both ways)
KernelRx = kernelRx, // kTLS decrypt: needs KernelTx; experimental (see the knob above)
});
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
try
{
tls = await r.GetService<TlsService>()!.AcceptAsync(conn);
if (tls.NegotiatedAlpn == "h2")
{
// The decrypt lives in the pipe, so the HTTP/2 code below is identical to the
// cleartext sample. Which halves the pipe uses is decided by what the handshake
// achieved, not by anything chosen here.
await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);
await new Http2Connection(pipe).RunBufferedAsync(_ => new Http2Response
{
Status = 200,
Body = body,
});
return;
}
// Anything else: HTTP/1.1 on the same port.
//
// The carry is not incidental. TLS hands back RECORDS, not requests, so a request
// split across two records decrypts twice - and answering on "plaintext arrived"
// would answer twice to one request. Framing is ours; ioxide does not parse HTTP.
var carry = new Carry();
// The client's first request usually rides in with its Finished flight, so the
// handshake already decrypted it and it is sitting in the session, not in any recv
// buffer. Miss this and that request is dropped and the loop parks on bytes that
// already arrived - which is exactly what happened here before.
carry.Append(tls.DrainPlaintext());
while (true)
{
bool wrote = false;
int end;
while ((end = carry.Span.IndexOf("\r\n\r\n"u8)) >= 0)
{
carry.Consume(end + 4);
// Correct whichever backend the session ended up with.
tls.Write(conn, http11Response);
wrote = true;
}
if (wrote)
{
await conn.FlushAsync();
}
RecvSnapshot snapshot = await conn.ReadAsync();
unsafe
{
while (conn.TryGetItem(snapshot, out ioxide.utils.SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
carry.Append(tls.Decrypt(item.Ptr, item.Len));
conn.ReturnBuffer(in item);
}
}
}
if (snapshot.IsClosed || tls.Closed) return;
conn.ResetRead();
}
}
catch (Exception e)
{
Console.Error.WriteLine($"[http2-tls] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http2-tls] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"ALPN h2 then http/1.1, cert {certPath}, "
+ $"rx={(kernelTx && kernelRx ? "kernel" : "openssl")}, "
+ $"tx={(kernelTx ? "kernel" : "openssl")}");
foreach (Thread thread in threads)
{
thread.Join();
}
// Decrypted-but-unframed bytes: append at the end, consume from the front. A List<byte> pressed
// into this job needs CollectionsMarshal to be searched and RemoveRange to be consumed; a plain
// array does both directly.
sealed class Carry
{
private byte[] _buf = new byte[8 * 1024];
private int _len;
public ReadOnlySpan<byte> Span => _buf.AsSpan(0, _len);
public void Append(ReadOnlySpan<byte> bytes)
{
if (_buf.Length - _len < bytes.Length)
{
Array.Resize(ref _buf, Math.Max(_buf.Length * 2, _len + bytes.Length));
}
bytes.CopyTo(_buf.AsSpan(_len));
_len += bytes.Length;
}
public void Consume(int count)
{
_buf.AsSpan(count, _len - count).CopyTo(_buf);
_len -= count;
}
}How a browser actually reaches h2: over TLS, with the protocol chosen during the handshake. Alpn = ["h2", "http/1.1"] is an ordered preference, not a weighting - the server takes the first entry the client also offered, and this sample then branches on what was agreed, so ONE port serves both. That is why the h2c samples are the exception rather than the rule. What Http2Connection is handed is a TlsConnectionDualPipe: it never learns TLS is involved, so the protocol code is byte-for-byte the h2c sample's. TLS is OpenSSL both ways by default; the kernelTx/kernelRx knobs at the top move either direction into the kernel, and cost the same 0.05µs per request either way.
// dotnet add package ioxide
// dotnet add package ioxide.http2
// curl -k --http2 https://127.0.0.1:8443/
using System.IO.Pipelines;
using System.Net.Security;
using System.Security.Authentication;
using System.Security.Cryptography.X509Certificates;
using ioxide;
using ioxide.http2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var certificate = X509Certificate2.CreateFromPemFile(certPath, keyPath);
certificate = X509CertificateLoader.LoadPkcs12(certificate.Export(X509ContentType.Pfx), null);
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
byte[] body = "ok"u8.ToArray();
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
SslStream? ssl = null;
try
{
ssl = new SslStream(new TcpConnectionStream(conn), leaveInnerStreamOpen: false);
await ssl.AuthenticateAsServerAsync(new SslServerAuthenticationOptions
{
ServerCertificate = certificate,
// Same ordered preference as the kTLS sample: h2 wins when the client offers both.
ApplicationProtocols = [SslApplicationProtocol.Http2, SslApplicationProtocol.Http11],
EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13,
});
if (ssl.NegotiatedApplicationProtocol != SslApplicationProtocol.Http2)
{
return; // this sample only serves h2; see Playground/Http2/Tls for the fallback
}
await new Http2Connection(new StreamDuplexPipe(ssl)).RunBufferedAsync(
_ => new Http2Response { Status = 200, Body = body });
}
catch (Exception e)
{
Console.Error.WriteLine($"[http2-sslstream] connection failed: {e.Message}");
}
finally
{
ssl?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http2-sslstream] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"h2 over SslStream (userspace both ways), cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}
/// <summary>
/// Any Stream as an IDuplexPipe. The BCL already has the two halves - this only pairs them, which
/// is the whole adapter needed to run ioxide's HTTP/2 over something that is not a ring connection.
/// </summary>
internal sealed class StreamDuplexPipe(Stream stream) : IDuplexPipe
{
public PipeReader Input { get; } = PipeReader.Create(stream, new StreamPipeReaderOptions(leaveOpen: true));
public PipeWriter Output { get; } = PipeWriter.Create(stream, new StreamPipeWriterOptions(leaveOpen: true));
}HTTP/2 over the BCL's SslStream, and the point is the ten-line Stream-to-IDuplexPipe adapter at the bottom. Nghttp2Connection takes a pipe, so the same HTTP/2 code runs over the ring directly, over ioxide's TLS, or over SslStream - the transport is a constructor argument, not a branch inside the protocol.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet add package ioxide.http3
// curl --http3-only -k https://127.0.0.1:8443/
using ioxide;
using ioxide.http3;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443; // https://127.0.0.1:8443/ over UDP
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// Response body size. 13 is "Hello, World!"; anything else is that many 'x'.
int bodyBytes = 13;
// UDP receive slots per reactor: how many datagrams the ring can have outstanding at once.
int udpRecvSlots = 16;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"]);
var config = new ServerConfig
{
ReactorCount = reactors,
Tcp = null, // QUIC only
Udp = new UdpOptions { RecvSlots = udpRecvSlots },
Quic = new QuicOptions
{
Port = quicPort,
LocalCidLength = 8,
ConnectionFactory = engine.CreateFactory(),
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
// Built once and reused: the h3 layer copies status, headers and body at submit and never retains
// the object, so a hot path should not rebuild it per request.
var response = new Http3Response
{
Body = bodyBytes == 13 ? "Hello, World!"u8.ToArray() : [.. Enumerable.Repeat((byte)'x', bodyBytes)],
};
response.Headers.Add(("content-type"u8.ToArray(), "text/plain"u8.ToArray()));
response.Headers.Add(("server"u8.ToArray(), "ioxide"u8.ToArray()));
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.QuicHandle = (r, conn) => new Http3Connection(conn).RunAsync(_ => response);
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http3-managed-buffered] {config.ReactorCount} reactors, h3 on udp :{config.Quic!.Port} "
+ $"(pure C#), {bodyBytes}-byte body, cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}HTTP/3 with no native library above the transport - frames, QPACK and Huffman are all managed code, and only QUIC itself stays native. Drop-in for : the diff is the package and three type names. It is also faster at every size measured on this rig - 2 reactors, h3x --connections 16 -m 8: 1.54× at a 13-byte body, 1.28× at 50 KiB, 1.32× at 1 MiB, using less memory at the large end. The small-body lead came from sending the header frame and a short body in ONE call rather than two; the large-body lead is the native shim copying every response body at submit, which this never does.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet add package ioxide.nghttp3
// curl --http3-only -k https://127.0.0.1:8443/
// curl --http3-only -kN https://127.0.0.1:8443/feed # never ends
using System.Text;
using ioxide;
using ioxide.nghttp3;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443; // https://127.0.0.1:8443/ over UDP
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// Chunks written per response on "/", and the size of each. The point of the sample is that the
// product of these two is never held in memory at once.
int chunkCount = 64;
int chunkBytes = 4 * 1024;
// UDP receive slots per reactor: how many datagrams the ring can have outstanding at once.
int udpRecvSlots = 16;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"]);
var config = new ServerConfig
{
ReactorCount = reactors,
Tcp = null, // QUIC only: no TCP listener is bound
Udp = new UdpOptions { RecvSlots = udpRecvSlots },
Quic = new QuicOptions
{
Port = quicPort,
LocalCidLength = 8,
ConnectionFactory = engine.CreateFactory(),
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
byte[] chunk = Encoding.ASCII.GetBytes(new string('x', chunkBytes - 1) + "\n");
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.QuicHandle = (r, conn) =>
new Nghttp3Connection(conn).RunStreamedResponseAsync(async (request, writer) =>
{
bool endless = request.Path.Span.SequenceEqual("/feed"u8);
// Headers first and once: HTTP/3 puts HEADERS before DATA and there is no correcting
// it later. No Content-Length here - the length is not known yet, and for /feed never
// will be.
var response = new Nghttp3Response { Status = 200 };
response.Headers.Add("content-type"u8.ToArray(),
endless ? "text/event-stream"u8.ToArray() : "text/plain"u8.ToArray());
writer.WriteHeaders(response);
for (int n = 0; endless || n < chunkCount; n++)
{
chunk.CopyTo(writer.GetSpan(chunk.Length));
writer.Advance(chunk.Length);
// Returns once nghttp3 has taken it. That await IS the backpressure - nothing
// queues up behind a peer that has stopped reading.
await writer.FlushAsync();
}
// CompleteAsync ends the stream. The runner calls it too if a handler returns without
// doing so, since the peer is owed an end either way.
await writer.CompleteAsync();
});
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[h3-streamed] {config.ReactorCount} reactors, h3 on udp :{config.Quic!.Port}, "
+ $"{chunkCount} x {chunkBytes}-byte chunks per response, cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}The response body produced OVER TIME instead of handed over whole - each flush becomes a DATA frame. That is what /feed demonstrates: an endless response has no final byte, so a buffered API cannot express it at all. Nghttp3ResponseWriter is an IBufferWriter<byte>, so a serializer or a framework's response sink writes into it unchanged, and FlushAsync returning only once nghttp3 has taken the chunk is what stops a producer outrunning a peer that has stopped reading. nghttp3 PULLS body bytes rather than accepting pushes, which is why this carries a resume and a drain the pure-C# writer does not need.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet add package ioxide.nghttp3
// curl --http3-only -k https://127.0.0.1:8443/
// curl --http3-only -k --data-binary @big.bin https://127.0.0.1:8443/echo
using System.Text;
using ioxide;
using ioxide.nghttp3;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443;
int reactors = Environment.ProcessorCount;
// Chunks written per response on "/", and the size of each. Their product is never held at once.
int chunkCount = 64;
int chunkBytes = 16 * 1024;
// Multishot recv slots per reactor. QPACK capacity 4096 advertises a decode-side dynamic table;
// 0 is static-only, nghttp3's default.
int udpRecvSlots = 16;
long qpackCapacity = 0;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
// The last argument bounds what one connection may retain unacknowledged, which is what keeps a
// streamed response streaming instead of quietly buffering whole. See Playground/Http3/Nghttp3Buffered
// for the full QUIC/h3 knob set.
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"], maxSendRetentionBytes: 16L << 20);
var config = new ServerConfig
{
ReactorCount = reactors,
Tcp = null, // QUIC only: no TCP listener is bound
Udp = new UdpOptions { RecvSlots = udpRecvSlots },
Quic = new QuicOptions
{
Port = quicPort,
LocalCidLength = 8,
ConnectionFactory = engine.CreateFactory(),
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
var h3Options = new Nghttp3Options
{
QpackDynamicTableCapacity = qpackCapacity, // 0 (default) = headers stay literal
QpackBlockedStreams = qpackCapacity > 0 ? 100 : 0, // raise both together for the dynamic table
};
byte[] chunk = Encoding.ASCII.GetBytes(new string('x', chunkBytes - 1) + "\n");
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.QuicHandle = (r, conn) =>
new Nghttp3Connection(conn, h3Options).RunStreamedResponseAsync(async (request, writer) =>
{
bool upload = request.Path.Span.SequenceEqual("/upload"u8);
bool echo = request.Path.Span.SequenceEqual("/echo"u8);
if (echo)
{
// Both directions at once, which is the shape a proxy needs: read a chunk, write
// a chunk, and never hold more than one. Neither side can run away from the other
// - ReadAsync waits on the peer, FlushAsync waits on nghttp3 - so memory stays
// flat however large the exchange is.
writer.WriteHeaders(Plain());
while (true)
{
ReadOnlyMemory<byte> part = await request.BodyReader!.ReadAsync();
if (part.IsEmpty)
{
break; // end of the request body
}
part.Span.CopyTo(writer.GetSpan(part.Length));
writer.Advance(part.Length);
await writer.FlushAsync();
}
await writer.CompleteAsync();
return;
}
if (upload)
{
// Read side only: pull the body a chunk at a time rather than waiting for all of
// it, so memory is bound by one chunk however large the upload is. Every read
// credits the peer's flow-control window, which is what throttles a fast sender.
long total = 0;
while (true)
{
ReadOnlyMemory<byte> part = await request.BodyReader!.ReadAsync();
if (part.IsEmpty) break;
total += part.Length; // a real app would parse or store the chunk here
}
writer.WriteHeaders(Plain());
byte[] count = Encoding.ASCII.GetBytes($"{total}\n");
count.CopyTo(writer.GetSpan(count.Length));
writer.Advance(count.Length);
await writer.FlushAsync();
await writer.CompleteAsync();
return;
}
// Headers first and once: HTTP/3 puts HEADERS before DATA and there is no correcting
// it later. No content-length - the length is not known when they go out. A GET
// arrives with an already-ended body reader, so there is nothing to drain.
writer.WriteHeaders(Plain());
for (int n = 0; n < chunkCount; n++)
{
chunk.CopyTo(writer.GetSpan(chunk.Length));
writer.Advance(chunk.Length);
// Returns once nghttp3 has taken the chunk. That await IS the backpressure -
// nothing queues up behind a peer that has stopped reading.
await writer.FlushAsync();
}
await writer.CompleteAsync();
});
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[nghttp3-streamed-both] {config.ReactorCount} reactors, h3 on udp :{config.Quic!.Port}, "
+ $"{chunkCount} x {chunkBytes}-byte chunks per response, cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}
static Nghttp3Response Plain()
{
var response = new Nghttp3Response { Status = 200 };
response.Headers.Add("content-type"u8.ToArray(), "text/plain"u8.ToArray());
return response;
}The fourth corner the other three nghttp3 samples leave empty: the request pulled through BodyReader while the response is pushed through the writer, both in one handler. One call arranges it - RunStreamedResponseAsync dispatches at end-of-headers, so the handler is running while the upload is still on the wire. /echo is the shape a proxy needs: read a chunk, write a chunk, never hold more than one, and neither side can outrun the other because ReadAsync waits on the peer and FlushAsync waits on nghttp3. Measured here on a 64 MiB echo: byte-identical out, and the server's RSS moved 51→57 MB - flat, in the sense that matters. Diff it against : same routes, opposite mechanism underneath. nghttp3 owns the framing and pulls body bytes when it has room to emit DATA, so a flush here means nghttp3 has taken the chunk, where the managed writer stages a DATA frame the moment you flush. That twin also serves an endless /feed; this one deliberately does not, because on this stack an endless response never reaches the wire.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet add package ioxide.http3
// curl --http3-only -k https://127.0.0.1:8443/
// curl --http3-only -kN https://127.0.0.1:8443/feed # never ends
// curl --http3-only -k --data-binary @big.bin https://127.0.0.1:8443/echo
using System.Text;
using ioxide;
using ioxide.http3;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443;
int reactors = Environment.ProcessorCount;
// Chunks written per response on "/", and the size of each. Their product is never held at once.
int chunkCount = 64;
int chunkBytes = 16 * 1024;
int udpRecvSlots = 16;
string? certOverride = null;
string? keyOverride = null;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"]);
var config = new ServerConfig
{
ReactorCount = reactors,
Tcp = null,
Udp = new UdpOptions { RecvSlots = udpRecvSlots },
Quic = new QuicOptions
{
Port = quicPort,
LocalCidLength = 8,
ConnectionFactory = engine.CreateFactory(),
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
byte[] chunk = Encoding.ASCII.GetBytes(new string('x', chunkBytes - 1) + "\n");
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.QuicHandle = (r, conn) =>
new Http3Connection(conn).RunStreamedResponseAsync(async (request, writer) =>
{
bool endless = request.Path.Span.SequenceEqual("/feed"u8);
bool upload = request.Path.Span.SequenceEqual("/upload"u8);
bool echo = request.Path.Span.SequenceEqual("/echo"u8);
if (echo)
{
// BOTH directions at once, which is the shape a proxy actually needs: read a
// chunk, write a chunk, and never hold more than one. Neither side can run away
// from the other - ReadAsync waits for the peer to send, FlushAsync waits for the
// connection to have room - so memory stays flat however large the exchange is.
writer.WriteHeaders(Plain());
if (request.BodyReader is { } duplex)
{
while (true)
{
ReadOnlyMemory<byte> part = await duplex.ReadAsync();
if (part.IsEmpty)
{
break; // end of the request body
}
part.Span.CopyTo(writer.GetSpan(part.Length));
writer.Advance(part.Length);
await writer.FlushAsync();
}
}
return;
}
if (upload)
{
// The other direction: pull the body a chunk at a time rather than waiting for
// all of it, so memory is bound by one chunk however large the upload is.
long total = 0;
if (request.BodyReader is { } body)
{
while (true)
{
ReadOnlyMemory<byte> part = await body.ReadAsync();
if (part.IsEmpty) break;
total += part.Length;
}
}
writer.WriteHeaders(Plain());
Encoding.ASCII.GetBytes($"{total}\n").CopyTo(writer.GetSpan(24));
writer.Advance(Encoding.ASCII.GetByteCount($"{total}\n"));
await writer.FlushAsync();
return;
}
writer.WriteHeaders(Plain(endless));
for (int n = 0; endless || n < chunkCount; n++)
{
chunk.CopyTo(writer.GetSpan(chunk.Length));
writer.Advance(chunk.Length);
// Returns once the chunk is queued, and waits when the connection is at its
// send-retention high-water. That await IS the backpressure.
await writer.FlushAsync();
}
});
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http3-managed-streamed-both] {config.ReactorCount} reactors, h3 on udp :{config.Quic!.Port} "
+ $"(pure C#), {chunkCount} x {chunkBytes}-byte chunks, cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}
static Http3Response Plain(bool eventStream = false)
{
var response = new Http3Response { Status = 200 };
response.Headers.Add(("content-type"u8.ToArray(),
eventStream ? "text/event-stream"u8.ToArray() : "text/plain"u8.ToArray()));
return response;
}Both directions streamed, in pure C#. The request body is PULLED a chunk at a time through Http3Request.BodyReader, so a large upload is never held whole; the response is PUSHED through Http3ResponseWriter, one DATA frame per flush, so a large download is never built whole. /echo runs both at once - read a chunk, write a chunk - which is what a proxy does. Owning the framing is what makes the push side simple: a chunk is just [0x00][varint length][payload] handed to the QUIC stream, with no data-reader callback to answer and nothing to defer. carries a resume and a drain because nghttp3 pulls instead; this measures 1.32× its throughput on the same 8×1 KiB response.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet add package ioxide.http3
// PLAYGROUND_CLIENT_CA=ca.crt dotnet run -c Release --project Playground/Http3/MutualTls
// curl --http3 --cacert ca.crt --cert client.crt --key client.key https://localhost:8443/
using System.Text;
using ioxide;
using ioxide.http3;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443;
int reactors = Environment.ProcessorCount;
// The server's own certificate and key. Null generates a self-signed pair on first run.
string? certOverride = null;
string? keyOverride = null;
// The CA that client certificates are checked against. This is what turns mTLS ON - leave both null
// and the server verifies nothing about the client, exactly as the other h3 samples do.
//
// Either the bundle's path, or the bundle itself as PEM text for a host that keeps its CA in a
// secrets store rather than on disk. Set one, not both.
string? clientCaPath = Environment.GetEnvironmentVariable("PLAYGROUND_CLIENT_CA");
string? clientCaPem = Environment.GetEnvironmentVariable("PLAYGROUND_CLIENT_CA_PEM");
// Refuse a client that offers no certificate, during the handshake. Off, so a client without one
// still connects and the handler decides what it may see - which is the more useful default when
// only part of a site is protected.
bool requireClientCertificate = Environment.GetEnvironmentVariable("PLAYGROUND_REQUIRE_CLIENT_CERT") == "1";
// UDP receive slots per reactor: how many datagrams the ring can have outstanding at once.
int udpRecvSlots = 16;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
if (clientCaPath is null && clientCaPem is null)
{
Console.Error.WriteLine("set PLAYGROUND_CLIENT_CA to a PEM bundle of the CA that signs your client "
+ "certificates, or PLAYGROUND_CLIENT_CA_PEM to that bundle as text.");
return 1;
}
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"],
clientCaPemPath: clientCaPath, requireClientCertificate: requireClientCertificate,
clientCaPem: clientCaPem);
var config = new ServerConfig
{
ReactorCount = reactors,
Tcp = null, // QUIC only
Udp = new UdpOptions { RecvSlots = udpRecvSlots },
Quic = new QuicOptions
{
Port = quicPort,
LocalCidLength = 8,
ConnectionFactory = engine.CreateFactory(),
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.QuicHandle = (_, connection) =>
new Http3Connection(connection).RunAsync(_ =>
{
// Read HERE, per request - not where the connection is accepted. That callback runs
// before the handshake finishes, so there is no identity yet at that point.
string? peer = (connection as QuicEngineConnection)?.PeerSubject;
var response = new Http3Response
{
Body = Encoding.UTF8.GetBytes(peer is null
? "anonymous\n"
: $"authenticated as {peer}\n"),
};
response.Headers.Add(("content-type"u8.ToArray(), "text/plain"u8.ToArray()));
return response;
});
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http3-mtls] {config.ReactorCount} reactors on :{quicPort}, "
+ $"client CA {clientCaPath ?? "from PEM text"}, "
+ $"client certificate {(requireClientCertificate ? "REQUIRED" : "optional")}");
foreach (Thread thread in threads)
{
thread.Join();
}
return 0;The client proves who it is too. clientCaPemPath is what turns it on - the CA that client certificates are checked against - and the handler reads PeerSubject to find out WHICH client it got. That distinction is the point: a server that can only answer some valid certificate has a gate, where one that can name the peer has an identity to authorise against. Read it per REQUEST, not where the connection is accepted - that callback runs before the handshake finishes, so there is no identity yet. QUIC settles client authentication during the handshake, and RFC 9001 §4.4 forbids doing it afterwards, so this is a property of the whole CONNECTION: there is no asking for a certificate later because a request reached a protected route. That needs a second port. requireClientCertificate decides whether a client offering none is refused during the handshake or arrives unauthenticated for the handler to judge.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet add package ioxide.nghttp3
// curl --http3-only -k https://127.0.0.1:8443/
using System.Runtime.InteropServices;
using ioxide;
using ioxide.nghttp3;
using ioxide.ngtcp2;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
// This sample is the QUIC/HTTP3 tuning reference: every knob the h3 path exposes is here as a
// literal, grouped by the type it feeds. The defaults are the shipping defaults - shown, not
// changed - so you can see the whole surface and edit one line.
ushort quicPort = 8443; // https://127.0.0.1:8443/ over UDP - h3 lives here
ushort tcpPort = 8080; // the TCP listener, so the process serves both
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// ── QuicEngine: the per-endpoint QUIC/TLS state, shared by every connection ───────────────────
uint cidLength = 8; // connection-id length this endpoint mints (1..20)
// Per-connection send-retention high-water. A response larger than this is streamed out paced by
// the peer's acks instead of buffered whole, so memory stays ~this-per-connection whatever the
// response size - the knob that lets HTTP/3 serve large files. Raise for more in-flight throughput
// on fat links; lower to cap memory under many connections. Default 16 MiB.
long maxSendRetentionBytes = 16L << 20;
// ── QuicOptions: the listener ─────────────────────────────────────────────────────────────────
int idleTimeoutMs = 60_000; // close a connection idle this long (no packets)
// ── UdpOptions: how datagrams are received ────────────────────────────────────────────────────
int udpRecvSlots = 16; // multishot recv slots per reactor - datagrams the ring can hold at once
bool gro = true; // UDP_GRO: coalesce received datagrams into one recv (fewer syscalls)
// ── Nghttp3Options: the HTTP/3 layer ──────────────────────────────────────────────────────────
// QPACK dynamic table. 0 keeps every header literal, which costs bytes but never blocks a stream
// on a table update; raise it and set QpackBlockedStreams to trade one for the other.
long qpackCapacity = 0;
long qpackBlockedStreams = qpackCapacity > 0 ? 100 : 0;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
using var engine = new QuicEngine(certPath, keyPath, cidLength, alpn: ["h3"], maxSendRetentionBytes);
var config = new ServerConfig
{
ReactorCount = reactors,
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Tcp = new TcpOptions
{
Port = tcpPort,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
Udp = new UdpOptions { RecvSlots = udpRecvSlots, Gro = gro },
Quic = new QuicOptions
{
Port = quicPort,
LocalCidLength = (int)cidLength, // must match the engine's cidLength
IdleTimeoutMs = idleTimeoutMs,
ConnectionFactory = engine.CreateFactory(),
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
var h3Options = new Nghttp3Options
{
QpackDynamicTableCapacity = qpackCapacity,
QpackBlockedStreams = qpackBlockedStreams,
};
// Built once and reused for every request - the h3 layer copies it into nghttp3 at submit and never
// retains it, so this costs zero allocations per request.
var response = new Nghttp3Response { Body = "Hello, World!"u8.ToArray() };
response.Headers.Add("content-type"u8.ToArray(), "text/plain"u8.ToArray());
response.Headers.Add("server"u8.ToArray(), "ioxide"u8.ToArray());
byte[] tcpResponse = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\r\nok"u8.ToArray();
List<(Reactor Reactor, Nghttp3Connection Connection)> live = [];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.QuicHandle = (r, quicConn) =>
{
var h3 = new Nghttp3Connection(quicConn, h3Options);
lock (live)
{
live.Add((r, h3));
}
// RunBufferedAsync, not RunStreamingAsync - that one call is the whole difference.
return h3.RunBufferedAsync(request =>
{
// Dispatch waited for end-of-stream, so the body is ALREADY here: request.Body is
// complete and request.Body.Length is just a property read. No BodyReader, no pacing.
// This overload is synchronous, but the awaiting one exists too - a PgPool query or a
// Redis command resumes inline on this reactor, so you can await it right here.
_ = request.Body.Length;
// One response object, reused for every request: zero allocations on this path. To
// route, compare request.Path.Span - it is post-QPACK bytes, so SequenceEqual against a
// u8 literal beats decoding it to a string.
return response;
});
};
reactor.TcpHandle = async (r, conn) =>
{
try
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer) conn.ReturnBuffer(in item);
}
conn.Write(tcpResponse);
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
using var drain = PosixSignalRegistration.Create(PosixSignal.SIGTERM, context =>
{
context.Cancel = true;
Console.WriteLine("[nghttp3-buffered] SIGTERM: draining connections (GOAWAY)...");
lock (live)
{
foreach ((Reactor r, Nghttp3Connection h3) in live)
{
r.ScheduleOnReactor(static state => ((Nghttp3Connection)state!).Shutdown(), h3);
}
live.Clear();
}
Thread.Sleep(2000);
Console.WriteLine("[nghttp3-buffered] drain complete, exiting");
Environment.Exit(0);
});
Console.WriteLine($"[nghttp3-buffered] {config.ReactorCount} reactors - tcp :{config.Tcp.Port}, "
+ $"udp :{quicPort} (ngtcp2 {QuicEngine.NativeVersion()})");
foreach (Thread thread in threads)
{
thread.Join();
}The same server as with the other dispatch mode - one method call is the whole difference. Buffered waits for end-of-stream, so the body is already in request.Body when your handler runs; the trade is that memory holds the whole body, which suits normal requests and not hostile uploads. Streamed runs you while the body is still arriving and credits the peer's flow-control window as you read, so memory is bound by one window instead.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet add package ioxide.nghttp3
// curl --http3-only -k https://127.0.0.1:8443/
using System.Buffers;
using System.Text;
using ioxide;
using ioxide.nghttp3;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443; // https://127.0.0.1:8443/ - UDP, not TCP
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// UDP receive slots per reactor: how many datagrams the ring can have outstanding at once.
int udpRecvSlots = 16;
// Per-connection send-retention high-water (default 16 MiB): a response larger than it streams out
// paced by acks instead of buffering whole. See Playground/Http3/Nghttp3Buffered for the full knob set.
long maxSendRetentionBytes = 16L << 20;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
// Permissive ALPN (no allowlist): h3 clients negotiate "h3"; everything else still handshakes
// and falls through to the echo branch.
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: null, maxSendRetentionBytes);
var config = new ServerConfig
{
ReactorCount = reactors, // one ring per reactor, one reactor per core
RingEntries = 8192, // io_uring SQ/CQ depth
DualStack = false, // IPv4-only sockets; true binds dual-stack IPv6 (::)
RecvBufferSize = 32 * 1024, // bytes per slot in the shared TCP recv ring
RecvSlots = 4096, // slots in that shared recv ring
Incremental = null, // shared recv ring; non-null = per-connection rings (kernel 6.12+)
Tcp = null, // QUIC-only: no TCP listener is bound
Udp = new UdpOptions
{
RecvSlots = udpRecvSlots, // multishot recv slots per reactor - datagrams the ring can hold at once
Gro = true, // UDP_GRO: coalesce a received datagram burst into one recv
},
Quic = new QuicOptions
{
Port = quicPort, // https://127.0.0.1:8443/ - UDP, not TCP
LocalCidLength = 8, // must match the engine's cidLength
IdleTimeoutMs = 60_000, // close a connection idle this long (no packets)
ConnectionFactory = engine.CreateFactory(), // the engine adopts each new connection
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.QuicHandle = async (r, conn) =>
{
// Peek only - the items stay queued for whichever branch takes over.
await conn.ReadAsync();
if (conn.NegotiatedProtocol == "h3")
{
// Owns the handler ref (DecRef on exit). Routes by byte compare - no per-request
// strings on the hot path.
await new Nghttp3Connection(conn).RunBufferedAsync(
static request => request.Path.Span.SequenceEqual("/plaintext"u8)
? Nghttp3Response.Text("Hello, World!")
: Nghttp3Response.Text(
$"hello {Encoding.ASCII.GetString(request.Path.Span)} over HTTP/3 via io_uring\n"));
return;
}
await PipeEcho(conn);
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[quic-alpn] {config.ReactorCount} reactors, quic-only on udp :{config.Quic!.Port} "
+ $"(h3 -> HTTP/3, other ALPN -> stream echo)");
foreach (Thread thread in threads)
{
thread.Join();
}
// Raw stream echo over the dual pipe: auto-binds to the client's first stream, echoes until the
// peer's fin, then Complete() half-closes with our own fin. A connection closed before the
// handshake falls through with a completed reader and exits clean.
static async Task PipeEcho(QuicConnection conn)
{
try
{
var pipe = new QuicConnectionDualPipe(conn);
while (true)
{
var result = await pipe.Input.ReadAsync();
foreach (ReadOnlyMemory<byte> segment in result.Buffer)
{
pipe.Output.Write(segment.Span);
}
await pipe.Output.FlushAsync();
pipe.Input.AdvanceTo(result.Buffer.End);
if (result.IsCompleted) break;
}
pipe.Output.Complete();
pipe.Input.Complete();
}
finally
{
conn.DecRef();
}
}One QUIC listener serving two protocols, chosen during the handshake: connections that negotiate h3 get the HTTP/3 loop, anything else gets raw stream echo over the dual pipe. QUIC-only - Tcp = null, so the process opens no TCP listener at all.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet run -c Release --project Playground/Quic/Raw # something to talk to
// dotnet run -c Release --project Playground/Clients/Quic
using System.Diagnostics;
using ioxide;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
string host = "127.0.0.1"; // IPv4 literal - resolving a name would block the reactor
ushort port = 8443; // the echo server's UDP port
string serverName = "localhost"; // SNI, and what the server certificate has to match
string alpn = "echo"; // must match what the server offers
int connections = 64; // each opens one bidirectional stream
int seconds = 10; // measured seconds, after a one-second warm-up
int payloadSize = 64; // bytes per round trip
// ─────────────────────────────────────────────────────────────────────────────────────────────
byte[] payload = new byte[payloadSize];
for (int i = 0; i < payload.Length; i++)
{
payload[i] = (byte)('a' + i % 26);
}
long roundTrips = 0;
var started = new TaskCompletionSource();
var deadline = Stopwatch.StartNew();
// A client needs no listener of any kind: QuicClientEngine.Connect asks the reactor for an
// outbound transport on an ephemeral port, so Tcp, Udp and Quic all stay null here.
var config = new ServerConfig
{
ReactorCount = 1,
RingEntries = 8192,
Tcp = null,
Udp = null,
Quic = null,
};
var engine = new QuicClientEngine(alpn);
var reactor = new Reactor(0, config);
reactor.OnStart = r =>
{
for (int i = 0; i < connections; i++)
{
QuicEngineConnection quic;
try
{
quic = engine.Connect(r, host, port, serverName);
}
catch (Exception e)
{
Console.Error.WriteLine($"connect failed: {e.Message}");
started.TrySetResult();
return;
}
// The stream cannot be opened before the handshake finishes, so the whole exchange hangs
// off this callback rather than running straight after Connect.
quic.HandshakeCompleted = () =>
{
started.TrySetResult();
_ = EchoLoop(quic);
};
}
};
var thread = new Thread(reactor.Run) { Name = "quic-echo", IsBackground = true };
thread.Start();
// Give the handshakes a moment; if none completes there is nothing to measure and saying so beats
// reporting a zero that reads like a regression.
Task first = await Task.WhenAny(started.Task, Task.Delay(10_000));
if (first != started.Task)
{
Console.Error.WriteLine($"no QUIC handshake completed against {host}:{port} (alpn {alpn})");
return 1;
}
await Task.Delay(1_000); // warm: handshakes settle, first streams open
Interlocked.Exchange(ref roundTrips, 0);
deadline.Restart();
await Task.Delay(seconds * 1000);
long total = Interlocked.Read(ref roundTrips);
double elapsed = deadline.Elapsed.TotalSeconds;
Console.WriteLine($"{total / elapsed:F2} req/s ({total} round trips in {elapsed:F1}s, "
+ $"{connections} connections, {payloadSize}-byte payload)");
return 0;
async Task EchoLoop(QuicEngineConnection quic)
{
long streamId = quic.OpenBidiStream();
if (streamId < 0)
{
return;
}
try
{
int outstanding = 0;
quic.SendStream(streamId, payload, fin: false);
while (deadline.Elapsed.TotalSeconds < seconds + 12)
{
QuicRecvSnapshot snapshot = await quic.ReadAsync();
while (quic.TryGetDelivery(in snapshot, out QuicRecvRing.Delivery delivery))
{
outstanding += delivery.AsSpan().Length;
quic.ReturnBuffer(in delivery);
}
// One payload back = one completed round trip. The echo can arrive split across
// deliveries, so this counts bytes rather than deliveries.
while (outstanding >= payload.Length)
{
outstanding -= payload.Length;
Interlocked.Increment(ref roundTrips);
quic.SendStream(streamId, payload, fin: false);
}
if (snapshot.IsClosed)
{
return;
}
quic.ResetRead();
}
}
catch
{
// A connection dying mid-run is not fatal to the measurement; the others keep going and
// the round-trip count reflects what actually completed.
}
}The client half of QUIC, and the other side of - everything else here is a server. QuicClientEngine.Connect needs no listener at all: it asks the reactor for an outbound transport on an ephemeral port, which is why Tcp, Udp and Quic are all null below. The handshake, the stream and the reads ride that reactor's ring and resume inline on it, exactly as they do server-side. It doubles as the load driver for the echo servers - they speak no HTTP, so wrk and h2load cannot touch them - which is what makes them benchmarkable.
// dotnet add package ioxide
// dotnet add package ioxide.timer
// curl http://127.0.0.1:8080/50
using System.Buffers.Text;
using System.Text.Unicode;
using ioxide;
using ioxide.timer;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080; // http://127.0.0.1:8080/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// The wait when the path names no number, so plain `curl http://127.0.0.1:8080/` shows the
// feature. A path that does name one wins, up to the ceiling - which is here because the delay
// comes from the request, and an unbounded one would let a client hold a connection all day.
int defaultDelayMs = 25;
int maxDelayMs = 60_000;
// true = await Task.Delay instead of the ring, so the two are measurable against each other
// rather than asserted. Same server, same response; only the wait changes.
bool useTaskDelay = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
// One timer for the connection's whole life, re-armed per request. A timer carries ONE
// wait at a time - one request's worth on HTTP/1.1; waiting on several deadlines at
// once wants a timer each. The reactor is what it submits to, so the deadline rides the
// ring this connection already lives on.
var timer = new RingTimer(r);
// The body names the wait, so the response cannot be pre-encoded the way Tcp/Raw's is.
// Per connection rather than per request, so it still allocates nothing per answer.
byte[] response = new byte[128];
try
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
int ms = defaultDelayMs;
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
ms = ParseDelay(item.AsSpan(), defaultDelayMs);
conn.ReturnBuffer(in item);
}
}
ms = Math.Clamp(ms, 0, maxDelayMs);
int result;
if (useTaskDelay)
{
// The deadline goes to the thread-pool timer queue and the continuation is
// posted back - the round trip Tcp/Hop is about.
await Task.Delay(ms);
result = RingTimer.ETime;
}
else
{
// The wait rides this reactor's ring and resumes on this thread, with the
// connection's state still warm.
result = await timer.DelayAsync(ms);
}
// An expired timeout reports -ETIME, which is this call's SUCCESS - so the check
// is Expired(), not result >= 0. Anything else is an errno.
conn.Write(response.AsSpan(0, Format(response, ms, result)));
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[timer] {config.ReactorCount} reactors on :{config.Tcp.Port} - GET /<ms> waits that long "
+ $"({(useTaskDelay ? "Task.Delay" : "RingTimer")}, default {defaultDelayMs}ms)");
foreach (Thread thread in threads)
{
thread.Join();
}
// "GET /50 HTTP/1.1" -> 50. The request line is in the first buffer of any request worth the
// name, so this looks at one and does not reassemble across reads.
static int ParseDelay(ReadOnlySpan<byte> request, int fallback)
{
int afterMethod = request.IndexOf((byte)' ');
if (afterMethod < 0) return fallback;
ReadOnlySpan<byte> target = request[(afterMethod + 1)..];
int end = target.IndexOf((byte)' ');
if (end < 0) return fallback;
target = target[..end].TrimStart((byte)'/');
return Utf8Parser.TryParse(target, out int ms, out int consumed) && consumed == target.Length
? ms
: fallback;
}
// The response, formatted into the connection's own buffer: how long the wait was, or the errno
// if the completion was not an expiry.
static int Format(Span<byte> destination, int ms, int result)
{
Span<byte> body = stackalloc byte[24];
int bodyLength;
if (RingTimer.Expired(result))
{
Utf8.TryWrite(body, $"{ms}ms", out bodyLength);
}
else
{
Utf8.TryWrite(body, $"errno {result}", out bodyLength);
}
Utf8.TryWrite(destination,
$"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {bodyLength}\r\n\r\n",
out int headerLength);
body[..bodyLength].CopyTo(destination[headerLength..]);
return headerLength + bodyLength;
}A wait is one IORING_OP_TIMEOUT: the kernel holds the deadline and the completion arrives on the reactor that took the request, with the connection's state still warm. Nothing is armed on the side, no syscall of its own is made to arrange it, and nothing is allocated per wait - a RingTimer holds ONE RingOpSource, so it carries one wait at a time and belongs to one connection. Which is the point at scale: 32,000 connections waiting is 32,000 deadlines the KERNEL holds, where a timerfd each would be 32,000 descriptors and a timerfd_settime per wait. Expiry arrives as -ETIME, io_uring's normal report for a timeout rather than a failure, which is why the check is RingTimer.Expired and not result >= 0. The useTaskDelay knob swaps in Task.Delay - same server, same response - so the hop it costs is measurable rather than asserted; is that hop on its own.
// dotnet add package ioxide
// dotnet add package ioxide.httpclient
// curl http://127.0.0.1:8080/get
using System.Text;
using ioxide;
using ioxide.httpclient;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080; // http://127.0.0.1:8080/get - cleartext IN
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// The https origin this calls OUT to. Point it somewhere real: "93.184.216.34" / "example.com"
// / 443. The IP must be a literal - resolving a name would block the reactor - and the NAME is
// what goes out as SNI and what the certificate has to match.
string originIp = "127.0.0.1";
string originName = "localhost";
ushort originPort = 443;
// Trust a private CA instead of the system store: "/path/ca.pem".
string? caFile = null;
// Skip certificate verification entirely. Read the warning at the call site before setting it.
bool insecure = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r =>
{
// One context per reactor, shared by that reactor's pool. Verification is on by default;
// turning it off leaves the connection encrypted but UNAUTHENTICATED, so anything in the
// path can present its own certificate and read or rewrite the whole exchange. A private
// CA belongs in CaFile, not here.
TlsClientContext tls = TlsClientContext.Create(new TlsClientOptions
{
ServerName = originName, // sent as SNI, checked against the cert (required)
AlpnProtocols = ["http/1.1"], // offered most-preferred first; [] offers none
CaFile = caFile, // PEM trust anchors; null = system store
VerifyCertificate = !insecure, // off = encrypted but UNAUTHENTICATED (tests only)
MinimumVersion = OpenSslVersions.Tls12, // lowest TLS accepted; Tls13 (0x0304) = 1.3-only
HandshakeTimeoutMs = 10_000, // handshake ceiling before the connect fails
});
HttpClientPool.Start(r, new HttpClientOptions
{
Host = originIp, // IPv4 literal - DNS would block the reactor
Port = originPort, // origin port
PoolSize = 2, // connections to the origin, per reactor
MaxResponseBytes = 8 * 1024 * 1024, // per-request ceiling for headers + body
SendBufferSize = 16 * 1024, // per-connection send buffer
ReceiveBufferSize = 16 * 1024, // per-connection recv buffer; grows to MaxResponseBytes
Tls = tls, // TLS context for https; null = cleartext
AcquireTimeoutMs = 10_000, // wait for a free connection when all are busy
});
};
reactor.TcpHandle = async (r, conn) =>
{
try
{
HttpClientPool upstream = r.GetService<HttpClientPool>()!;
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
string path = "/";
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
if (TryReadTarget(item.AsSpan(), out ReadOnlySpan<byte> target))
{
path = Encoding.ASCII.GetString(target);
}
conn.ReturnBuffer(in item);
}
}
try
{
// Encrypted on the way out, decrypted on the way back, and neither shows up
// here: the pool hands back the same HttpClientResponse a cleartext origin
// would have produced.
using HttpClientResponse response = await upstream.GetAsync(path);
conn.Write(Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Length: {response.Body.Length}\r\n\r\n"));
conn.Write(response.Body.Span);
}
catch (Exception e)
{
// A refused certificate arrives here like any other upstream failure - the
// handshake is part of opening the connection, so the pool reports it the same
// way it reports a dead origin.
byte[] message = Encoding.ASCII.GetBytes($"upstream: {e.Message}");
conn.Write(Encoding.ASCII.GetBytes(
$"HTTP/1.1 502 Bad Gateway\r\nContent-Length: {message.Length}\r\n\r\n"));
conn.Write(message);
}
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[tls-httpsclient] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"origin https://{originName} ({originIp}:{originPort}), "
+ $"verify={(insecure ? "OFF" : caFile ?? "system store")}");
foreach (Thread thread in threads)
{
thread.Join();
}
// "GET /path?query HTTP/1.1" -> "/path"
static bool TryReadTarget(ReadOnlySpan<byte> request, out ReadOnlySpan<byte> target)
{
target = default;
int firstSpace = request.IndexOf((byte)' ');
if (firstSpace < 0) return false;
ReadOnlySpan<byte> afterMethod = request[(firstSpace + 1)..];
int secondSpace = afterMethod.IndexOf((byte)' ');
if (secondSpace < 0) return false;
target = afterMethod[..secondSpace];
int query = target.IndexOf((byte)'?');
if (query >= 0) target = target[..query];
return true;
}TLS in the other direction: not terminating it for inbound connections but speaking it outbound, so a handler can reach an https:// origin on the same ring that accepted the request. One TlsClientContext per reactor, shared by that reactor's pool. Verification is on by default - turning it off leaves the hop encrypted but unauthenticated, and a private CA belongs in CaFile instead.
// dotnet add package ioxide
// curl http://127.0.0.1:8080/
using System.IO.Pipelines;
using ioxide;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080;
int reactors = Environment.ProcessorCount;
int bodyBytes = 2;
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
// handler code is identical either way; this only changes how recv buffers are handed out.
bool incrementalBuffers = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
// PLAYGROUND_BODY sizes the body (2 = "ok"), matching Tcp.Raw so the two are comparable.
byte[] body = bodyBytes == 2 ? "ok"u8.ToArray() : [.. Enumerable.Repeat((byte)'x', bodyBytes - 1), (byte)'\n'];
byte[] response =
[
.. System.Text.Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {body.Length}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
var reader = new TcpConnectionPipeReader(conn);
var writer = new TcpConnectionPipeWriter(conn);
try
{
while (true)
{
// io_uring recv behind the PipeReader surface - still resumes inline on the reactor.
ReadResult result = await reader.ReadAsync();
// This sample doesn't parse the request either: consume everything. A real handler
// would walk result.Buffer for complete requests and AdvanceTo(consumed, examined)
// so a partial one stays buffered for the next read.
reader.AdvanceTo(result.Buffer.End);
response.CopyTo(writer.GetSpan(response.Length));
writer.Advance(response.Length);
await writer.FlushAsync();
if (result.IsCompleted) return;
}
}
finally
{
reader.Complete();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[pipe] {config.ReactorCount} reactors on :{config.Tcp.Port}");
foreach (Thread thread in threads)
{
thread.Join();
}The IDuplexPipe seam over a plain TCP connection - what ioxide.Kestrel and the HTTP/2 layers sit on. Compare : same server, one abstraction lower.
// dotnet add package ioxide
// curl http://127.0.0.1:8080/
using System.Text;
using ioxide;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080;
int reactors = 12;
int bodyBytes = 2;
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
// handler code is identical either way; this only changes how recv buffers are handed out.
bool incrementalBuffers = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
// The engine. Every ServerConfig + TcpOptions knob is set here at its default, so the whole tuning
// surface is visible in one place - edit any line. One ring per reactor, one reactor per thread.
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
// One pre-encoded response, built once and written for every request. PLAYGROUND_BODY sizes the
// body (2 = "ok"); 1024 matches the object size load-generator grids conventionally measure.
byte[] body = bodyBytes == 2 ? "ok"u8.ToArray() : [.. Enumerable.Repeat((byte)'x', bodyBytes - 1), (byte)'\n'];
byte[] response =
[
.. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {body.Length}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
// Each reactor owns its ring, its SO_REUSEPORT listener and its connections outright.
// Nothing is shared between them, so nothing is locked.
var reactor = new Reactor(i, config);
// The handler. It runs on the reactor thread and every await resumes right back on it.
reactor.TcpHandle = async (r, conn) =>
{
try
{
while (true)
{
// io_uring recv. The continuation resumes inline on this thread - no thread pool.
RecvSnapshot snapshot = await conn.ReadAsync();
// ioxide hands you raw bytes and stays out of HTTP. This sample doesn't parse the
// request at all - it just returns every received buffer to the ring.
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
conn.ReturnBuffer(in item);
}
}
// Stage into the connection's write slab, then one io_uring send.
conn.Write(response);
await conn.FlushAsync();
if (snapshot.IsClosed) return; // peer went away
conn.ResetRead(); // arm the next read
}
}
finally
{
conn.DecRef(); // hand the connection object back to the reactor's pool
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[raw] {config.ReactorCount} reactors on :{config.Tcp.Port}, {bodyBytes}-byte body");
foreach (Thread thread in threads)
{
thread.Join();
}The bottom of the stack: no pipes, no protocol, just the ring. Buffers come from one shared provided-buffer ring per reactor - the default mode. hands them out per connection instead, and is the comparison.
// dotnet add package ioxide
// curl http://127.0.0.1:8080/
using System.Text;
using ioxide;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080; // http://127.0.0.1:8080/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 2; // "ok"
// The per-connection ring geometry - this block IS the mode. A connection gets its own ring of
// incRecvSlots buffers of incRecvBufferSize each, and the kernel appends across recvs into them,
// so a request split over several reads arrives contiguous. It costs memory per connection:
// incMaxConnections * incRecvSlots * incRecvBufferSize per reactor.
int incMaxConnections = 4096; // per reactor
int incRecvSlots = 16; // per connection
int incRecvBufferSize = 4096; // bytes per buffer
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
// Selecting the mode is setting this block. The shared-ring knobs (RecvBufferSize,
// RecvSlots) go unused once it is set.
Incremental = new IncrementalOptions
{
MaxConnections = incMaxConnections, // per reactor
RecvSlots = incRecvSlots , // per connection
RecvBufferSize = incRecvBufferSize, // bytes per buffer, kernel appends across recvs
},
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
byte[] body = bodyBytes == 2 ? "ok"u8.ToArray() : [.. Enumerable.Repeat((byte)'x', bodyBytes - 1), (byte)'\n'];
byte[] response =
[
.. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {body.Length}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
// Identical to Tcp.Raw on purpose: the read surface doesn't change with the mode, so a
// handler written against the shared ring runs unmodified here.
reactor.TcpHandle = async (r, conn) =>
{
try
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
conn.ReturnBuffer(in item);
}
}
conn.Write(response);
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[incremental] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"{config.Incremental!.RecvSlots}x{config.Incremental.RecvBufferSize}B per connection "
+ $"(kernel 6.12+)");
foreach (Thread thread in threads)
{
thread.Join();
}Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring. The kernel appends across recvs into the same buffer, so a request split over several reads arrives contiguous. Costs memory per connection - see .
Shared buffer ring (default, kernel 6.1+): one pool of buffers per reactor, drawn on by every connection. One recv consumes one whole buffer no matter how few bytes arrived - simple and elastic across connections, but small messages waste buffer space, and a buffer-hoarding connection eats from everyone's pool.
Incremental (kernel 6.12+): a small buffer ring per connection, and the kernel keeps appending successive recvs into the same buffer until it fills. Small messages pack densely and every connection's memory is isolated and bounded - at the cost of refcounted recycling (a buffer returns once you and the kernel are both done with it) and a ring registration per connection.
| shared | incremental | |
|---|---|---|
| buffer ownership | one pool per reactor | one small ring per connection |
| fill behavior | one recv = one whole buffer | recvs append into the same buffer |
| best for | medium/large messages, simplicity | many connections, small messages |
| memory shape | elastic, shared | isolated, bounded per connection |
| return path | push the id back | refcounted: you + kernel both done |
| kernel | 6.1+ | 6.12+ |
The handler code is identical in both modes - set Incremental to an
IncrementalOptions (or null for the shared ring); ReturnBuffers
routes the right return path internally.
The pipes tab is API sugar over either mode: the reader owns the carry for you.
// dotnet add package ioxide
// sudo modprobe tls # kTLS needs the Linux 'tls' module + OpenSSL 3
// curl -k https://127.0.0.1:8443/
using System.Text;
using ioxide;
using ioxide.tls;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 8 * 1024; // TLS cost is per-byte, so a 2-byte "ok" would hide it
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory)
KeyPath = keyPath, // PEM key file (or KeyPem for in-memory)
Alpn = ["http/1.1"], // protocols this port serves, most-preferred first
// NOT the default - TLS is OpenSSL both ways unless you ask for this. It is what lets the
// handler below write PLAINTEXT to the connection: the kernel turns it into records on send.
// Remove this line and conn.Write would put cleartext on the wire.
KernelTx = true,
// Full kTLS: the kernel decrypts inbound too. Experimental - about one first connection in
// twelve fails outright, and a client sending a TLS 1.3 KeyUpdate loses the connection.
// Tls/KtlsTx is this same server without this line: kernel TX, OpenSSL RX, deployable today.
KernelRx = true,
};
byte[] body = new byte[bodyBytes];
ReadOnlySpan<byte> fill = "ioxide-ktls-payload "u8;
for (int i = 0; i < bodyBytes; i++)
{
body[i] = fill[i % fill.Length];
}
byte[] response =
[
.. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {bodyBytes}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
// One TlsService per reactor: it owns the OpenSSL contexts and drives handshakes on this ring.
reactor.OnStart = r => TlsService.Start(r, tlsOptions);
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
// Decrypted bytes waiting to be framed. TLS hands back records, not requests: split one
// request across two records and each decrypts on its own, so answering per decrypt
// answers twice. The carry is what turns "some plaintext arrived" into "a request
// arrived", exactly as the plaintext samples do - ioxide does not parse HTTP for you.
var carry = new Carry();
try
{
// The handshake reads and writes through this same connection; after it, the socket
// carries kTLS records the kernel en/decrypts.
tls = await r.GetService<TlsService>().AcceptAsync(conn);
// A request can ride in with the handshake's final flight - answer it before parking
// in ReadAsync, or the client waits on a response we never send.
carry.Append(tls.DrainPlaintext());
if (Answer(conn, carry, response))
{
await conn.FlushAsync();
}
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
// Records in, plaintext out - the session decrypts what the ring received.
Decrypt(tls, in item, carry);
conn.ReturnBuffer(in item);
}
}
if (Answer(conn, carry, response))
{
await conn.FlushAsync(); // plaintext: the kernel encrypts on send
}
if (snapshot.IsClosed || tls.Closed) return;
conn.ResetRead();
}
}
catch (Exception e)
{
// Handlers run fire-and-forget, so a thrown handshake error would vanish silently -
// and a missing 'tls' kernel module manifests exactly here.
Console.Error.WriteLine($"[tls-ktls] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[tls-ktls] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"{bodyBytes}-byte body, tx=kernel, rx=kernel (experimental), cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}
// Decrypt takes a raw pointer (the buffer belongs to the ring); the pointer work stays out of the
// async handler, which cannot contain unsafe code.
static unsafe void Decrypt(TlsSession tls, in SpscRecvRing.Item item, Carry carry)
=> carry.Append(tls.Decrypt(item.Ptr, item.Len));
// One response per COMPLETE request in the carry, and none for a partial one. Returns whether
// anything was written, so the caller flushes once for the batch rather than once per request.
static bool Answer(TcpConnection conn, Carry carry, ReadOnlyMemory<byte> response)
{
bool wrote = false;
int end;
while ((end = carry.Span.IndexOf("\r\n\r\n"u8)) >= 0)
{
carry.Consume(end + 4);
// PLAINTEXT into the slab - safe only because KernelTx = true above. TlsSession.Write
// is the call that is correct either way, and is what every other sample uses; this one
// spells it out because demonstrating the kTLS write path is the point of the file.
conn.Write(response.Span);
wrote = true;
}
return wrote;
}
// Decrypted-but-unframed bytes: append at the end, consume from the front. A List<byte> pressed
// into this job needs CollectionsMarshal to be searched and RemoveRange to be consumed; a plain
// array does both directly.
sealed class Carry
{
private byte[] _buf = new byte[8 * 1024];
private int _len;
public ReadOnlySpan<byte> Span => _buf.AsSpan(0, _len);
public void Append(ReadOnlySpan<byte> bytes)
{
if (_buf.Length - _len < bytes.Length)
{
Array.Resize(ref _buf, Math.Max(_buf.Length * 2, _len + bytes.Length));
}
bytes.CopyTo(_buf.AsSpan(_len));
_len += bytes.Length;
}
public void Consume(int count)
{
_buf.AsSpan(count, _len - count).CopyTo(_buf);
_len -= count;
}
}Opt-in - and FULL kTLS: both directions in the kernel, set right on the options. Kernel RX is experimental, so Tls/KtlsTx in the repo is this server minus the KernelRx line - the half you would deploy today. The KernelTx line is what makes the conn.Write below legal: it puts PLAINTEXT into the slab and the kernel turns it into records. Without it the same call would put cleartext on the wire, which is why every other sample goes through TlsSession.Write - correct in either mode. Compare .
// dotnet add package ioxide
// sudo modprobe tls
// curl -k https://127.0.0.1:8443/
using System.Buffers;
using System.IO.Pipelines;
using System.Text;
using ioxide;
using ioxide.tls;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 8 * 1024; // TLS cost is per-byte, so a 2-byte "ok" would hide it
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory)
KeyPath = keyPath, // PEM key file (or KeyPem for in-memory)
Alpn = ["http/1.1"], // protocols this port serves, most-preferred first
// NOT the default - TLS is OpenSSL both ways unless you ask for this.
// Playground/Tls/OpenSslPipes is the same server without these two lines.
KernelTx = true,
// Full kTLS: the kernel decrypts inbound too. Experimental - about one first connection in
// twelve fails outright, and a KeyUpdate is fatal. Drop this line for the hybrid.
KernelRx = true,
};
byte[] body = new byte[bodyBytes];
"ioxide-tls-pipes "u8.CopyTo(body);
for (int i = "ioxide-tls-pipes "u8.Length; i < bodyBytes; i++)
{
body[i] = (byte)('a' + (i % 26));
}
byte[] response =
[
.. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Length: {bodyBytes}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r => TlsService.Start(r, tlsOptions);
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
try
{
tls = await r.GetService<TlsService>().AcceptAsync(conn);
// From here down, nothing knows or cares which backend is in use.
await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);
while (true)
{
ReadResult read = await pipe.Input.ReadAsync();
// Answer per REQUEST, not per read. TLS hands back RECORDS, so one request split
// across two records would otherwise draw two responses.
int answered = 0;
SequencePosition consumed = read.Buffer.Start;
var reader = new SequenceReader<byte>(read.Buffer);
while (reader.TryReadTo(out ReadOnlySequence<byte> _, "\r\n\r\n"u8, advancePastDelimiter: true))
{
consumed = reader.Position;
answered++;
}
// Consumed only whole requests; examined everything, so a partial head parks
// until more arrives instead of spinning on the same bytes.
pipe.Input.AdvanceTo(consumed, read.Buffer.End);
for (int n = 0; n < answered; n++)
{
pipe.Output.Write(response);
}
if (answered > 0)
{
await pipe.Output.FlushAsync();
}
if (read.IsCompleted || read.IsCanceled)
{
return;
}
}
}
catch (Exception e)
{
Console.Error.WriteLine($"[tls-ktls-pipes] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[tls-ktls-pipes] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"{bodyBytes}-byte body, tx=kernel, rx=kernel (experimental)");
foreach (Thread thread in threads)
{
thread.Join();
}The same full-kTLS server behind an IDuplexPipe, for the frameworks that serve from one. Now compare : its serve loop is byte-identical to this one. Over a pipe the backend is invisible, because TlsConnectionDualPipe composes its halves from the session rather than from configuration.
// dotnet add package ioxide
// sudo modprobe tls # the kernel half still needs the module
// curl -k https://127.0.0.1:8443/
using System.Text;
using ioxide;
using ioxide.tls;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 8 * 1024; // TLS cost is per-byte, so a 2-byte "ok" would hide it
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
// handler code is identical either way; this only changes how recv buffers are handed out.
bool incrementalBuffers = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory)
KeyPath = keyPath, // PEM key file (or KeyPem for in-memory)
Alpn = ["http/1.1"], // protocols this port serves, most-preferred first
// NOT the default - TLS is OpenSSL both ways unless you ask for this. It is what lets the
// handler below write PLAINTEXT to the connection: the kernel turns it into records on send.
// Remove this line and conn.Write would put cleartext on the wire.
KernelTx = true,
KernelRx = false, // kTLS receive (experimental; requires KernelTx)
};
byte[] body = new byte[bodyBytes];
ReadOnlySpan<byte> fill = "ioxide-ktls-tx "u8;
for (int i = 0; i < bodyBytes; i++)
{
body[i] = fill[i % fill.Length];
}
byte[] response =
[
.. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {bodyBytes}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
// One TlsService per reactor: it owns the OpenSSL contexts and drives handshakes on this ring.
reactor.OnStart = r => TlsService.Start(r, tlsOptions);
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
// Decrypted bytes waiting to be framed. TLS hands back records, not requests: split one
// request across two records and each decrypts on its own, so answering per decrypt
// answers twice. The carry is what turns "some plaintext arrived" into "a request
// arrived", exactly as the plaintext samples do - ioxide does not parse HTTP for you.
var carry = new Carry();
try
{
tls = await r.GetService<TlsService>().AcceptAsync(conn);
// A request can ride in with the handshake's final flight - answer it before parking
// in ReadAsync, or the client waits on a response we never send.
carry.Append(tls.DrainPlaintext());
if (Answer(conn, carry, response))
{
await conn.FlushAsync();
}
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
// Records in, plaintext out - the session decrypts what the ring received.
Decrypt(tls, in item, carry);
conn.ReturnBuffer(in item);
}
}
if (Answer(conn, carry, response))
{
await conn.FlushAsync();
}
if (snapshot.IsClosed || tls.Closed) return;
conn.ResetRead();
}
}
catch (Exception e)
{
// Handlers run fire-and-forget, so a thrown handshake error would vanish silently -
// and a missing 'tls' kernel module manifests exactly here.
Console.Error.WriteLine($"[tls-ktls-tx] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[tls-ktls-tx] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"{bodyBytes}-byte body, tx=kernel, rx=openssl, cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}
// Decrypt takes a raw pointer (the buffer belongs to the ring); the pointer work stays out of the
// async handler, which cannot contain unsafe code.
static unsafe void Decrypt(TlsSession tls, in SpscRecvRing.Item item, Carry carry)
=> carry.Append(tls.Decrypt(item.Ptr, item.Len));
// One response per COMPLETE request in the carry, and none for a partial one. Returns whether
// anything was written, so the caller flushes once for the batch rather than once per request.
static bool Answer(TcpConnection conn, Carry carry, ReadOnlyMemory<byte> response)
{
bool wrote = false;
int end;
while ((end = carry.Span.IndexOf("\r\n\r\n"u8)) >= 0)
{
carry.Consume(end + 4);
// PLAINTEXT into the slab - safe only because KernelTx = true above. TlsSession.Write
// is the call that is correct either way, and is what every other sample uses; this one
// spells it out because demonstrating the kTLS write path is the point of the file.
conn.Write(response.Span);
wrote = true;
}
return wrote;
}
// Decrypted-but-unframed bytes: append at the end, consume from the front. A List<byte> pressed
// into this job needs CollectionsMarshal to be searched and RemoveRange to be consumed; a plain
// array does both directly.
sealed class Carry
{
private byte[] _buf = new byte[8 * 1024];
private int _len;
public ReadOnlySpan<byte> Span => _buf.AsSpan(0, _len);
public void Append(ReadOnlySpan<byte> bytes)
{
if (_buf.Length - _len < bytes.Length)
{
Array.Resize(ref _buf, Math.Max(_buf.Length * 2, _len + bytes.Length));
}
bytes.CopyTo(_buf.AsSpan(_len));
_len += bytes.Length;
}
public void Consume(int count)
{
_buf.AsSpan(count, _len - count).CopyTo(_buf);
_len -= count;
}
}The deployable kernel mode: kernel TX, OpenSSL RX. The handler still writes plaintext - the kernel makes the records on send, which is what keeps sendfile and NIC offload reachable - while receive takes the well-trodden userspace path instead of kernel RX's experimental one. is this plus kernel receive; is the default, with the kernel in neither direction.
// dotnet add package ioxide
// curl -k https://127.0.0.1:8443/ # no modprobe needed
using System.Text;
using ioxide;
using ioxide.tls;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 8 * 1024; // TLS cost is per-byte, so a 2-byte "ok" would hide it
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
// handler code is identical either way; this only changes how recv buffers are handed out.
bool incrementalBuffers = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory)
KeyPath = keyPath, // PEM key file (or KeyPem for in-memory)
Alpn = ["http/1.1"], // protocols this port serves, most-preferred first
// KernelTx stays false (the default), so this sample IS the stock configuration: no TLS
// ULP on the socket, OpenSSL encrypts and decrypts, MSG_WAITALL stays on, and nothing here
// needs the 'tls' kernel module. Tls/Ktls is the sample that opts into the kernel path.
KernelTx = false,
KernelRx = false, // kTLS receive (experimental; requires KernelTx)
};
byte[] body = new byte[bodyBytes];
ReadOnlySpan<byte> fill = "ioxide-ktls-payload "u8;
for (int i = 0; i < bodyBytes; i++)
{
body[i] = fill[i % fill.Length];
}
byte[] response =
[
.. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {bodyBytes}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
// One TlsService per reactor: it owns the OpenSSL contexts and drives handshakes on this ring.
reactor.OnStart = r => TlsService.Start(r, tlsOptions);
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
// Decrypted bytes waiting to be framed. TLS hands back records, not requests: split one
// request across two records and each decrypts on its own, so answering per decrypt
// answers twice. The carry is what turns "some plaintext arrived" into "a request
// arrived", exactly as the plaintext samples do - ioxide does not parse HTTP for you.
var carry = new Carry();
try
{
// The handshake reads and writes through this same connection. Unlike Tls/Ktls, no
// ULP is attached afterwards - the socket stays an ordinary TCP socket and every
// record is made and opened by OpenSSL.
tls = await r.GetService<TlsService>().AcceptAsync(conn);
// A request can ride in with the handshake's final flight - answer it before parking
// in ReadAsync, or the client waits on a response we never send.
carry.Append(tls.DrainPlaintext());
if (Answer(conn, tls, carry, response))
{
await conn.FlushAsync();
}
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
// Records in, plaintext out - OpenSSL decrypts what the ring received.
Decrypt(tls, in item, carry);
conn.ReturnBuffer(in item);
}
}
if (Answer(conn, tls, carry, response))
{
await conn.FlushAsync();
}
if (snapshot.IsClosed || tls.Closed) return;
conn.ResetRead();
}
}
catch (Exception e)
{
// Handlers run fire-and-forget, so a thrown handshake error would vanish silently.
// Note what CANNOT happen here that can in Tls/Ktls: a missing 'tls' kernel module.
Console.Error.WriteLine($"[tls-openssl] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[tls-openssl] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"{bodyBytes}-byte body, cert {certPath}, no kernel TLS");
foreach (Thread thread in threads)
{
thread.Join();
}
// Decrypt takes a raw pointer (the buffer belongs to the ring); the pointer work stays out of the
// async handler, which cannot contain unsafe code.
static unsafe void Decrypt(TlsSession tls, in SpscRecvRing.Item item, Carry carry)
=> carry.Append(tls.Decrypt(item.Ptr, item.Len));
// One response per COMPLETE request in the carry, and none for a partial one. Returns whether
// anything was written, so the caller flushes once for the batch rather than once per request.
static bool Answer(TcpConnection conn, TlsSession tls, Carry carry, ReadOnlyMemory<byte> response)
{
bool wrote = false;
int end;
while ((end = carry.Span.IndexOf("\r\n\r\n"u8)) >= 0)
{
carry.Consume(end + 4);
// The one line that differs from Tls/Ktls. There, kTLS is producing the records so the
// handler writes plaintext; here OpenSSL has to encrypt before anything reaches the slab.
tls.Write(conn, response.Span);
wrote = true;
}
return wrote;
}
// Decrypted-but-unframed bytes: append at the end, consume from the front. A List<byte> pressed
// into this job needs CollectionsMarshal to be searched and RemoveRange to be consumed; a plain
// array does both directly.
sealed class Carry
{
private byte[] _buf = new byte[8 * 1024];
private int _len;
public ReadOnlySpan<byte> Span => _buf.AsSpan(0, _len);
public void Append(ReadOnlySpan<byte> bytes)
{
if (_buf.Length - _len < bytes.Length)
{
Array.Resize(ref _buf, Math.Max(_buf.Length * 2, _len + bytes.Length));
}
bytes.CopyTo(_buf.AsSpan(_len));
_len += bytes.Length;
}
public void Consume(int count)
{
_buf.AsSpan(count, _len - count).CopyTo(_buf);
_len -= count;
}
}The default. No TLS ULP is attached at all - OpenSSL encrypts and decrypts, and the response goes through TlsSession.Write, which is correct whichever backend the session ended up with. That drops every constraint kTLS imposes: no kernel module, TLS 1.2, any ciphersuite, session resumption back, no handshake-alignment problem. What it gives up is sendfile and NIC offload. It costs nothing measurable here - against the plaintext baseline, 4 reactors, wrk -t4 -c64: 0.79× for both at a 64-byte response, 0.35× kTLS vs 0.44× OpenSSL at 64 KiB. Which backend you pick matters far less than the cost of TLS.
// dotnet add package ioxide
// curl -k https://127.0.0.1:8443/ # no modprobe needed
using System.Buffers;
using System.IO.Pipelines;
using System.Text;
using ioxide;
using ioxide.tls;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 8 * 1024; // TLS cost is per-byte, so a 2-byte "ok" would hide it
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
// handler code is identical either way; this only changes how recv buffers are handed out.
bool incrementalBuffers = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory)
KeyPath = keyPath, // PEM key file (or KeyPem for in-memory)
Alpn = ["http/1.1"], // protocols this port serves, most-preferred first
// KernelTx stays false (the default). No TLS ULP on the socket, so OpenSSL encrypts and
// decrypts and nothing here needs the 'tls' kernel module - which also keeps TLS 1.2, any
// ciphersuite and session resumption. Playground/Tls/KtlsPipes adds the one line that
// changes this.
KernelTx = false,
KernelRx = false, // kTLS receive (experimental; requires KernelTx)
};
byte[] body = new byte[bodyBytes];
"ioxide-tls-pipes "u8.CopyTo(body);
for (int i = "ioxide-tls-pipes "u8.Length; i < bodyBytes; i++)
{
body[i] = (byte)('a' + (i % 26));
}
byte[] response =
[
.. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Length: {bodyBytes}\r\n\r\n"),
.. body,
];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r => TlsService.Start(r, tlsOptions);
reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
try
{
tls = await r.GetService<TlsService>().AcceptAsync(conn);
// From here down, nothing knows or cares which backend is in use.
await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);
while (true)
{
ReadResult read = await pipe.Input.ReadAsync();
// Answer per REQUEST, not per read. TLS hands back RECORDS, so one request split
// across two records would otherwise draw two responses.
int answered = 0;
SequencePosition consumed = read.Buffer.Start;
var reader = new SequenceReader<byte>(read.Buffer);
while (reader.TryReadTo(out ReadOnlySequence<byte> _, "\r\n\r\n"u8, advancePastDelimiter: true))
{
consumed = reader.Position;
answered++;
}
// Consumed only whole requests; examined everything, so a partial head parks
// until more arrives instead of spinning on the same bytes.
pipe.Input.AdvanceTo(consumed, read.Buffer.End);
for (int n = 0; n < answered; n++)
{
pipe.Output.Write(response);
}
if (answered > 0)
{
await pipe.Output.FlushAsync();
}
if (read.IsCompleted || read.IsCanceled)
{
return;
}
}
}
catch (Exception e)
{
Console.Error.WriteLine($"[tls-openssl-pipes] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[tls-openssl-pipes] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"{bodyBytes}-byte body, tx={(tlsOptions.KernelTx ? "kernel" : "openssl")}");
foreach (Thread thread in threads)
{
thread.Join();
}Diff this against and the only functional difference is KernelTx; the rest is the banner and the log tag. That is the point of the pipe seam - TlsConnectionDualPipe pairs TcpConnectionPipeReader or TlsDecryptingPipeReader with TcpConnectionPipeWriter or TlsEncryptingPipeWriter, chosen from the session. It has to be the session and not the config, because a handshake that left a partial record keeps the userspace reader whatever was asked for.
// dotnet add package ioxide
// dotnet add package ioxide.http2
// curl --http2-prior-knowledge http://127.0.0.1:8080/
using System.Text;
using ioxide;
using ioxide.http2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080;
int reactors = Environment.ProcessorCount;
int bodyBytes = 2;
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
// handler code is identical either way; this only changes how recv buffers are handed out.
bool incrementalBuffers = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
byte[] body = bodyBytes == 2
? "ok"u8.ToArray()
: [.. Enumerable.Repeat((byte)'x', bodyBytes)];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
try
{
// The connection owns the read loop from here: it parses frames, dispatches each
// request once its stream ends, and flushes the batch in one write.
await new Http2Connection(conn).RunBufferedAsync(_ => new Http2Response
{
Status = 200,
Body = body,
});
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http2-managed-buffered] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"{body.Length}-byte body (h2c prior knowledge)");
foreach (Thread thread in threads)
{
thread.Join();
}h2c with prior knowledge: the peer opens with the HTTP/2 connection preface and there is no upgrade dance. For h2 over TLS see - the protocol code there is byte-for-byte this one, because Http2Connection takes an IDuplexPipe and never learns what is under it. BUFFERED is the dispatch mode: the handler runs once the request has fully arrived, so request.Body holds the whole body and the answer is one Http2Response. That is the right default, and the wrong one for a large upload or an endless response - the three tabs after it are those cases.
// dotnet add package ioxide
// dotnet add package ioxide.http2
// curl --http2-prior-knowledge http://127.0.0.1:8080/
// curl --http2-prior-knowledge -N http://127.0.0.1:8080/feed # never ends
using System.Text;
using ioxide;
using ioxide.http2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080;
int reactors = Environment.ProcessorCount;
int bodyBytes = 2; // unused here: the body is produced chunk by chunk
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
// handler code is identical either way; this only changes how recv buffers are handed out.
bool incrementalBuffers = false;
// Chunks written per response on "/", and the size of each. Their product is never held at once.
int chunkCount = 64;
int chunkBytes = 16 * 1024;
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
byte[] body = bodyBytes == 2
? "ok"u8.ToArray()
: [.. Enumerable.Repeat((byte)'x', bodyBytes)];
var threads = new Thread[config.ReactorCount];
byte[] chunk = Encoding.ASCII.GetBytes(new string('x', chunkBytes - 1) + "\n");
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
try
{
await new Http2Connection(conn).RunAsync(async (request, writer) =>
{
bool endless = request.Path.Span.SequenceEqual("/feed"u8);
// Headers first and once. No content-length: the length is not known yet, and for
// /feed never will be - END_STREAM is what marks the end instead.
var response = new Http2Response { Status = 200 };
response.Headers.Add("content-type"u8.ToArray(),
endless ? "text/event-stream"u8.ToArray() : "text/plain"u8.ToArray());
writer.WriteHeaders(response);
for (int n = 0; endless || n < chunkCount; n++)
{
chunk.CopyTo(writer.GetSpan(chunk.Length));
writer.Advance(chunk.Length);
// Waits when either window is exhausted, and resumes on the WINDOW_UPDATE.
await writer.FlushAsync();
}
});
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http2-managed-streamed-response] {config.ReactorCount} reactors on :{config.Tcp!.Port} "
+ $"(pure C#), {chunkCount} x {chunkBytes}-byte chunks per response");
foreach (Thread thread in threads)
{
thread.Join();
}The RESPONSE body produced over time instead of returned whole - each flush becomes a DATA frame. /feed is why the mode exists: an endless response has no final byte, so a buffered API cannot express it at all. Http2ResponseWriter is an IBufferWriter<byte>, so a serializer or a framework's response sink writes into it unchanged. What HTTP/2 adds over is that credit is SHARED: every stream rides one TCP connection, so a flush waits on whichever of the stream and connection windows runs out first, and a WINDOW_UPDATE for either wakes it.
// dotnet add package ioxide
// dotnet add package ioxide.http2
// head -c 50000000 /dev/zero | curl --http2-prior-knowledge --data-binary @- \
// http://127.0.0.1:8080/upload
using System.Text;
using ioxide;
using ioxide.http2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080;
int reactors = Environment.ProcessorCount;
// Advertised per stream. This is the ceiling on how far ahead of the handler a peer may get, so
// on a streamed request it is the memory bound - not a throughput knob.
int streamWindow = 256 * 1024;
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/*
Tcp = new TcpOptions
{
Port = port,
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var http2 = new Http2Options
{
StreamRequestBodies = true, // the whole point: dispatch at the headers, body follows
InitialWindowSize = streamWindow, // how far ahead of the handler the peer may run
};
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
try
{
await new Http2Connection(conn, http2).RunBufferedAsync(async request =>
{
// BodyReader is set because StreamRequestBodies is on; with it off the body would
// be in request.Body instead and this would be null.
long total = 0;
if (request.BodyReader is { } body)
{
while (true)
{
// Empty means end of body. Each read hands back the peer's credit for the
// chunk it returns, which is what lets the next one arrive - and the
// memory it points at is recycled by the NEXT read, so anything worth
// keeping has to be copied out here.
ReadOnlyMemory<byte> chunk = await body.ReadAsync();
if (chunk.IsEmpty)
{
break;
}
total += chunk.Length;
}
}
return new Http2Response
{
Status = 200,
Body = Encoding.ASCII.GetBytes($"{total} bytes\n"),
};
});
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http2-managed-streamed-request] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"request bodies streamed, {streamWindow / 1024} KiB window per stream");
foreach (Thread thread in threads)
{
thread.Join();
}The other direction, and a different problem. StreamRequestBodies dispatches at the HEADERS and hands the handler an Http2BodyReader, so it runs while the upload is still arriving. What changes is what bounds memory: buffered holds the whole body, so MaxRequestBytes is all that stands between a hostile peer and the arena; streamed holds ONE flow-control window, because a chunk credits the peer's window only as the handler reads it. Fall behind and the peer runs out of credit and stops sending - backpressure the peer takes part in, rather than a buffer you hope is big enough.
// dotnet add package ioxide
// dotnet add package ioxide.http2
// curl --http2-prior-knowledge -N http://127.0.0.1:8080/feed
// head -c 50000000 /dev/zero | curl --http2-prior-knowledge --data-binary @- \
// http://127.0.0.1:8080/echo
using ioxide;
using ioxide.http2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080;
int reactors = Environment.ProcessorCount;
int chunkBytes = 1024; // one DATA frame per flush on /feed
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/*
Tcp = new TcpOptions
{
Port = port,
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
// Both halves are opt-in and independent: this one turns the REQUEST direction on, RunAsync below
// is what turns the RESPONSE direction on.
var http2 = new Http2Options { StreamRequestBodies = true };
byte[] chunk = [.. Enumerable.Repeat((byte)'x', chunkBytes)];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
try
{
await new Http2Connection(conn, http2).RunAsync(async (request, writer) =>
{
bool echo = request.Path.Span.SequenceEqual("/echo"u8);
// No content-length: on /feed the length will never be known, and on /echo it is
// not known yet. END_STREAM is what marks the end instead.
var response = new Http2Response { Status = 200 };
response.Headers.Add("content-type"u8.ToArray(), "text/plain"u8.ToArray());
writer.WriteHeaders(response);
if (echo)
{
// Both directions at once. Nothing here holds more than one chunk: the read
// credits the peer for what it hands back, and the flush waits for room on the
// way out - so a fast uploader is paced by the slower of the two, not buffered.
while (true)
{
ReadOnlyMemory<byte> incoming = await request.BodyReader!.ReadAsync();
if (incoming.IsEmpty)
{
break;
}
incoming.Span.CopyTo(writer.GetSpan(incoming.Length));
writer.Advance(incoming.Length);
await writer.FlushAsync();
}
return;
}
// /feed: a response with no end at all. There is no final byte to wait for, which
// is the case a buffered API has no way to express.
while (true)
{
chunk.CopyTo(writer.GetSpan(chunk.Length));
writer.Advance(chunk.Length);
await writer.FlushAsync();
}
});
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[http2-managed-streamed-both] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"request pulled and response pushed ({chunkBytes}-byte chunks on /feed)");
foreach (Thread thread in threads)
{
thread.Join();
}Both at once, which is the shape a proxy needs: /echo reads a chunk and writes a chunk, so neither the upload nor the download is ever held whole. The two directions are separate switches - StreamRequestBodies for the read side, RunAsync with a writer for the write side - because they solve different problems and most servers want exactly one of them. Mirrors ; the difference is HTTP/2's shared connection window, which a handler that stops reading holds down for every other stream on the connection.
// dotnet add package ioxide
// dotnet add package ioxide.nghttp2
// curl --http2-prior-knowledge http://127.0.0.1:8080/
using System.Text;
using ioxide;
using ioxide.nghttp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080;
int reactors = Environment.ProcessorCount;
int bodyBytes = 2;
// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
// handler code is identical either way; this only changes how recv buffers are handed out.
bool incrementalBuffers = false;
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
byte[] body = bodyBytes == 2
? "ok"u8.ToArray()
: [.. Enumerable.Repeat((byte)'x', bodyBytes)];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
try
{
// The connection owns the read loop from here: it feeds nghttp2, dispatches each
// request once its stream ends, and drains the egress once per batch.
await new Nghttp2Connection(conn).RunBufferedAsync(_ => new Nghttp2Response
{
Status = 200,
Body = body,
});
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[nghttp2] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"{body.Length}-byte body (h2c prior knowledge)");
foreach (Thread thread in threads)
{
thread.Join();
}The same h2c server on the reference implementation. Kept because being battle-tested is a property no amount of benchmarking substitutes for: nghttp2 is continuously fuzzed, patched by people whose job it is when the next HTTP/2 CVE lands, and has a decade of interop against every other stack. It streams responses too now - see - though not request bodies, where is still the only one. Measured as a client on this rig, the managed stack runs 1.35×-1.39× it. Take this one when you want the reference implementation's coverage; take for everything else.
// dotnet add package ioxide
// dotnet add package ioxide.nghttp2
// curl --http2-prior-knowledge http://127.0.0.1:8080/
// curl --http2-prior-knowledge -N http://127.0.0.1:8080/feed # never ends
using ioxide;
using ioxide.nghttp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080;
int reactors = Environment.ProcessorCount;
int chunkCount = 8; // DATA frames per response on /
int chunkBytes = 1024; // bytes per chunk
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/*
Tcp = new TcpOptions
{
Port = port,
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
byte[] chunk = [.. Enumerable.Repeat((byte)'x', chunkBytes)];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.TcpHandle = async (r, conn) =>
{
try
{
await new Nghttp2Connection(conn).RunAsync(async (request, writer) =>
{
bool endless = request.Path.Span.SequenceEqual("/feed"u8);
// Headers first and once. No content-length: the length is not known yet, and for
// /feed never will be - END_STREAM is what marks the end instead.
var response = new Nghttp2Response { Status = 200 };
response.Headers.Add("content-type"u8.ToArray(),
endless ? "text/event-stream"u8.ToArray() : "text/plain"u8.ToArray());
writer.WriteHeaders(response);
for (int n = 0; endless || n < chunkCount; n++)
{
chunk.CopyTo(writer.GetSpan(chunk.Length));
writer.Advance(chunk.Length);
await writer.FlushAsync();
}
});
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[nghttp2-streamed-response] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"{chunkCount} x {chunkBytes}-byte chunks per response");
foreach (Thread thread in threads)
{
thread.Join();
}The same streamed response as , and the sample code is nearly identical - but what happens underneath is the opposite. nghttp2 owns the framing, so it PULLS: it asks for body bytes when it is ready to emit DATA, and its read callback defers whenever nothing is buffered rather than ending the stream. Every write buffers a chunk natively and resumes the deferred stream. So a flush here means handed over, not on the wire - nghttp2 decides frame boundaries and timing - where the managed writer stages a DATA frame the moment you flush. That difference is the whole reason this needed new native entry points (ih2_submit_response_stream, ih2_stream_write, ih2_stream_close) rather than being a C# change.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet run -c Release --project Playground/Quic/Pipe
using System.Buffers;
using System.IO.Pipelines;
using ioxide;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443; // QUIC is UDP - this is a UDP port
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// UDP receive slots per reactor. Each slot pins ~64 KiB, enough for a full GRO train.
int udpRecvSlots = 16;
// Per-connection send-retention high-water. A response larger than this streams out paced by
// acks instead of being buffered whole, so QUIC serves large responses without unbounded
// per-connection memory.
long maxSendRetentionBytes = 16L << 20;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["echo"],
maxSendRetentionBytes: maxSendRetentionBytes);
var config = new ServerConfig
{
ReactorCount = reactors,
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false,
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Tcp = null, // QUIC only: no TCP listener is opened
// QUIC rides UDP, so these are its socket tunables. Ports is for RAW datagram sockets
// (Reactor.OnDatagram) - QUIC binds its own port and needs none listed here.
Udp = new UdpOptions
{
Ports = [],
RecvSlots = udpRecvSlots,
Gro = true,
},
Quic = new QuicOptions
{
Port = quicPort, // every reactor binds it via SO_REUSEPORT
LocalCidLength = 8, // short headers carry no CID length on the wire
ConnectionFactory = engine.CreateFactory(),
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
IdleTimeoutMs = 60_000, // transport backstop; 0 disables the sweep
},
};
var threads = new Thread[config.ReactorCount];
for (int id = 0; id < threads.Length; id++)
{
var reactor = new Reactor(id, config);
reactor.QuicHandle = async (r, conn) =>
{
var pipe = new QuicConnectionDualPipe(conn);
while (true)
{
ReadResult result = await pipe.Input.ReadAsync();
ReadOnlySequence<byte> buffer = result.Buffer;
foreach (ReadOnlyMemory<byte> segment in buffer)
{
await pipe.Output.WriteAsync(segment);
}
pipe.Input.AdvanceTo(buffer.End);
if (result.IsCompleted) return;
}
};
threads[id] = new Thread(reactor.Run) { Name = $"reactor-{id}" };
threads[id].Start();
}
Console.WriteLine($"[quic-pipe] {config.ReactorCount} reactors, QUIC on udp :{config.Quic!.Port} "
+ $"(ngtcp2 {QuicEngine.NativeVersion()}), cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}QUIC behind an IDuplexPipe, the transport's twin of . A PipeReader is ONE byte stream, so this binds to a single QUIC stream - which is exactly why HTTP/3 cannot use it and takes instead. ngtcp2 + picotls ship as one native, so TLS 1.3 is inside the transport and the certificate IS the config.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet run -c Release --project Playground/Quic/Raw
using ioxide;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443; // QUIC is UDP - this is a UDP port
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// UDP receive slots per reactor. Each slot pins ~64 KiB, enough for a full GRO train.
int udpRecvSlots = 16;
// Per-connection send-retention high-water. A response larger than this streams out paced by
// acks instead of being buffered whole, so QUIC serves large responses without unbounded
// per-connection memory.
long maxSendRetentionBytes = 16L << 20;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["echo"],
maxSendRetentionBytes: maxSendRetentionBytes);
var config = new ServerConfig
{
ReactorCount = reactors,
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false,
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Tcp = null, // QUIC only: no TCP listener is opened
// QUIC rides UDP, so these are its socket tunables. Ports is for RAW datagram sockets
// (Reactor.OnDatagram) - QUIC binds its own port and needs none listed here.
Udp = new UdpOptions
{
Ports = [],
RecvSlots = udpRecvSlots,
Gro = true,
},
Quic = new QuicOptions
{
Port = quicPort, // every reactor binds it via SO_REUSEPORT
LocalCidLength = 8, // short headers carry no CID length on the wire
ConnectionFactory = engine.CreateFactory(),
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
IdleTimeoutMs = 60_000, // transport backstop; 0 disables the sweep
},
};
var threads = new Thread[config.ReactorCount];
for (int id = 0; id < threads.Length; id++)
{
var reactor = new Reactor(id, config);
reactor.QuicHandle = async (r, conn) =>
{
while (true)
{
QuicRecvSnapshot snapshot = await conn.ReadAsync();
// Each delivery names its stream, so echoing back on delivery.StreamId keeps every
// stream independent - which is exactly what a multi-stream protocol needs.
while (conn.TryGetDelivery(in snapshot, out QuicRecvRing.Delivery delivery))
{
conn.SendStream(delivery.StreamId, delivery.AsSpan(), fin: false);
conn.ReturnBuffer(in delivery);
}
if (snapshot.IsClosed) return;
conn.ResetRead();
}
};
threads[id] = new Thread(reactor.Run) { Name = $"reactor-{id}" };
threads[id].Start();
}
Console.WriteLine($"[quic-raw] {config.ReactorCount} reactors, QUIC on udp :{config.Quic!.Port} "
+ $"(ngtcp2 {QuicEngine.NativeVersion()}), cert {certPath}");
foreach (Thread thread in threads)
{
thread.Join();
}Every stream on the connection, not just one: each delivery names its stream id, so a multi-stream protocol demuxes right here. This is the surface ioxide.nghttp3 sits on.
// dotnet add package ioxide
// dotnet add package ioxide.ngtcp2
// dotnet add package ioxide.nghttp3
// curl --http3-only -k https://127.0.0.1:8443/
using System.Runtime.InteropServices;
using ioxide;
using ioxide.nghttp3;
using ioxide.ngtcp2;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort quicPort = 8443; // https://127.0.0.1:8443/ over UDP - h3 lives here
ushort tcpPort = 8080; // the TCP listener, so the process serves both
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// Multishot recv slots per reactor - datagrams the ring can hold at once. QPACK capacity 4096
// advertises a decode-side dynamic table; 0 is static-only, nghttp3's default.
int udpRecvSlots = 16;
// Response body size. 13 is "Hello, World!"; anything else is that many 'x'. A buffered response
// holds the whole body, so this is also what a streamed response is measured against.
int bodyBytes = 13;
long qpackCapacity = 0;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
// One engine for the whole server. ALPN pinned to h3, so nothing else negotiates. The last arg is
// the per-connection send-retention high-water (default 16 MiB): a response larger than it streams
// out paced by acks instead of buffering whole, so h3 serves large files in bounded memory. See
// Playground/Http3/Nghttp3Buffered for the full QUIC/h3 knob set.
using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"], maxSendRetentionBytes: 16L << 20);
var config = new ServerConfig
{
ReactorCount = reactors, // one ring per reactor, one reactor per core
RingEntries = 8192, // io_uring SQ/CQ depth
DualStack = false, // IPv4-only listeners; true binds dual-stack IPv6 (::)
RecvBufferSize = 32 * 1024, // bytes per slot in the shared TCP recv ring
RecvSlots = 4096, // slots in that shared recv ring
Incremental = null, // shared recv ring; non-null = per-connection rings (kernel 6.12+)
Tcp = new TcpOptions
{
Port = tcpPort, // the TCP listener, so the process serves both
ExtraPorts = [], // extra listener ports, each bound by every reactor
ListenBacklog = 1024, // listen() accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write slab before overflow
PoolMax = 1024, // max pooled connection objects per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // grow the slab; Segmented chains pooled slabs
ZeroCopySend = false, // plain SEND; SEND_ZC only wins for large responses
RecvQueueEntries = 64, // per-connection SPSC recv queue depth (power of two)
},
Udp = new UdpOptions
{
RecvSlots = udpRecvSlots, // multishot recv slots per reactor - datagrams the ring can hold at once
Gro = true, // UDP_GRO: coalesce a received datagram burst into one recv
},
Quic = new QuicOptions
{
Port = quicPort, // https://127.0.0.1:8443/ over UDP - h3 lives here
LocalCidLength = 8, // must match the engine's cidLength
IdleTimeoutMs = 60_000, // close a connection idle this long (no packets)
ConnectionFactory = engine.CreateFactory(), // the engine adopts each new connection
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
var h3Options = new Nghttp3Options
{
QpackDynamicTableCapacity = qpackCapacity, // 0 (default) = headers stay literal
QpackBlockedStreams = qpackCapacity > 0 ? 100 : 0, // raise both together for the dynamic table
};
// THE allocation-free pattern: build the response ONCE and reuse the instance for every request.
// Legal because the h3 layer copies status, headers and body into nghttp3 synchronously at submit
// and never retains the object - unlike Nghttp3Response.Text($"..."), which encodes a fresh string
// every time. This is what a hot path should look like.
var response = new Nghttp3Response
{
Body = bodyBytes == 13 ? "Hello, World!"u8.ToArray() : [.. Enumerable.Repeat((byte)'x', bodyBytes)],
};
response.Headers.Add("content-type"u8.ToArray(), "text/plain"u8.ToArray());
response.Headers.Add("server"u8.ToArray(), "ioxide"u8.ToArray());
// A fixed TCP response for :8080, which still listens alongside the QUIC port.
byte[] tcpResponse = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\r\nok"u8.ToArray();
// Live connections, so SIGTERM can GOAWAY them all. Each reactor only ever adds its own, but the
// signal handler runs OFF the reactor threads, so a plain lock keeps it honest.
List<(Reactor Reactor, Nghttp3Connection Connection)> live = [];
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
// ---- HTTP/3 on udp :8443 ----------------------------------------------------------------
// Nghttp3Connection owns the connection's read loop: it feeds stream data into nghttp3,
// assembles each request - headers, body and all - and calls your function once per request.
// QPACK, control streams, fin and stream teardown are its problem, not yours.
reactor.QuicHandle = (r, quicConn) =>
{
var h3 = new Nghttp3Connection(quicConn, h3Options);
lock (live)
{
live.Add((r, h3));
}
return h3.RunStreamingAsync(async request =>
{
// Streamed dispatch, so we are running while the body is still on the wire. Read it to
// the end: every read credits the peer's flow-control window, which is what throttles a
// fast sender instead of buffering it. Chunks are valid until the next ReadAsync, and a
// request with no body simply finds it empty on the first read.
long total = 0;
while (true)
{
ReadOnlyMemory<byte> chunk = await request.BodyReader!.ReadAsync();
if (chunk.IsEmpty) break;
total += chunk.Length; // a real app would parse or store the chunk here
}
// One response object, reused for every request: zero allocations on this path. To
// route, compare request.Path.Span - it is post-QPACK bytes, so SequenceEqual against a
// u8 literal beats decoding it to a string.
return response;
});
};
// ---- plain HTTP/1.1 on tcp :8080 ---------------------------------------------------------
reactor.TcpHandle = async (r, conn) =>
{
try
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer) conn.ReturnBuffer(in item);
}
conn.Write(tcpResponse);
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
// Graceful shutdown. Without this the process dies mid-request and clients see resets. This runs
// OFF the reactor threads, so each Shutdown is marshalled back onto its owning reactor - nghttp3
// and the send path must only be touched there.
using var drain = PosixSignalRegistration.Create(PosixSignal.SIGTERM, context =>
{
context.Cancel = true;
Console.WriteLine("[nghttp3] SIGTERM: draining connections (GOAWAY)...");
lock (live)
{
foreach ((Reactor r, Nghttp3Connection h3) in live)
{
r.ScheduleOnReactor(static state => ((Nghttp3Connection)state!).Shutdown(), h3);
}
live.Clear();
}
Thread.Sleep(2000); // let in-flight requests finish
Console.WriteLine("[nghttp3] drain complete, exiting");
Environment.Exit(0);
});
Console.WriteLine($"[nghttp3] {config.ReactorCount} reactors - tcp :{config.Tcp.Port}, "
+ $"udp :{quicPort} (ngtcp2 {QuicEngine.NativeVersion()})");
foreach (Thread thread in threads)
{
thread.Join();
}HTTP/3 over QUIC, dispatched as the body streams. Compare , which waits for end-of-stream instead - one method call is the whole difference.
A reverse proxy is two hops, and in ioxide each one is a type. The frontend protocol is the server you hand the connection to; the upstream protocol is the pool you call. Nothing else changes between the nine samples below - not the reactor, not the config, not the handler's shape.
Every hop is TLS, which is the only way an h2 frontend is reachable from a browser
at all. It is reached two different ways: kTLS facing (OpenSSL handshake over the
ring, then the kernel owns transmit, so the handler writes plaintext), and a
TlsClientContext upstream. The h3 frontend needs neither - QUIC has no cleartext
mode, so TLS 1.3 is already inside the transport.
The origin leg is HTTP/1.1 in all three. That is the leg an origin almost always speaks, and keeping the client to one protocol is what lets it depend on ioxide alone.
| upstream h1 | |
|---|---|
| frontend h1 a raw TCP loop |
|
frontend h2Nghttp2Connection |
|
frontend h3Nghttp3Connection |
|
HttpClientPool - one pool type, opened on
the reactor that will use it, so the outbound call never crosses a thread. | |
Both hops ride one ring. The inbound connection and the outbound call are
completions on the same reactor, and each await resumes inline on the thread
that owns them - a proxied request never leaves the core it arrived on, and the pool needs no
lock because only one thread can ever touch it.
The pool sizes differ, and only for one reason. h2 and h3 multiplex, so
PoolSize = 1 carries every concurrent request on a single connection. h1 does
not, so an h1 upstream needs a connection per in-flight request - which is why
h2 → h1 is the one sample here that sizes its pool for concurrency.
Every pane is the corresponding
Playground/Proxy
program, whole, with the sample's environment-variable defaults inlined. Two caveats they
share. They do not filter hop-by-hop headers - a production proxy must, and
Connection, Keep-Alive and Transfer-Encoding are
protocol errors to forward into h2 or h3 at all. And they trust the self-signed certificate
the playground generates, so they run against each other out of the box; a real deployment
points CaFile at a private CA or leaves it null for the system store. Turning
VerifyCertificate off leaves the hop encrypted but unauthenticated.
// dotnet add package ioxide ioxide.httpclient
// PLAYGROUND_PORT=8444 dotnet run --project Playground/Tls/Ktls # a TLS origin
// curl -k https://127.0.0.1:8443/
using System.Text;
using ioxide;
using ioxide.httpclient;
using ioxide.tls;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
int reactors = Environment.ProcessorCount;
ushort port = 8443;
string upstreamHost = "127.0.0.1";
ushort upstreamPort = 8444;
string upstreamSni = "localhost";
int upstreamPool = 8;
string? upstreamCa = null;
bool upstreamInsecure = false;
string? certOverride = null; // a real PEM pair, or null to self-sign on first run
string? keyOverride = null;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors,
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only frontend)
Quic = null, // no QUIC listener; the frontend is TLS-over-TCP
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
string upstreamName = upstreamSni; // sent as SNI, checked against the cert
// The playground's origins use a self-signed cert, so trust that file rather than the system
// store. PLAYGROUND_UPSTREAM_CA points at a private CA instead; PLAYGROUND_UPSTREAM_INSECURE=1
// skips verification, which leaves the hop encrypted but UNAUTHENTICATED - anything in the path
// can present its own certificate and rewrite the whole exchange.
upstreamCa ??= certPath;
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r =>
{
// Inbound: terminate TLS for clients. ALPN is an ordered list and this proxy speaks one
// protocol, so it offers exactly one.
TlsService.Start(r, new TlsOptions
{
CertificatePath = certPath, // PEM cert chain file (set exactly one of Path/Pem)
CertificatePem = null, // in-memory PEM alternative to CertificatePath
KeyPath = keyPath, // PEM private key file (set exactly one of Path/Pem)
KeyPem = null, // in-memory PEM alternative to KeyPath
Alpn = ["http/1.1"], // protocols offered, most preferred first
KernelTx = false, // kTLS encrypt (off = OpenSSL both ways)
KernelRx = false, // kTLS decrypt; requires KernelTx, experimental
});
// Outbound: one context per reactor, shared by that reactor's pool - it holds no
// per-connection state, so every connection opened from it gets its own SSL.
HttpClientPool.Start(r, new HttpClientOptions
{
Host = upstreamHost, // origin address (IPv4 literal; DNS would block the reactor)
Port = upstreamPort, // origin port
PoolSize = upstreamPool, // connections kept open; round-robin, one request each
MaxResponseBytes = 8 * 1024 * 1024, // per-request ceiling for headers + body
SendBufferSize = 16 * 1024, // per-connection send buffer
ReceiveBufferSize = 16 * 1024, // per-connection recv buffer (grows to MaxResponseBytes)
AcquireTimeoutMs = 10_000, // how long a request waits for a free connection
Tls = TlsClientContext.Create(new TlsClientOptions
{
ServerName = upstreamName, // sent as SNI, checked against the cert
AlpnProtocols = ["http/1.1"], // ALPN offer, most preferred first
VerifyCertificate = !upstreamInsecure, // off = encrypted but UNAUTHENTICATED
CaFile = upstreamInsecure ? null : upstreamCa, // PEM trust anchors; null = system store
MinimumVersion = OpenSslVersions.Tls12, // lowest TLS accepted; Tls13 = 1.3 only
HandshakeTimeoutMs = 10_000, // handshake deadline
}),
});
};
reactor.TcpHandle = async (r, conn) =>
{
HttpClientPool client = r.GetService<HttpClientPool>();
TlsSession? tls = null;
try
{
// The handshake reads and writes through this same connection.
tls = await r.GetService<TlsService>().AcceptAsync(conn);
// A request can ride in with the handshake's final flight - answer it before parking
// in ReadAsync, or the client waits on a response we never send.
if (TryReadTarget(tls.DrainPlaintext(), out ReadOnlySpan<byte> early))
{
await ProxyAsync(conn, tls, client, Encoding.ASCII.GetString(early));
}
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
string? path = null;
unsafe
{
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
// Records in, plaintext out - the session decrypts what the ring got.
if (TryReadTarget(tls.Decrypt(item.Ptr, item.Len), out ReadOnlySpan<byte> target))
{
path = Encoding.ASCII.GetString(target);
}
conn.ReturnBuffer(in item);
}
}
}
if (path is not null)
{
await ProxyAsync(conn, tls, client, path);
}
if (snapshot.IsClosed || tls.Closed) return;
conn.ResetRead();
}
}
catch (Exception e)
{
Console.Error.WriteLine($"[proxy h1->h1] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[proxy h1->h1] {config.ReactorCount} reactors, https on :{config.Tcp!.Port} "
+ $"-> https://{upstreamName} ({upstreamHost}:{upstreamPort}), "
+ $"{upstreamPool} connections each, "
+ $"verify={(upstreamInsecure ? "OFF" : upstreamCa ?? "system store")}");
foreach (Thread thread in threads)
{
thread.Join();
}
// One request out, one response back. tls.Write encrypts correctly whichever backend the
// session ended up with.
static async ValueTask ProxyAsync(TcpConnection conn, TlsSession tls, HttpClientPool client, string path)
{
try
{
// The outbound call. Same ring, same thread - this await resumes inline. The response
// owns its bytes, so dispose it when you're done with them.
using HttpClientResponse response = await client.GetAsync(path);
tls.Write(conn, Encoding.ASCII.GetBytes(
$"HTTP/1.1 {response.Status} OK\r\nContent-Length: {response.Body.Length}\r\n\r\n"));
tls.Write(conn, response.Body.Span); // bytes straight through, no decode
}
catch (Exception e)
{
// A dead origin surfaces here rather than hanging: the pool bounds the whole acquire. A
// REFUSED CERTIFICATE arrives the same way - the handshake is part of opening the
// connection, so a name mismatch reads like any other upstream failure.
byte[] message = Encoding.ASCII.GetBytes($"upstream: {e.Message}");
tls.Write(conn, Encoding.ASCII.GetBytes(
$"HTTP/1.1 502 Bad Gateway\r\nContent-Length: {message.Length}\r\n\r\n"));
tls.Write(conn, message);
}
await conn.FlushAsync();
}
// "GET /sleep?x=1 HTTP/1.1" -> "/sleep". Your framework of choice would do this for you; ioxide
// deliberately doesn't, so here it is in full.
static bool TryReadTarget(ReadOnlySpan<byte> request, out ReadOnlySpan<byte> target)
{
target = default;
int firstSpace = request.IndexOf((byte)' ');
if (firstSpace < 0) return false;
ReadOnlySpan<byte> afterMethod = request[(firstSpace + 1)..];
int secondSpace = afterMethod.IndexOf((byte)' ');
if (secondSpace < 0) return false;
target = afterMethod[..secondSpace];
int query = target.IndexOf((byte)'?');
if (query >= 0) target = target[..query];
return true;
}The baseline, and the one to read first. Everything else in this section is this program with one type changed.
// dotnet add package ioxide ioxide.http2 ioxide.httpclient
// PLAYGROUND_PORT=8444 dotnet run --project Playground/Tls/Ktls # a TLS origin
// curl -k --http2 https://127.0.0.1:8443/
using System.Text;
using ioxide;
using ioxide.httpclient;
using ioxide.http2;
using ioxide.tls;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
int reactors = Environment.ProcessorCount;
ushort port = 8443;
string upstreamHost = "127.0.0.1";
ushort upstreamPort = 8444;
string upstreamSni = "localhost";
int upstreamPool = 32;
string? upstreamCa = null;
bool upstreamInsecure = false;
string? certOverride = null; // a real PEM pair, or null to self-sign on first run
string? keyOverride = null;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
var config = new ServerConfig
{
ReactorCount = reactors,
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only frontend)
Quic = null, // no QUIC listener; the frontend is TLS-over-TCP
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
string upstreamName = upstreamSni; // sent as SNI, checked against the cert
// The playground's origins use a self-signed cert, so trust that file rather than the system
// store. PLAYGROUND_UPSTREAM_CA points at a private CA instead; PLAYGROUND_UPSTREAM_INSECURE=1
// skips verification, which leaves the hop encrypted but UNAUTHENTICATED - anything in the path
// can present its own certificate and rewrite the whole exchange.
upstreamCa ??= certPath;
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r =>
{
// Inbound: terminate TLS and offer only h2, because that is all this frontend serves. A
// client that cannot do h2 fails ALPN rather than silently getting something else.
TlsService.Start(r, new TlsOptions
{
CertificatePath = certPath, // PEM cert chain file (set exactly one of Path/Pem)
CertificatePem = null, // in-memory PEM alternative to CertificatePath
KeyPath = keyPath, // PEM private key file (set exactly one of Path/Pem)
KeyPem = null, // in-memory PEM alternative to KeyPath
Alpn = ["h2"], // protocols offered, most preferred first
KernelTx = false, // kTLS encrypt (off = OpenSSL both ways)
KernelRx = false, // kTLS decrypt; requires KernelTx, experimental
});
// Outbound: one context per reactor, shared by that reactor's pool - it holds no
// per-connection state, so every connection opened from it gets its own SSL.
HttpClientPool.Start(r, new HttpClientOptions
{
Host = upstreamHost, // origin address (IPv4 literal; DNS would block the reactor)
Port = upstreamPort, // origin port
PoolSize = upstreamPool, // connections kept open; round-robin, one request each
MaxResponseBytes = 8 * 1024 * 1024, // per-request ceiling for headers + body
SendBufferSize = 16 * 1024, // per-connection send buffer
ReceiveBufferSize = 16 * 1024, // per-connection recv buffer (grows to MaxResponseBytes)
AcquireTimeoutMs = 10_000, // how long a request waits for a free connection
Tls = TlsClientContext.Create(new TlsClientOptions
{
ServerName = upstreamName, // sent as SNI, checked against the cert
AlpnProtocols = ["http/1.1"], // ALPN offer, most preferred first
VerifyCertificate = !upstreamInsecure, // off = encrypted but UNAUTHENTICATED
CaFile = upstreamInsecure ? null : upstreamCa, // PEM trust anchors; null = system store
MinimumVersion = OpenSslVersions.Tls12, // lowest TLS accepted; Tls13 = 1.3 only
HandshakeTimeoutMs = 10_000, // handshake deadline
}),
});
};
reactor.TcpHandle = async (r, conn) =>
{
HttpClientPool client = r.GetService<HttpClientPool>();
TlsSession? tls = null;
try
{
tls = await r.GetService<TlsService>().AcceptAsync(conn);
// The decrypt pump lives in the pipe, so everything below is the cleartext sample -
// and the pipe resumes inline on this reactor, so the upstream call below does too.
await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);
// Buffered + async: each stream dispatches with its body assembled, and the handler
// may await - the upstream round trip resumes inline on this reactor. Concurrent
// streams interleave here, which is exactly why the h1 pool has to be deep.
await new Http2Connection(pipe).RunBufferedAsync(async request =>
{
try
{
// Method, path and body forward as the bytes they already are: h2 decoded
// them out of HPACK, and the h1 client writes them back out as a request line.
using HttpClientResponse response = await client.SendAsync(new HttpClientRequest(
request.Method, request.Path) { Body = request.Body });
// Copy before Dispose: the response arena is freed then, and the h2 response
// is framed only AFTER this handler returns. A real proxy would also
// drop hop-by-hop headers - Connection, Keep-Alive, Transfer-Encoding are all
// illegal in h2 and would be a protocol error to forward.
var proxied = new Http2Response
{
Status = response.Status,
Body = response.Body.ToArray(),
};
if (response.TryGetHeader("content-type"u8, out ReadOnlyMemory<byte> contentType))
{
proxied.Headers.Add("content-type"u8.ToArray(), contentType.ToArray());
}
return proxied;
}
catch (Exception e)
{
// Upstream down is a gateway error on this stream, not a dead h2 connection:
// every other stream on it keeps working. A refused certificate arrives the
// same way - the handshake is part of opening the upstream connection.
return new Http2Response
{
Status = 502,
Body = Encoding.ASCII.GetBytes($"upstream failed: {e.Message}\n"),
};
}
});
}
catch (Exception e)
{
Console.Error.WriteLine($"[proxy h2->h1] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[proxy h2->h1] {config.ReactorCount} reactors, h2 over TLS on :{config.Tcp!.Port} "
+ $"-> https://{upstreamName} ({upstreamHost}:{upstreamPort}), "
+ $"{upstreamPool} connections each, "
+ $"verify={(upstreamInsecure ? "OFF" : upstreamCa ?? "system store")}");
foreach (Thread thread in threads)
{
thread.Join();
}The classic edge: clients get multiplexing and header compression, the origin keeps speaking what it already speaks. The one combination here whose pool must size for concurrency - a hundred h2 streams need a hundred h1 connections, because h1 has no multiplexing to borrow.
// dotnet add package ioxide ioxide.ngtcp2 ioxide.nghttp3 ioxide.httpclient
// PLAYGROUND_PORT=8444 dotnet run --project Playground/Tls/Ktls # a TLS origin
// curl --http3-only -k https://127.0.0.1:8443/
using ioxide;
using ioxide.httpclient;
using ioxide.nghttp3;
using ioxide.ngtcp2;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
int reactors = Environment.ProcessorCount;
ushort quicPort = 8443;
int udpRecvSlots = 16;
string upstreamHost = "127.0.0.1";
ushort upstreamPort = 8444;
string upstreamSni = "localhost";
int upstreamPool = 8;
string? upstreamCa = null;
bool upstreamInsecure = false;
string? certOverride = null; // a real PEM pair, or null to self-sign on first run
string? keyOverride = null;
// ─────────────────────────────────────────────────────────────────────────────────────────────
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
using var engine = new QuicEngine(certPath, keyPath,
cidLength: 8, // CID bytes this endpoint mints (1..20)
alpn: ["h3"], // the only protocol offered (else no_application_protocol)
maxSendRetentionBytes: 16L << 20); // per-connection send-retention high-water (default 16 MiB)
var config = new ServerConfig
{
ReactorCount = reactors,
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Tcp = null, // the proxy serves h3 only; its TCP sockets are all outbound
Udp = new UdpOptions
{
RecvSlots = udpRecvSlots, // multishot recv slots per reactor (datagrams in flight)
Gro = true, // UDP_GRO: coalesce datagrams into one recv (fewer syscalls)
},
Quic = new QuicOptions
{
Port = quicPort, // h3 over UDP - the QUIC listener
LocalCidLength = 8, // CID bytes this endpoint mints (must match the engine)
IdleTimeoutMs = 60_000, // transport idle backstop; 0 disables sweep eviction
ConnectionFactory = engine.CreateFactory(), // adopts new connections into the engine
// Where a moved client's packets go when several reactors share the port. Forward costs
// nothing until a client actually changes address; KernelFilter has the kernel route by
// connection id instead, which costs a little on every packet. See /how-ioxide-does-h3.
Routing = QuicRouting.Forward,
},
};
string upstreamName = upstreamSni; // sent as SNI, checked against the cert
// The playground's origins use a self-signed cert, so trust that file rather than the system
// store. PLAYGROUND_UPSTREAM_CA points at a private CA instead; PLAYGROUND_UPSTREAM_INSECURE=1
// skips verification, which leaves the hop encrypted but UNAUTHENTICATED - anything in the path
// can present its own certificate and rewrite the whole exchange.
upstreamCa ??= certPath;
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r =>
{
// Outbound: one context per reactor, shared by that reactor's pool - it holds no
// per-connection state, so every connection opened from it gets its own SSL.
HttpClientPool.Start(r, new HttpClientOptions
{
Host = upstreamHost, // origin address (IPv4 literal; DNS would block the reactor)
Port = upstreamPort, // origin port
PoolSize = upstreamPool, // connections kept open; round-robin, one request each
MaxResponseBytes = 8 * 1024 * 1024, // per-request ceiling for headers + body
SendBufferSize = 16 * 1024, // per-connection send buffer
ReceiveBufferSize = 16 * 1024, // per-connection recv buffer (grows to MaxResponseBytes)
AcquireTimeoutMs = 10_000, // how long a request waits for a free connection
Tls = TlsClientContext.Create(new TlsClientOptions
{
ServerName = upstreamName, // sent as SNI, checked against the cert
AlpnProtocols = ["http/1.1"], // ALPN offer, most preferred first
VerifyCertificate = !upstreamInsecure, // off = encrypted but UNAUTHENTICATED
CaFile = upstreamInsecure ? null : upstreamCa, // PEM trust anchors; null = system store
MinimumVersion = OpenSslVersions.Tls12, // lowest TLS accepted; Tls13 = 1.3 only
HandshakeTimeoutMs = 10_000, // handshake deadline
}),
});
};
reactor.QuicHandle = (r, conn) =>
{
HttpClientPool client = r.GetService<HttpClientPool>();
// Buffered + async: each request dispatches with its body assembled, and the handler may
// await - the upstream round trip resumes inline on this reactor.
return new Nghttp3Connection(conn).RunBufferedAsync(async request =>
{
try
{
// Method, path and body forward as the bytes they already are. The request's
// memories stay valid across the await - the handler owns them until it returns.
using HttpClientResponse response = await client.SendAsync(new HttpClientRequest(
request.Method, request.Path) { Body = request.Body });
// The upstream response is arena-backed and freed at Dispose, but nghttp3 copies
// the h3 response only AFTER this handler returns - so take a copy now. A real
// proxy would also filter hop-by-hop headers here.
var proxied = new Nghttp3Response
{
Status = response.Status,
Body = response.Body.ToArray(),
};
if (response.TryGetHeader("content-type"u8, out ReadOnlyMemory<byte> contentType))
{
proxied.Headers.Add("content-type"u8.ToArray(), contentType.ToArray());
}
return proxied;
}
catch (Exception e)
{
// Upstream down is a gateway error, not a dead h3 connection.
return new Nghttp3Response
{
Status = 502,
Body = System.Text.Encoding.ASCII.GetBytes($"upstream failed: {e.Message}\n"),
};
}
});
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[proxy h3->h1] {config.ReactorCount} reactors, h3 on udp :{config.Quic!.Port} "
+ $"-> https://{upstreamName} ({upstreamHost}:{upstreamPort}), "
+ $"verify={(upstreamInsecure ? "OFF" : upstreamCa ?? "system store")}");
foreach (Thread thread in threads)
{
thread.Join();
}QUIC-only frontend: Tcp = null, so every TCP socket this process owns is outbound. The h3 server and the h1 client share the reactor and nothing else.
// dotnet add package ioxide
// dotnet add package ioxide.httpclient
// dotnet run -c Release --project Playground/Http3/Nghttp3Request # an origin advertising h3
// PLAYGROUND_UPSTREAM_PORT=8080 dotnet run -c Release --project Playground/Clients/Http
using System.Text;
using ioxide;
using ioxide.httpclient;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8090; // http://127.0.0.1:8090/ - what this proxy serves
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// The origin this forwards to. The host must be an IPv4 literal - a DNS lookup would block the
// reactor - and PoolSize is per protocol, per reactor.
string upstreamHost = "127.0.0.1";
ushort upstreamPort = 8081;
int upstreamPool = 8;
// The path every inbound request is forwarded to, whatever was asked for.
string upstreamPath = "/";
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors,
RingEntries = 8192, // io_uring SQ/CQ depth
DualStack = false, // true = one AF_INET6 socket takes v6 + v4-mapped
// Shared recv ring - the default mode, used while Incremental is null.
RecvBufferSize = 32 * 1024,
RecvSlots = 4096,
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // more listeners; conn.ListenerPort says which
ListenBacklog = 1024, // accept queue per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer
PoolMax = 1024, // connection objects recycled per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Segmented = chained slabs, one SENDMSG
ZeroCopySend = false, // SEND_ZC; only pays off on large responses
RecvQueueEntries = 64, // per-connection SPSC queue, power of two
},
};
var upstream = new HttpClientOptions
{
Host = upstreamHost,
Port = upstreamPort,
PoolSize = upstreamPool,
};
var threads = new Thread[config.ReactorCount];
for (int id = 0; id < threads.Length; id++)
{
var reactor = new Reactor(id, config);
// The pool opens its connections on THIS reactor's ring, which is what keeps both hops on
// one thread.
reactor.OnStart = r => HttpClientPool.Start(r, upstream);
reactor.TcpHandle = async (r, conn) =>
{
HttpClientPool http = r.GetService<HttpClientPool>();
try
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
// This sample forwards a fixed path, so the request bytes are drained rather than
// parsed - the Proxy/* samples show real target extraction.
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer) conn.ReturnBuffer(in item);
}
try
{
// The response owns its bytes - dispose it when done.
using HttpClientResponse response = await http.GetAsync(upstreamPath);
conn.Write(Encoding.ASCII.GetBytes(
$"HTTP/1.1 {response.Status} OK\r\nContent-Length: {response.Body.Length}\r\n\r\n"));
conn.Write(response.Body.Span); // bytes straight through, no decode
}
catch (Exception e)
{
// A dead origin surfaces here rather than hanging: the pool bounds the acquire.
byte[] message = Encoding.ASCII.GetBytes($"upstream: {e.Message}");
conn.Write(Encoding.ASCII.GetBytes(
$"HTTP/1.1 502 Bad Gateway\r\nContent-Length: {message.Length}\r\n\r\n"));
conn.Write(message);
}
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[id] = new Thread(reactor.Run) { Name = $"reactor-{id}" };
threads[id].Start();
}
Console.WriteLine($"[http] {config.ReactorCount} reactors on :{config.Tcp!.Port} -> "
+ $"{upstream.Host}:{upstream.Port} ({upstream.PoolSize} conns per reactor)");
foreach (Thread thread in threads)
{
thread.Join();
}Both hops - the inbound connection and the outbound call - ride this reactor's ring and resume inline, so a request never leaves the thread it arrived on. The knob is Policy: Negotiate starts on HTTP/1.1 and moves to HTTP/3 once the origin advertises it via Alt-Svc, so the upgrade costs nothing at the call site. The nine samples pin a protocol instead.
// dotnet add package ioxide
// dotnet add package ioxide.pg
// curl http://127.0.0.1:8080/
using System.Text;
using ioxide;
using ioxide.pg;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080; // http://127.0.0.1:8080/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// The Postgres this queries. The host must be an IPv4 literal - resolving a name would block the
// reactor - and the pool is per reactor, not global. A null password means trust auth; set one and
// the driver does SCRAM-SHA-256. CommandTimeoutMs tears down the connection when the oldest
// in-flight command passes it; 0 disables that.
string pgHost = "127.0.0.1";
ushort pgPort = 5432;
string pgUser = "bench";
string? pgPassword = null;
string pgDatabase = "bench";
int pgPoolSize = 4;
int pgTimeoutMs = 30_000;
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var pgOptions = new PgOptions
{
Host = pgHost, // IPv4 literal - resolve names up front, DNS blocks the reactor
Port = pgPort, // Postgres wire port
User = pgUser, // required
Password = pgPassword, // null = trust auth; else SCRAM-SHA-256
Database = pgDatabase, // required
PoolSize = pgPoolSize, // per reactor, not global
MaxReceiveBytes = 64 * 1024 * 1024, // ceiling on one backend message; buffer grows to this (64 MB)
CommandTimeoutMs = pgTimeoutMs, // oldest in-flight command past this -> torn down; 0 disables
};
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
// OnStart runs ON the reactor thread, so every connection the pool opens belongs to THIS
// reactor's ring. Start registers the pool as a reactor service; the handler fetches it back
// with GetService. One pool per reactor, no sharing, no lock.
reactor.OnStart = r => PgPool.Start(r, pgOptions);
reactor.TcpHandle = async (r, conn) =>
{
PgPool pg = r.GetService<PgPool>();
try
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
// Drain the recv, pulling the request target out on the way past. The buffers must
// go back to the ring, so read what you need before returning them.
string path = "/";
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
if (TryReadTarget(item.AsSpan(), out ReadOnlySpan<byte> target))
{
path = Encoding.ASCII.GetString(target);
}
conn.ReturnBuffer(in item);
}
}
try
{
string responseBody;
if (path.StartsWith("/add/", StringComparison.Ordinal))
{
// Parameterized (prepared): each distinct SQL text is Parse'd once per
// connection, then Bind/Execute'd on reuse - the server plans it once.
long n = long.TryParse(path["/add/".Length..], out long parsed) ? parsed : 41;
PgResult result = await pg.QueryAsync("SELECT $1::bigint + 1", [PgParam.Int(n)]);
responseBody = $"{n}+1={result.Value}";
}
else if (path.StartsWith("/upper/", StringComparison.Ordinal))
{
// A text parameter rides the same prepared path - no SQL escaping, ever.
PgResult result = await pg.QueryAsync("SELECT upper($1)", [PgParam.Text(path["/upper/".Length..])]);
responseBody = $"upper={result.Value}";
}
else if (path.StartsWith("/rows/", StringComparison.Ordinal))
{
// Multiple rows stream through the inline callback as they arrive - fields
// read by column name, no materialized result set.
long count = Math.Clamp(long.TryParse(path["/rows/".Length..], out long c) ? c : 5, 1, 100);
var values = new StringBuilder();
int rows = await pg.QueryRowsAsync(
$"SELECT n, n * n AS square FROM generate_series(1, {count}) AS n",
row => values.Append(Encoding.ASCII.GetString(row.Field("n")))
.Append("^2=")
.Append(Encoding.ASCII.GetString(row.Field("square")))
.Append(' '));
responseBody = $"rows={rows}: {values}";
}
else
{
string sql = path switch
{
"/sleep" => "SELECT 42 FROM pg_sleep(0.1)",
"/hang" => "SELECT pg_sleep(10)", // outlives PLAYGROUND_PG_TIMEOUT -> torn down
"/err" => "SELECT * FROM this_table_does_not_exist",
_ => "SELECT 42",
};
// io_uring send + recv to Postgres, on the same ring that accepted the
// request. The continuation resumes inline, on this thread.
PgResult result = await pg.QueryAsync(sql);
responseBody = $"db={result.Value}";
}
conn.Write(Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {responseBody.Length}\r\n\r\n{responseBody}"));
}
catch (PgException e)
{
// A server error is just an exception. The connection goes back to the pool
// usable - Postgres told us about a bad query, it didn't break the socket.
Console.Error.WriteLine($"[pg] query failed: {e.Message}");
conn.Write("HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n"u8);
}
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[pg] {config.ReactorCount} reactors on :{config.Tcp.Port} "
+ $"-> {pgOptions.Host}:{pgOptions.Port}, {pgOptions.PoolSize} connections each");
foreach (Thread thread in threads)
{
thread.Join();
}
// "GET /sleep?x=1 HTTP/1.1" -> "/sleep". Your framework of choice would do this for you; ioxide
// deliberately doesn't, so here it is in full.
static bool TryReadTarget(ReadOnlySpan<byte> request, out ReadOnlySpan<byte> target)
{
target = default;
int firstSpace = request.IndexOf((byte)' ');
if (firstSpace < 0) return false;
ReadOnlySpan<byte> afterMethod = request[(firstSpace + 1)..];
int secondSpace = afterMethod.IndexOf((byte)' ');
if (secondSpace < 0) return false;
target = afterMethod[..secondSpace];
int query = target.IndexOf((byte)'?');
if (query >= 0) target = target[..query];
return true;
}One pool per reactor, opened on the reactor thread, so a query rides the same ring that accepted the request and resumes the handler inline. The host must be an IPv4 literal - a DNS lookup would block the reactor.
// dotnet add package ioxide
// dotnet add package ioxide.redis
// curl http://127.0.0.1:8080/
using System.Text;
using ioxide;
using ioxide.redis;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
ushort port = 8080; // http://127.0.0.1:8080/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
// The Redis this talks to. IPv4 literal for the same reason as everywhere else - a DNS lookup
// would block the reactor - and the pool is per reactor. A null password means no AUTH.
string redisHost = "127.0.0.1";
ushort redisPort = 6379;
string? redisPassword = null;
int redisPoolSize = 4;
// ─────────────────────────────────────────────────────────────────────────────────────────────
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var redisOptions = new RedisOptions
{
Host = redisHost, // IPv4 literal - resolve names up front, DNS blocks the reactor
Port = redisPort, // Redis wire port
Password = redisPassword, // AUTH password; null = no auth
User = null, // ACL username (Redis 6+); null = default user
Database = 0, // logical DB index to SELECT on connect
PoolSize = redisPoolSize, // per reactor, not global
CommandTimeoutMs = 30_000, // oldest in-flight command past this -> torn down; 0 disables
};
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
// The pool lives on the reactor: its connections ride this reactor's ring, so a query is
// an io_uring send + recv with the continuation resuming right here.
reactor.OnStart = r =>
{
RedisPool pool = RedisPool.Start(r, redisOptions);
_ = SeedAsync(pool); // the "/" route reads this key; seed it so the demo answers
};
reactor.TcpHandle = async (r, conn) =>
{
RedisPool pool = r.GetService<RedisPool>();
try
{
while (true)
{
RecvSnapshot snapshot = await conn.ReadAsync();
string path = ReadPath(conn, snapshot);
try
{
string body = path switch
{
// Cache-aside: try the key; on a miss compute, SET with a TTL, return.
var p when p.StartsWith("/cache/", StringComparison.Ordinal)
=> await CacheAside(pool, p["/cache/".Length..]),
// One RESP reply type per route, through the generic ExecuteAsync.
"/incr" => $"counter = {(await pool.ExecuteAsync("INCR", "play:counter")).AsInteger()}",
"/hash" => await Hash(pool),
"/list" => await List(pool),
// Several commands, one round trip, replies in order.
"/pipeline" => await Pipeline(pool),
// The hot path: a single GET per request. wrk this one.
_ => $"redis = {await pool.GetAsync("play:bench") ?? "(nil)"}",
};
WriteText(conn, "200 OK", body);
}
catch (RedisException e)
{
// A server error is an exception, not a broken socket: the connection has
// resynced and goes back to the pool usable.
WriteText(conn, "500 Internal Server Error", e.Message);
}
await conn.FlushAsync();
if (snapshot.IsClosed) return;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
Console.WriteLine($"[redis] {config.ReactorCount} reactors on :{config.Tcp!.Port} "
+ $"-> {redisOptions.Host}:{redisOptions.Port}, {redisOptions.PoolSize} connections each");
foreach (Thread thread in threads)
{
thread.Join();
}
static async Task SeedAsync(RedisPool pool)
{
try
{
await pool.ExecuteAsync("SET", "play:bench", "42");
}
catch (Exception e) { Console.Error.WriteLine($"[redis] seed failed: {e.Message}"); }
}
static async Task<string> CacheAside(RedisPool pool, string key)
{
key = "play:cache:" + key;
string? cached = await pool.GetAsync(key);
if (cached != null)
{
return $"hit {key} = {cached}";
}
string value = $"value-{key.Length}"; // stands in for the expensive computation
await pool.SetExAsync(key, value, seconds: 60);
return $"miss {key} = {value} (cached 60s)";
}
// RESP array of bulk strings: field, value, field, value, ...
static async Task<string> Hash(RedisPool pool)
{
await pool.ExecuteAsync("HSET", "play:hash", "lang", "csharp", "engine", "io_uring");
RespValue all = await pool.ExecuteAsync("HGETALL", "play:hash");
return "hash = [" + string.Join(", ", all.Items.Select(v => v.AsString())) + "]";
}
static async Task<string> List(RedisPool pool)
{
await pool.ExecuteAsync("DEL", "play:list");
await pool.ExecuteAsync("RPUSH", "play:list", "a", "b", "c");
RespValue items = await pool.ExecuteAsync("LRANGE", "play:list", "0", "-1");
return "list = [" + string.Join(", ", items.Items.Select(v => v.AsString())) + "]";
}
static async Task<string> Pipeline(RedisPool pool)
{
RespValue[] replies = await pool.PipelineAsync(
new RedisCommand("SET", "play:pipe", "1"),
new RedisCommand("INCR", "play:pipe"),
new RedisCommand("GET", "play:pipe"));
return $"set={replies[0].AsString()} incr={replies[1].AsInteger()} get={replies[2].AsString()}";
}
// "GET /cache/x HTTP/1.1" -> "/cache/x", draining the recv on the way past.
static string ReadPath(TcpConnection conn, RecvSnapshot snapshot)
{
string path = "/";
while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
ReadOnlySpan<byte> request = item.AsSpan();
int firstSpace = request.IndexOf((byte)' ');
if (firstSpace >= 0)
{
ReadOnlySpan<byte> rest = request[(firstSpace + 1)..];
int secondSpace = rest.IndexOf((byte)' ');
if (secondSpace > 0)
{
ReadOnlySpan<byte> target = rest[..secondSpace];
int query = target.IndexOf((byte)'?');
path = Encoding.ASCII.GetString(query >= 0 ? target[..query] : target);
}
}
conn.ReturnBuffer(in item);
}
}
return path;
}
static void WriteText(TcpConnection conn, string status, string body)
=> conn.Write(Encoding.ASCII.GetBytes(
$"HTTP/1.1 {status}\r\nContent-Type: text/plain\r\nContent-Length: {body.Length}\r\n\r\n{body}"));RESP2 with pipelining, one pool per reactor. Same shape as - the client is ring-native, so the round trip never leaves the core the request landed on.
// dotnet add package ioxide
// dotnet add package ioxide.file
// curl http://127.0.0.1:8080/index.html
// PLAYGROUND_FILE_TLS=ktls # then curl -k https://127.0.0.1:8443/index.html
using System.Buffers.Text;
using System.Runtime.InteropServices;
using ioxide;
using ioxide.file;
using ioxide.tls;
using ioxide.utils;
// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
string dir = "/tmp/ioxide-assets"; // served root, walked once into a snapshot
// Which transport terminates here. This is what decides whether the slab path below is legal
// at all - see the header. none | ktls | openssl
string tlsMode = "none";
// Read the file into a pooled buffer and copy it into the slab, instead of reading it straight
// into the slab. Forced for openssl, which cannot use the slab path.
bool buffered = false;
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
bool tlsOn = tlsMode is "ktls" or "openssl";
bool ktls = tlsMode == "ktls";
ushort port = tlsOn ? (ushort)8443 : (ushort)8080;
// ─────────────────────────────────────────────────────────────────────────────────────────────
// The constraint this sample exists to show, enforced instead of described: with OpenSSL owning
// transmit the slab must hold CIPHERTEXT, so a file read straight into it would go out in the
// clear. There is no slab path for that backend - refuse rather than serve cleartext.
if (tlsMode == "openssl" && !buffered)
{
Console.Error.WriteLine("[file] openssl TLS has no slab path: the slab must hold ciphertext, so "
+ "the body has to pass through TlsSession.Write. Use "
+ "PLAYGROUND_FILE_BUFFERED=1, or PLAYGROUND_FILE_TLS=ktls for the slab.");
Environment.Exit(1);
}
const string certPath = "cert.pem"; // any PEM pair
const string keyPath = "key.pem";
// Built once for the whole process, BEFORE the reactors start: every reactor shares this snapshot.
var assets = new StaticAssets(dir); // served root (PLAYGROUND_DIR), walked once into a snapshot
var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn
Tcp = new TcpOptions
{
Port = port, // 8080 cleartext, 8443 with TLS
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer; ReadFileAsync grows it to fit a bigger file
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};
var threads = new Thread[config.ReactorCount];
for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);
reactor.OnStart = r =>
{
r.AddService(assets); // the shared snapshot
// The buffer path needs a pool of native read buffers; the slab path reads into the
// connection's own write slab and needs none, but registering it is harmless either way.
AssetReader.CreatePool(r,
readers: 4, // concurrent ring reads bounded per reactor (pool size)
bufferBytes: 1 << 20); // native read buffer per reader (1 MiB) - size for the largest asset
if (tlsOn)
{
TlsService.Start(r, new TlsOptions
{
CertificatePath = certPath,
KeyPath = keyPath,
// The whole point: with transmit in the kernel the slab carries PLAINTEXT and the
// kernel makes the records, so ReadFileAsync may still read the file straight into
// it. With this false, OpenSSL encrypts and the slab must hold ciphertext - which
// is why the slab path is refused above. Receive is OpenSSL either way.
KernelTx = ktls,
});
}
};
reactor.TcpHandle = async (r, conn) =>
{
StaticAssets snapshot = r.GetService<StaticAssets>();
RingPool<AssetReader> readers = r.GetService<RingPool<AssetReader>>();
TlsSession? tls = null;
// Request bytes waiting to be framed. Both transports go through this, so the two differ
// only in how bytes get IN - otherwise the plaintext arm would answer once per recv batch
// and the TLS arm once per record, and the two would not be comparable.
var carry = new Carry();
try
{
if (tlsOn)
{
tls = await r.GetService<TlsService>().AcceptAsync(conn);
// A request can ride in with the handshake's final flight; serve it before parking
// in ReadAsync or the client waits on a response that never comes.
carry.Append(tls.DrainPlaintext());
await ServeAsync(conn, tls, carry, snapshot, readers, buffered);
}
while (true)
{
RecvSnapshot recv = await conn.ReadAsync();
while (conn.TryGetItem(recv, out SpscRecvRing.Item item))
{
if (item.HasBuffer)
{
Append(tls, in item, carry);
conn.ReturnBuffer(in item);
}
}
await ServeAsync(conn, tls, carry, snapshot, readers, buffered);
if (recv.IsClosed || (tls?.Closed ?? false)) return;
conn.ResetRead();
}
}
catch (Exception e)
{
Console.Error.WriteLine($"[file] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};
threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}
// Reload on SIGHUP (kill -HUP <pid>): a fresh snapshot is opened and swapped in atomically, and the
// old descriptors close after a grace period - so in-flight requests finish on the bytes they
// started with.
using var reload = PosixSignalRegistration.Create(PosixSignal.SIGHUP, context =>
{
context.Cancel = true; // handle it; don't let the default action terminate us
assets.Reload();
Console.WriteLine($"[file] reloaded - now serving {assets.Count} files");
});
Console.WriteLine($"[file] {config.ReactorCount} reactors on :{config.Tcp.Port} - "
+ $"{assets.Count} files under {assets.RootDir} "
+ $"({(buffered ? "buffer path" : "slab path")}, "
+ $"tls={tlsMode}, nothing cached)");
foreach (Thread thread in threads)
{
thread.Join();
}
// Bytes in. Cleartext appends the ring buffer as-is; TLS decrypts it first. Decrypt takes a raw
// pointer because the buffer belongs to the ring, and the pointer work has to stay out of the
// async handler, which cannot contain unsafe code.
static unsafe void Append(TlsSession? tls, in SpscRecvRing.Item item, Carry carry)
=> carry.Append(tls is null ? item.AsSpan() : tls.Decrypt(item.Ptr, item.Len));
// One response per COMPLETE request in the carry, none for a partial one. Under kTLS the response
// bytes go into the slab as plaintext and the kernel encrypts them on send, so both the slab and
// buffer paths below are written exactly as they are for cleartext.
static async Task ServeAsync(TcpConnection conn, TlsSession? tls, Carry carry, StaticAssets snapshot,
RingPool<AssetReader> readers, bool buffered)
{
int end;
while ((end = carry.Span.IndexOf("\r\n\r\n"u8)) >= 0)
{
bool found = false;
AssetCache.Asset asset = default;
// The lease pins the snapshot for the request, so a concurrent Reload() cannot close the
// fd out from under an in-flight read.
using (StaticAssets.Lease lease = snapshot.Acquire())
{
if (TryReadTarget(carry.Span[..end], out ReadOnlySpan<byte> target))
{
found = lease.TryGet(target, out asset);
}
carry.Consume(end + 4);
if (found)
{
if (buffered)
{
await SendBufferedAsync(conn, tls, readers, asset);
}
else
{
await SendSlabAsync(conn, tls, asset);
}
}
}
if (!found)
{
Emit(conn, tls, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n"u8);
await conn.FlushAsync();
}
}
}
// Cleartext fast path: read the whole file STRAIGHT INTO the write slab (it grows to fit), right
// after the header, so header + body leave in one flush - no reader buffer, no copy. Ideal for the
// typical multi-KB web asset; a very large file grows the slab by its full size, so stream those
// with the buffer path instead.
static async Task SendSlabAsync(TcpConnection conn, TlsSession? tls, AssetCache.Asset asset)
{
Span<byte> header = stackalloc byte[256];
Emit(conn, tls, header[..WriteHeader(header, asset.Path, asset.Length)]);
// io_uring positional read into the slab at the current tail; AdvanceWrite commits the bytes.
int n = await conn.ReadFileAsync(asset.Fd, (int)asset.Length, fileOffset: 0);
conn.AdvanceWrite(n);
await conn.FlushAsync();
}
// Buffer path: read into a reader buffer, then move those bytes on. The copy into the slab stands in
// for the transform a TLS connection does here instead - tls.Write(conn, reader.Buffer[..n]), which
// encrypts into the slab. Files bigger than the buffer take several reads at advancing offsets,
// one flush each.
static async Task SendBufferedAsync(TcpConnection conn, TlsSession? tls, RingPool<AssetReader> readers, AssetCache.Asset asset)
{
AssetReader reader = await readers.RentAsync();
try
{
int first = await reader.ReadAsync(asset.Fd, offset: 0); // io_uring positional read
if (first < 0)
{
Emit(conn, tls, "HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n"u8);
await conn.FlushAsync();
return;
}
// Header + first chunk go out together in one flush.
Span<byte> header = stackalloc byte[256];
Emit(conn, tls, header[..WriteHeader(header, asset.Path, asset.Length)]);
WriteNative(conn, tls, reader.Buffer, first);
await conn.FlushAsync();
long offset = first;
while (offset < asset.Length)
{
int read = await reader.ReadAsync(asset.Fd, offset);
if (read <= 0) break; // EOF or mid-stream error; the response is already committed
WriteNative(conn, tls, reader.Buffer, read);
await conn.FlushAsync();
offset += read;
}
}
finally
{
readers.Return(reader);
}
}
// Copy native memory (a ring-read buffer) into the connection's write slab in one go.
static unsafe void WriteNative(TcpConnection conn, TlsSession? tls, nint data, int length)
=> Emit(conn, tls, new ReadOnlySpan<byte>((void*)data, length));
// Every response byte goes through here. TlsSession.Write is correct under BOTH backends - it
// writes plaintext to the slab when the kernel encrypts, and encrypts into the slab when OpenSSL
// does - so no call site has to know which one is in play. A bare conn.Write would be right only
// for cleartext and kTLS, and silently wrong for OpenSSL.
static void Emit(TcpConnection conn, TlsSession? tls, ReadOnlySpan<byte> bytes)
{
if (tls is null) conn.Write(bytes);
else tls.Write(conn, bytes);
}
// Write the 200 status line + Content-Type + Content-Length for this file.
static int WriteHeader(Span<byte> destination, string path, long bodyLength)
{
int h = 0;
h += Copy(destination[h..], "HTTP/1.1 200 OK\r\nContent-Type: "u8);
h += Copy(destination[h..], MimeFor(path));
h += Copy(destination[h..], "\r\nContent-Length: "u8);
Utf8Formatter.TryFormat(bodyLength, destination[h..], out int digits);
h += digits;
h += Copy(destination[h..], "\r\n\r\n"u8);
return h;
}
static int Copy(Span<byte> destination, ReadOnlySpan<byte> source)
{
source.CopyTo(destination);
return source.Length;
}
static ReadOnlySpan<byte> MimeFor(string path) => Path.GetExtension(path) switch
{
".html" => "text/html"u8,
".css" => "text/css"u8,
".js" => "application/javascript"u8,
".json" => "application/json"u8,
".svg" => "image/svg+xml"u8,
".png" => "image/png"u8,
".webp" => "image/webp"u8,
".woff2" => "font/woff2"u8,
".txt" => "text/plain"u8,
_ => "application/octet-stream"u8
};
static bool TryReadTarget(ReadOnlySpan<byte> request, out ReadOnlySpan<byte> target)
{
target = default;
int firstSpace = request.IndexOf((byte)' ');
if (firstSpace < 0) return false;
ReadOnlySpan<byte> afterMethod = request[(firstSpace + 1)..];
int secondSpace = afterMethod.IndexOf((byte)' ');
if (secondSpace < 0) return false;
target = afterMethod[..secondSpace];
int query = target.IndexOf((byte)'?');
if (query >= 0) target = target[..query];
return true;
}
// Plaintext waiting to be framed into requests. TLS hands back records, not requests: a request
// split across two records decrypts as two pieces, so answering per decrypt answers twice.
sealed class Carry
{
private byte[] _buffer = new byte[8192];
private int _length;
public ReadOnlySpan<byte> Span => _buffer.AsSpan(0, _length);
public void Append(ReadOnlySpan<byte> more)
{
if (more.IsEmpty) return;
if (_length + more.Length > _buffer.Length)
{
Array.Resize(ref _buffer, Math.Max(_buffer.Length * 2, _length + more.Length));
}
more.CopyTo(_buffer.AsSpan(_length));
_length += more.Length;
}
public void Consume(int count)
{
_buffer.AsSpan(count, _length - count).CopyTo(_buffer);
_length -= count;
}
}Every file under the root is opened once and the descriptors shared across reactors, read positionally off the ring - nothing locked, nothing cached in memory. The knob worth reading is tlsMode, because it decides whether the fast path is even legal, and the whole question is what is the write slab supposed to hold when it is sent? Cleartext: the slab IS the wire. kTLS: the slab holds plaintext and the kernel makes the records - so ReadFileAsync can still read the file straight into it and the bytes are never copied. OpenSSL: the slab must hold ciphertext, so the body has to pass through TlsSession.Write, which needs a source buffer - and that buffer IS the copy. The sample refuses that combination rather than serving the file in the clear. Measured on one saturated reactor at 4 KiB, skipping the copy is worth 1.22× cleartext and 1.21× under kTLS; at 64 KiB the kernel's per-byte crypto overtakes it and OpenSSL wins outright.