One ring per reactor, one reactor per thread - run one per core. Every await -
an HTTP read, a Postgres query, a file read - is a completion on that ring, resumed inline
on the same thread. No thread pool on the hot path. The ring is the I/O.
You write ordinary C#. Underneath, every await is a request placed on this core's queue, and your code resumes right where it left off when the kernel answers. The same model rides TCP and QUIC (ngtcp2 + picotls bundled in the package, TLS 1.3 inside the transport) - pick a transport and a surface, pipes first. HTTP/3 ships as a layer over the QUIC surface.
var config = new ServerConfig
{
ReactorCount = Environment.ProcessorCount,
RingEntries = 8192, // io_uring submission/completion queue depth
// reactor recv machinery: io_uring provided-buffer rings
RecvBufferSize = 32 * 1024, // size of each receive buffer
BufferRingEntries = 4096, // receive buffers shared by a reactor's connections
Incremental = false, // kernel 6.12+: per-connection buffer rings
MaxConnections = 4096, // one buffer group per live connection
ConnBufRingEntries = 16, // buffers per connection ring
IncRecvBufferSize = 4096, // bytes per incremental buffer
Tcp = new TcpOptions
{
Port = 8080,
WriteSlabSize = 16 * 1024, // per-connection write buffer
PoolMax = 1024, // connection objects recycled per reactor
RecvQueueEntries = 64, // per-connection queue of received slices
},
};
var reactor = new Reactor(id, config);
// Clients opened here ride this reactor's ring.
reactor.OnStart = r => PgPool.Start(r, pgOptions);
reactor.TcpHandle = async (r, conn) =>
{
var pool = r.GetService<PgPool>();
// Zero-copy pipes over the ring. The reader owns the carry: unconsumed
// bytes are held across reads (no copy, no slab) and buffers return to
// the ring automatically once fully consumed. The writer stages into the
// connection's write slab.
var reader = new TcpConnectionPipeReader(conn);
var writer = new TcpConnectionPipeWriter(conn);
while (true)
{
// io_uring recv - resumes inline on the reactor.
var result = await reader.ReadAsync();
var buffer = result.Buffer; // every unconsumed byte received so far
// Walk every complete request; a partial one stays buffered for the
// next read. TryParseRequest, Request, and SqlFor are YOUR code -
// ioxide hands you raw bytes and stays out of HTTP.
bool respond = false;
while (TryParseRequest(ref buffer, out Request request))
{
// io_uring send + recv to Postgres, on the same ring.
var rows = await pool.QueryAsync(SqlFor(request.Path));
// ioxide doesn't speak HTTP for you - you write the bytes.
string body = $"db={rows.Value}";
writer.Write(Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {body.Length}\r\n\r\n{body}"));
respond = true;
}
// Consumed bytes release their buffers; the partial tail is kept.
reader.AdvanceTo(buffer.Start, buffer.End);
if (respond) await writer.FlushAsync(); // io_uring send, once per batch
if (result.IsCompleted)
{
reader.Complete();
writer.Complete();
conn.DecRef();
return;
}
}
};
// One reactor per core.
new Thread(reactor.Run).Start();
var config = new ServerConfig
{
ReactorCount = Environment.ProcessorCount,
RingEntries = 8192, // io_uring submission/completion queue depth
// reactor recv machinery: io_uring provided-buffer rings
RecvBufferSize = 32 * 1024, // size of each receive buffer
BufferRingEntries = 4096, // receive buffers shared by a reactor's connections
Incremental = false, // kernel 6.12+: per-connection buffer rings
MaxConnections = 4096, // one buffer group per live connection
ConnBufRingEntries = 16, // buffers per connection ring
IncRecvBufferSize = 4096, // bytes per incremental buffer
Tcp = new TcpOptions
{
Port = 8080,
WriteSlabSize = 16 * 1024, // per-connection write buffer
PoolMax = 1024, // connection objects recycled per reactor
RecvQueueEntries = 64, // per-connection queue of received slices
},
};
var reactor = new Reactor(id, config);
// Clients opened here ride this reactor's ring.
reactor.OnStart = r => PgPool.Start(r, pgOptions);
reactor.TcpHandle = async (r, conn) =>
{
var pool = r.GetService<PgPool>();
// Carry for bytes a read leaves behind - the head of a split request.
var inflight = new byte[16 * 1024];
int inflightTail = 0;
while (true)
{
// io_uring recv - resumes inline on the reactor.
var snapshot = await conn.ReadAsync();
var rings = conn.GetSnapshotMemories(snapshot);
if (rings.Length > 0)
{
ReadOnlySequence<byte> data;
if (inflightTail == 0 && rings.Length == 1)
{
// Hot path: one ring, no carry - a single zero-copy segment.
data = new ReadOnlySequence<byte>(rings[0].Memory);
}
else if (inflightTail == 0)
{
// Several rings, no carry - chain them, still zero-copy.
data = rings.ToReadOnlySequence();
}
else
{
// Cold path: the carry goes first so a split request reads whole.
var first = new RingSegment(inflight.AsMemory(0, inflightTail), 0);
var last = first;
for (int i = 0; i < rings.Length; i++)
last = last.Append(rings[i].Memory, rings[i].BufferId);
data = new ReadOnlySequence<byte>(first, 0, last, last.Memory.Length);
}
// Walk every complete request; stop at the first partial one.
// TryParseRequest, Request, and SqlFor are YOUR code - ioxide
// hands you raw bytes and stays out of HTTP.
long consumed = 0;
bool respond = false;
while (TryParseRequest(data.Slice(consumed), out Request request, out long length))
{
consumed += length;
// io_uring send + recv to Postgres, on the same ring.
var rows = await pool.QueryAsync(SqlFor(request.Path));
// ioxide doesn't speak HTTP for you - you write the bytes.
string body = $"db={rows.Value}";
conn.Write(Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {body.Length}\r\n\r\n{body}"));
respond = true;
}
// Whatever wasn't consumed (a partial request, or everything when
// nothing completed) moves to the front of the carry - only then
// do the buffers go back to the ring.
ReadOnlySequence<byte> rest = data.Slice(consumed);
rest.CopyTo(inflight);
inflightTail = (int)rest.Length;
conn.ReturnBuffers(rings);
if (respond) await conn.FlushAsync(); // io_uring send, once per batch
}
if (snapshot.IsClosed)
{
conn.DecRef();
return;
}
conn.ResetRead();
}
};
// One reactor per core.
new Thread(reactor.Run).Start();
var config = new ServerConfig
{
ReactorCount = Environment.ProcessorCount,
RingEntries = 8192, // io_uring submission/completion queue depth
// reactor recv machinery: per-connection buffer rings, incremental loop variant
// (RecvBufferSize / BufferRingEntries are shared-ring knobs - unused here)
Incremental = true, // per-connection buffer rings (kernel 6.12+)
MaxConnections = 4096, // one buffer group per live connection
ConnBufRingEntries = 16, // buffers per connection ring
IncRecvBufferSize = 4096, // bytes per buffer - the kernel appends into it
Tcp = new TcpOptions
{
Port = 8080,
WriteSlabSize = 16 * 1024, // per-connection write buffer
PoolMax = 1024, // connection objects recycled per reactor
RecvQueueEntries = 64, // per-connection queue of received slices
},
};
var reactor = new Reactor(id, config);
// Clients opened here ride this reactor's ring.
reactor.OnStart = r => PgPool.Start(r, pgOptions);
reactor.TcpHandle = async (r, conn) =>
{
var pool = r.GetService<PgPool>();
// Carry for bytes a read leaves behind - the head of a split request.
var inflight = new byte[16 * 1024];
int inflightTail = 0;
while (true)
{
// io_uring recv - resumes inline on the reactor.
var snapshot = await conn.ReadAsync();
// Incremental: slices often share one buffer - the kernel keeps
// appending successive recvs into it until it fills.
var rings = conn.GetSnapshotMemories(snapshot);
if (rings.Length > 0)
{
ReadOnlySequence<byte> data;
if (inflightTail == 0 && rings.Length == 1)
{
// Hot path: one ring, no carry - a single zero-copy segment.
data = new ReadOnlySequence<byte>(rings[0].Memory);
}
else if (inflightTail == 0)
{
// Several rings, no carry - chain them, still zero-copy.
data = rings.ToReadOnlySequence();
}
else
{
// Cold path: the carry goes first so a split request reads whole.
var first = new RingSegment(inflight.AsMemory(0, inflightTail), 0);
var last = first;
for (int i = 0; i < rings.Length; i++)
last = last.Append(rings[i].Memory, rings[i].BufferId);
data = new ReadOnlySequence<byte>(first, 0, last, last.Memory.Length);
}
// Walk every complete request; stop at the first partial one.
// TryParseRequest, Request, and SqlFor are YOUR code - ioxide
// hands you raw bytes and stays out of HTTP.
long consumed = 0;
bool respond = false;
while (TryParseRequest(data.Slice(consumed), out Request request, out long length))
{
consumed += length;
// io_uring send + recv to Postgres, on the same ring.
var rows = await pool.QueryAsync(SqlFor(request.Path));
// ioxide doesn't speak HTTP for you - you write the bytes.
string body = $"db={rows.Value}";
conn.Write(Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {body.Length}\r\n\r\n{body}"));
respond = true;
}
// Whatever wasn't consumed (a partial request, or everything when
// nothing completed) moves to the front of the carry - only then
// do the buffers go back to the ring.
ReadOnlySequence<byte> rest = data.Slice(consumed);
rest.CopyTo(inflight);
inflightTail = (int)rest.Length;
// Refcounted return: a buffer recycles once you AND the kernel
// are both done with it.
conn.ReturnBuffers(rings);
if (respond) await conn.FlushAsync(); // io_uring send, once per batch
}
if (snapshot.IsClosed)
{
conn.DecRef();
return;
}
conn.ResetRead();
}
};
// One reactor per core.
new Thread(reactor.Run).Start();
Shared buffer ring (default, kernel 6.1+): one pool of buffers per reactor, drawn on by every connection. One recv consumes one whole buffer no matter how few bytes arrived - simple and elastic across connections, but small messages waste buffer space, and a buffer-hoarding connection eats from everyone's pool.
Incremental (kernel 6.12+): a small buffer ring per connection, and the kernel keeps appending successive recvs into the same buffer until it fills. Small messages pack densely and every connection's memory is isolated and bounded - at the cost of refcounted recycling (a buffer returns once you and the kernel are both done with it) and a ring registration per connection.
| shared | incremental | |
|---|---|---|
| buffer ownership | one pool per reactor | one small ring per connection |
| fill behavior | one recv = one whole buffer | recvs append into the same buffer |
| best for | medium/large messages, simplicity | many connections, small messages |
| memory shape | elastic, shared | isolated, bounded per connection |
| return path | push the id back | refcounted: you + kernel both done |
| kernel | 6.1+ | 6.12+ |
The handler code is identical in both modes - flip Incremental and size the
per-connection knobs; ReturnBuffers routes the right return path internally.
The pipes tab is API sugar over either mode: the reader owns the carry for you.
// One engine for the whole server: ngtcp2 + picotls, compiled into the package.
var engine = new QuicEngine("cert.pem", "key.pem", cidLength: 8,
alpn: ["echo"]); // TLS ALPN allowlist - ["h3"] when serving HTTP/3
var config = new ServerConfig
{
ReactorCount = Environment.ProcessorCount,
Udp = new UdpOptions { RecvSlots = 16 }, // multishot UDP recv slots per reactor
Quic = new QuicOptions
{
Port = 8443, // every reactor binds it via SO_REUSEPORT
LocalCidLength = 8,
ConnectionFactory = engine.CreateFactory(),
},
};
var reactor = new Reactor(id, config);
reactor.QuicHandle = async (r, conn) =>
{
// One QUIC stream as a duplex pipe. QUIC multiplexes streams and a
// PipeReader is one byte stream, so the pipe binds to a single stream:
// the first one the peer opens (auto-bind), or an explicit id.
// Multi-stream protocols use the raw surface - that's what ioxide.h3 does.
var pipe = new QuicConnectionDualPipe(conn);
while (true)
{
// Decrypted stream bytes - resumed inline on the reactor.
var result = await pipe.Input.ReadAsync();
var buffer = result.Buffer;
// Same shape as TCP: walk every complete request, keep the partial
// tail buffered. TryParseRequest and Respond are YOUR code - ioxide
// hands you the stream's bytes and stays out of your protocol.
while (TryParseRequest(ref buffer, out Request request))
pipe.Output.Write(Respond(request));
// No parked flush: framing, encryption, pacing and retransmission
// are the engine's job - FlushAsync hands the bytes over and returns.
await pipe.Output.FlushAsync();
pipe.Input.AdvanceTo(buffer.Start, buffer.End);
if (result.IsCompleted) // peer's fin, stream reset, or connection close
break;
}
pipe.Output.Complete(); // half-close: our fin rides out
pipe.Input.Complete();
conn.DecRef();
};
// One reactor per core.
new Thread(reactor.Run).Start();
// One engine for the whole server: ngtcp2 + picotls, compiled into the package.
var engine = new QuicEngine("cert.pem", "key.pem", cidLength: 8,
alpn: ["echo"]); // TLS ALPN allowlist - ["h3"] when serving HTTP/3
var config = new ServerConfig
{
ReactorCount = Environment.ProcessorCount,
Udp = new UdpOptions { RecvSlots = 16 }, // multishot UDP recv slots per reactor
Quic = new QuicOptions
{
Port = 8443, // every reactor binds it via SO_REUSEPORT
LocalCidLength = 8,
ConnectionFactory = engine.CreateFactory(),
},
};
var reactor = new Reactor(id, config);
reactor.QuicHandle = async (r, conn) =>
{
try
{
while (true)
{
// Decrypted stream events - resumed inline on the reactor.
QuicRecvSnapshot snap = await conn.ReadAsync();
// Drain the queue: each item carries stream id + fin, so this one
// loop serves every stream the peer multiplexes - here, an echo.
// Replies queue on the engine; framing, encryption, pacing and
// retransmission are its job (no flush to await).
while (conn.TryGetItem(in snap, out QuicRecvRing.Item item))
{
conn.SendStream(item.StreamId, item.AsSpan(), item.Fin);
conn.ReturnItem(in item);
}
if (snap.IsClosed)
break;
conn.ResetRead();
}
}
finally
{
conn.DecRef();
}
};
// One reactor per core.
new Thread(reactor.Run).Start();
// ioxide.h3: nghttp3 over the QUIC surface - real HTTP/3, ALPN-pinned.
var engine = new QuicEngine("cert.pem", "key.pem", cidLength: 8, alpn: ["h3"]);
var config = new ServerConfig
{
ReactorCount = Environment.ProcessorCount,
Udp = new UdpOptions { RecvSlots = 16 }, // multishot UDP recv slots per reactor
Quic = new QuicOptions
{
Port = 8443, // every reactor binds it via SO_REUSEPORT
LocalCidLength = 8,
ConnectionFactory = engine.CreateFactory(),
},
};
var reactor = new Reactor(id, config);
// H3Connection owns the connection's read loop: it feeds stream items into
// nghttp3, assembles each request - headers, body and all - and calls your
// function once per request. QPACK, control streams, fin and stream teardown
// are its problem, not yours.
reactor.QuicHandle = (r, conn) =>
new H3Connection(conn).RunAsync(req => req.Path switch
{
"/" => H3Response.Text("hello from HTTP/3 over io_uring\n"),
"/plaintext" => H3Response.Text("Hello, World!"),
_ => H3Response.Text("not found\n", status: 404),
});
// One reactor per core.
new Thread(reactor.Run).Start();
// curl --http3-only -k https://localhost:8443/
Each of those awaits is backed by a reusable IValueTaskSource: submitting the
op parks your continuation, and the moment its completion arrives the reactor calls
SetResult - which runs your code immediately, on the same thread, with nothing
scheduled anywhere.
Every tab is runnable code: the
Examples project in the repo
builds each one - pg and raw variants plus a quic-h3 mode that serves HTTP/3 and a
pipe echo off one listener - with
benchmark results.