fix: harden uring completion boundaries

cq_overflow_test.go

@@ -21,6 +21,7 @@ func newCQOverflowTestRing() (*ioUring, *uint32, *uint32, *uint32) {
 	overflow := uint32(0)
 
 	return &ioUring{
+			params: &ioUringParams{},
 			sq: ioUringSq{
 				kFlags: &sqFlags,
 			},

cqe_direct_darwin.go

@@ -11,7 +11,7 @@ import (
 	"code.hybscloud.com/iox"
 )
 
-// DirectCQE is a zero-overhead CQE for Direct mode operations.
+// DirectCQE is a compact copied CQE for Direct mode operations.
 type DirectCQE struct {
 	Res      int32
 	Flags    uint32
@@ -37,9 +37,9 @@ func (c *DirectCQE) BufID() uint16 { return cqeBufID(c.Flags) }
 // IsNotification reports whether this is a zero-copy notification CQE.
 func (c *DirectCQE) IsNotification() bool { return cqeIsNotification(c.Flags) }
 
-// WaitDirect retrieves completion events using Direct mode fast-path (darwin stub).
-// On single-issuer rings it is not safe for concurrent use with submit, Stop,
-// or ResizeRings; caller must serialize those operations.
+// WaitDirect retrieves completion events using the Direct mode fast path (darwin stub).
+// On single-issuer rings it is not safe for concurrent use with submit, Wait,
+// WaitDirect, WaitExtended, or Stop; caller must serialize those operations.
 func (ur *Uring) WaitDirect(cqes []DirectCQE) (int, error) {
 	if err := ur.ioUring.enter(); err != nil {
 		return 0, err

cqe_direct_linux.go

@@ -14,15 +14,14 @@ import (
 	"code.hybscloud.com/spin"
 )
 
-// DirectCQE is a zero-overhead CQE for Direct mode operations.
-// It contains the completion result and unpacked context fields
-// without any mode checking or pointer indirection.
+// DirectCQE is a compact copied CQE for Direct mode operations.
+// It stores the completion result and unpacked context fields without mode
+// checking or pointer indirection.
 //
-// Use WaitDirect when your application exclusively uses Direct mode
-// (PackDirect) for all submissions. This avoids the 3-way mode check
-// that the generic Wait/CQEView path requires per-CQE.
+// Use WaitDirect when every submitted operation uses Direct mode (PackDirect).
+// That path skips the generic Wait/CQEView mode dispatch per CQE.
 //
-// Layout: 24 bytes (fits in 1/3 cache line, no padding needed)
+// Layout: 16 bytes on supported platforms.
 type DirectCQE struct {
 	Res      int32  // Completion result (bytes transferred or negative errno)
 	Flags    uint32 // CQE flags (IORING_CQE_F_*)
@@ -68,15 +67,18 @@ func (c *DirectCQE) IsNotification() bool {
 	return cqeIsNotification(c.Flags)
 }
 
-// WaitDirect retrieves completion events using Direct mode fast-path.
+// WaitDirect retrieves completion events using the Direct mode fast path.
 // This method skips mode detection since all CQEs are assumed to be
 // from Direct mode submissions (PackDirect).
 //
-// For applications using only Direct mode, this skips the mode dispatch
-// that Wait([]CQEView) performs per CQE.
+// For applications using only Direct mode, this skips the generic mode dispatch
+// that Wait performs per CQE.
 //
-// On single-issuer rings it is not safe for concurrent use with submit, Stop,
-// or ResizeRings; caller must serialize those operations.
+// On single-issuer rings it is not safe for concurrent use with submit, Wait,
+// WaitDirect, WaitExtended, Stop, or ResizeRings; caller must serialize those
+// operations.
+// On IOPOLL rings WaitDirect also performs the nonblocking poll enter needed
+// to make completions visible.
 // Returns the number of CQEs retrieved, ErrCQOverflow when the ring enters CQ
 // overflow and no CQEs are immediately claimable, or iox.ErrWouldBlock if none
 // are available.
@@ -104,9 +106,11 @@ func (ur *ioUring) waitBatchDirect(cqes []DirectCQE) (int, error) {
 		h := atomic.LoadUint32(ur.cq.kHead)
 		t := atomic.LoadUint32(ur.cq.kTail)
 		if h == t {
-			err := ur.cqEmptyErr()
-			ur.unlockSubmitState()
-			return 0, err
+			if err := ur.observeCQEmptyLocked(); err != nil {
+				ur.unlockSubmitState()
+				return 0, err
+			}
+			continue
 		}
 
 		// Calculate batch size

cqe_extended_darwin.go

@@ -11,7 +11,7 @@ import (
 	"code.hybscloud.com/iox"
 )
 
-// ExtCQE is a zero-overhead CQE for Extended mode operations.
+// ExtCQE is a compact copied CQE for Extended mode operations.
 type ExtCQE struct {
 	Res   int32
 	Flags uint32
@@ -43,9 +43,9 @@ func (c *ExtCQE) Op() uint8 { return c.Ext.SQE.opcode }
 // FD returns the file descriptor from the stored SQE.
 func (c *ExtCQE) FD() iofd.FD { return iofd.FD(c.Ext.SQE.fd) }
 
-// WaitExtended retrieves completion events using Extended mode fast-path (darwin stub).
-// On single-issuer rings it is not safe for concurrent use with submit, Stop,
-// or ResizeRings; caller must serialize those operations.
+// WaitExtended retrieves completion events using the Extended mode fast path (darwin stub).
+// On single-issuer rings it is not safe for concurrent use with submit, Wait,
+// WaitDirect, WaitExtended, or Stop; caller must serialize those operations.
 func (ur *Uring) WaitExtended(cqes []ExtCQE) (int, error) {
 	if err := ur.ioUring.enter(); err != nil {
 		return 0, err

cqe_extended_linux.go

@@ -14,14 +14,15 @@ import (
 	"code.hybscloud.com/spin"
 )
 
-// ExtCQE is a zero-overhead CQE for Extended mode operations.
-// It provides direct access to the borrowed ExtSQE pointer without mode checking.
+// ExtCQE is a compact copied CQE for Extended mode operations.
+// It stores the completion result, CQE flags, and borrowed ExtSQE pointer
+// without mode checking.
 //
-// Use WaitExtended when your application exclusively uses Extended mode
-// (PackExtended) for all submissions. This avoids the 3-way mode check
-// that the generic Wait/CQEView path requires per-CQE.
+// Use WaitExtended when every submitted operation uses Extended mode
+// (PackExtended). That path skips the generic Wait/CQEView mode dispatch per
+// CQE.
 //
-// Layout: 16 bytes (fits in 1/4 cache line)
+// Layout: 16 bytes on supported 64-bit platforms.
 type ExtCQE struct {
 	Res   int32   // Completion result (bytes transferred or negative errno)
 	Flags uint32  // CQE flags (IORING_CQE_F_*)
@@ -89,11 +90,14 @@ func (c *ExtCQE) FD() iofd.FD {
 // This method skips mode detection since all CQEs are assumed to be
 // from Extended mode submissions (PackExtended).
 //
-// For applications using only Extended mode, this skips the mode dispatch
-// that Wait([]CQEView) performs per CQE.
+// For applications using only Extended mode, this skips the generic mode
+// dispatch that Wait performs per CQE.
 //
-// On single-issuer rings it is not safe for concurrent use with submit, Stop,
-// or ResizeRings; caller must serialize those operations.
+// On single-issuer rings it is not safe for concurrent use with submit, Wait,
+// WaitDirect, WaitExtended, Stop, or ResizeRings; caller must serialize those
+// operations.
+// On IOPOLL rings WaitExtended also performs the nonblocking poll enter needed
+// to make completions visible.
 // Caller-side completion code must keep completion referents reachable until
 // CQE reap and serialize retirement.
 // For multishot CQEs, return Ext to the pool only after !HasMore().
@@ -124,9 +128,11 @@ func (ur *ioUring) waitBatchExtended(cqes []ExtCQE) (int, error) {
 		h := atomic.LoadUint32(ur.cq.kHead)
 		t := atomic.LoadUint32(ur.cq.kTail)
 		if h == t {
-			err := ur.cqEmptyErr()
-			ur.unlockSubmitState()
-			return 0, err
+			if err := ur.observeCQEmptyLocked(); err != nil {
+				ur.unlockSubmitState()
+				return 0, err
+			}
+			continue
 		}
 
 		// Calculate batch size

cqe_view.go

@@ -12,6 +12,9 @@ import "code.hybscloud.com/iofd"
 // It exposes kernel completion facts directly and lets caller-side runtime
 // code decide how to route or interpret them. When available,
 // it also exposes the submission context that produced those facts.
+// A copied CQEView is a completion observation, not durable route state. If
+// caller code stores it beyond the current dispatch turn, caller code must keep
+// its own route state.
 //
 // # Property Patterns
 //
@@ -27,8 +30,8 @@ import "code.hybscloud.com/iofd"
 //	for i := range n {
 //	    cqe := cqes[i]
 //	    // Observe the kernel facts first.
-//	    if cqe.Res < 0 {
-//	        return fmt.Errorf("completion failed: op=%d fd=%d res=%d", cqe.Op(), cqe.FD(), cqe.Res)
+//	    if err := cqe.Err(); err != nil {
+//	        return fmt.Errorf("completion failed: op=%d fd=%d: %w", cqe.Op(), cqe.FD(), err)
 //	    }
 //	    fmt.Printf("completed op=%d on fd=%d with res=%d\n", cqe.Op(), cqe.FD(), cqe.Res)
 //	    if cqe.HasMore() {

cqe_view_darwin.go

@@ -8,7 +8,10 @@ package uring
 
 import "code.hybscloud.com/iofd"
 
-// CQEView provides a compatibility view into a completion queue entry.
+// CQEView provides a view into a completion queue entry.
+// A copied CQEView is a completion observation, not durable route state. If
+// caller code stores it beyond the current dispatch turn, caller code must keep
+// its own route state.
 type CQEView struct {
 	Res   int32
 	Flags uint32