CI: per-test-binary DDS domains + a ReleaseSmall lane

.github/workflows/release.yml

@@ -111,6 +111,16 @@ jobs:
       - name: Test (ReleaseFast)
         run: zig build test -Doptimize=ReleaseFast
 
+      # Linux x86_64 only: `test-release-small` forces the LLVM backend to work
+      # around a Zig 0.16 self-hosted-x86_64 bug that mis-aligns read-only
+      # globals at -OReleaseSmall (see the step's comment in build.zig). That
+      # workaround path has only been exercised on Linux; broaden this to the
+      # whole matrix -- as a plain `zig build test -Doptimize=ReleaseSmall` --
+      # at the Zig 0.17 bump, when the underlying bug is gone.
+      - name: Test (ReleaseSmall, LLVM backend)
+        if: matrix.os == 'ubuntu-latest'
+        run: zig build test-release-small -Doptimize=ReleaseSmall
+
       # Binding smoke tests, mirroring ci.yml's test-other job: the release
       # gate previously ran only `zig build test` (+ReleaseFast +TSan) and
       # never built a single binding, so a PR breaking the C/C++/Java

CHANGELOG.md

@@ -8,6 +8,31 @@ see [`docs/implementation_status.md`](docs/implementation_status.md); for planne
 Dated entries (no release tags past `v0.2.1-zig.0.16.0`; `build.zig.zon` is
 `0.2.1-zig.0.16.0-dev`).
 
+## 2026-08-29
+
+- **CI flake fix — unique DDS domain per test binary.** `zig build test` runs the ~29
+  participant-creating test binaries as parallel Run steps; they all stood up participants
+  on domain 0 and contended for the same fixed RTPS ports (SPDP multicast 7400, metatraffic
+  unicast), so on slow runners (ARM64, DebugAllocator/TSan lanes) the loser hit
+  `error.BindFailed`, discovery stalled, and a loopback test timed out. `build.zig` now
+  gives each test binary's Run step a distinct `ZZDDS_TEST_DOMAIN_BASE` (per-lane counter,
+  see `addTestRun`); the new `test/support/domain.zig` reads it, and every DCPS/C-ABI test's
+  `create_participant` (plus `loopback_test` / `wlp_loopback_test`'s hand-wired
+  `UdpTransport.init` / `SpdpSedpDiscovery.init`) uses `test_domain.get()`. Distinct domains
+  map to disjoint port sets (250-port stride). `mock_loopback_test` is unchanged (it never
+  binds a real socket).
+- **CI flake fix — `tcp_transport_test` reconnect race.** "connectionGeneration increments
+  on reconnect" replaced a fixed `sleepMs(100)` (a guess at how long the local TCP stack
+  takes to process a peer FIN) with a poll-until-observed loop: drive the reconnecting
+  `send()` until a new `TcpConnection` is seen, or a 5 s deadline. Same invariant asserted,
+  no magic constant.
+- **`ReleaseSmall` CI lane.** New `zig build test-release-small` runs the unit suite at
+  `-OReleaseSmall`, wired into `run_deterministic_matrix.py` and `release.yml` (Linux
+  x86_64). It forces the LLVM backend to work around a Zig 0.16 self-hosted-x86_64 codegen
+  bug — read-only globals emitted without alignment at `-OReleaseSmall` — that is fixed on
+  Zig 0.17 master; switch back to the plain self-hosted backend at the 0.17 bump. See
+  `docs/design/ci-platform-coverage-expansion.md`.
+
 ## 2026-08-26
 
 - **`zzdds-examples` folded into `examples/`.** The standalone `zz-iot/zzdds-examples` repo

build.zig

@@ -1,6 +1,21 @@
 const std = @import("std");
 const builtin = @import("builtin");
 
+/// Run a test binary, giving it a unique DDS domain via `ZZDDS_TEST_DOMAIN_BASE`.
+///
+/// `zig build test` runs test binaries as parallel build-graph steps; the
+/// participant-creating ones would otherwise all bind domain-0 RTPS ports
+/// (SPDP multicast 7400, …) and race for them — a real CI flake, worst on the
+/// slow ARM64 / DebugAllocator / TSan lanes. `next` is a per-lane counter
+/// (each `zig build <step>` runs one lane's binaries at once); domains stay in
+/// `1..232` (`7400 + 250*domain < 65536`). See `test/support/domain.zig`.
+fn addTestRun(b: *std.Build, t: *std.Build.Step.Compile, next: *u32) *std.Build.Step.Run {
+    const r = b.addRunArtifact(t);
+    r.setEnvironmentVariable("ZZDDS_TEST_DOMAIN_BASE", b.fmt("{d}", .{next.*}));
+    next.* += 1;
+    return r;
+}
+
 /// Directories containing `jni.h`/`jni_md.h` for the JDK backing `java_path`
 /// (the `java` executable's own path, e.g. from `b.findProgram`). Resolves
 /// symlink chains (PATH/update-alternatives-style) back to a real
@@ -1121,6 +1136,13 @@ pub fn build(b: *std.Build) void {
 
     // ── Unit tests ────────────────────────────────────────────────────────────
 
+    // Shared by every DCPS test module (see `addTestRun` / test/support/domain.zig).
+    // link_libc: it reads its env var via std.c.getenv.
+    const test_domain_mod = b.createModule(.{
+        .root_source_file = b.path("test/support/domain.zig"),
+        .link_libc = true,
+    });
+
     const test_step = b.step("test", "Run Zenzen DDS tests");
 
     // emit-tests: compile all test binaries to zig-out/tests/ for kcov coverage analysis.
@@ -1293,6 +1315,7 @@ pub fn build(b: *std.Build) void {
         "test/dcps/participant_vtable_test.zig",
         "test/dcps/writer_vtable_test.zig",
     };
+    var dcps_test_domain: u32 = 1;
     for (dcps_test_files) |src| {
         const t = b.addTest(.{
             .name = std.fs.path.stem(src),
@@ -1303,11 +1326,12 @@ pub fn build(b: *std.Build) void {
                     .{ .name = "zzdds", .module = zzdds_mod },
                     .{ .name = "zzdds_generated", .module = generated_dcps_mod },
                     .{ .name = "zidl_rt", .module = zidl_rt_mod },
+                    .{ .name = "test_domain", .module = test_domain_mod },
                 },
             }),
         });
         t.root_module.link_libc = true;
-        test_step.dependOn(&b.addRunArtifact(t).step);
+        test_step.dependOn(&addTestRun(b, t, &dcps_test_domain).step);
         emit_tests_step.dependOn(&b.addInstallArtifact(t, .{
             .dest_dir = .{ .override = .{ .custom = "tests" } },
         }).step);
@@ -1398,6 +1422,27 @@ pub fn build(b: *std.Build) void {
 
     const emit_tests_llvm_step = b.step("emit-tests-llvm", "Build test binaries (LLVM backend, baseline CPU) for external DWARF-reading tools like Valgrind");
 
+    // `zig build test-release-small -Doptimize=ReleaseSmall` — run the whole
+    // unit suite at ReleaseSmall. It reuses the LLVM-backend / baseline-CPU
+    // module graph above (the same `.use_llvm = true` artifacts emit-tests-llvm
+    // installs) rather than the normal self-hosted `test` step, purely to dodge
+    // an upstream Zig bug: Zig 0.16.0's self-hosted x86_64 backend, *only* at
+    // -OReleaseSmall, emits read-only global constants (vtable structs,
+    // `CAbiViews` values) with no alignment -- `&SomeImpl.views` lands at an odd
+    // address and `zidl_rt`'s `@alignCast(box.vtable)` traps it (`panic:
+    // incorrect alignment`), crashing ~37 C-ABI tests. The LLVM backend aligns
+    // rodata correctly in every mode; Debug/ReleaseSafe/ReleaseFast on the
+    // self-hosted backend are also fine -- ReleaseSmall is the only broken cell.
+    // The bug is fixed on Zig master (0.17.0-dev.1902+); it is not in any stable
+    // release (0.16.0, tagged 2026-04-13, is latest). Minimal repro + full trail:
+    // `zz-dev/releasesmall-misaligned-rodata-investigation.md`.
+    //
+    // AT THE ZIG 0.17 BUMP: delete this step and its run-artifact deps below,
+    // and add a plain `release-small` entry to scripts/run_deterministic_matrix.py
+    // and release.yml, mirroring `release-fast` (`zig build test
+    // -Doptimize=ReleaseSmall` on the normal self-hosted backend).
+    const test_release_small_step = b.step("test-release-small", "Run the unit suite at ReleaseSmall (LLVM backend -- see comment: works around a Zig 0.16 self-hosted-backend rodata-alignment bug)");
+
     const zzdds_tests_llvm = b.addTest(.{
         .name = "zzdds_lib",
         .root_module = zzdds_mod_llvm_safe,
@@ -1406,6 +1451,7 @@ pub fn build(b: *std.Build) void {
     emit_tests_llvm_step.dependOn(&b.addInstallArtifact(zzdds_tests_llvm, .{
         .dest_dir = .{ .override = .{ .custom = "tests-llvm" } },
     }).step);
+    test_release_small_step.dependOn(&b.addRunArtifact(zzdds_tests_llvm).step);
 
     for (fuzz_test_files) |src| {
         const t = b.addTest(.{
@@ -1423,6 +1469,7 @@ pub fn build(b: *std.Build) void {
         emit_tests_llvm_step.dependOn(&b.addInstallArtifact(t, .{
             .dest_dir = .{ .override = .{ .custom = "tests-llvm" } },
         }).step);
+        test_release_small_step.dependOn(&b.addRunArtifact(t).step);
     }
 
     for (transport_test_files) |src| {
@@ -1441,6 +1488,7 @@ pub fn build(b: *std.Build) void {
         emit_tests_llvm_step.dependOn(&b.addInstallArtifact(t, .{
             .dest_dir = .{ .override = .{ .custom = "tests-llvm" } },
         }).step);
+        test_release_small_step.dependOn(&b.addRunArtifact(t).step);
     }
 
     for (discovery_test_files) |src| {
@@ -1459,6 +1507,7 @@ pub fn build(b: *std.Build) void {
         emit_tests_llvm_step.dependOn(&b.addInstallArtifact(t, .{
             .dest_dir = .{ .override = .{ .custom = "tests-llvm" } },
         }).step);
+        test_release_small_step.dependOn(&b.addRunArtifact(t).step);
     }
 
     for (rtps_test_files) |src| {
@@ -1477,8 +1526,10 @@ pub fn build(b: *std.Build) void {
         emit_tests_llvm_step.dependOn(&b.addInstallArtifact(t, .{
             .dest_dir = .{ .override = .{ .custom = "tests-llvm" } },
         }).step);
+        test_release_small_step.dependOn(&b.addRunArtifact(t).step);
     }
 
+    var rs_dcps_test_domain: u32 = 1;
     for (dcps_test_files) |src| {
         const t = b.addTest(.{
             .name = std.fs.path.stem(src),
@@ -1490,13 +1541,15 @@ pub fn build(b: *std.Build) void {
                     .{ .name = "zzdds", .module = zzdds_mod_llvm_safe },
                     .{ .name = "zzdds_generated", .module = generated_dcps_mod_llvm_safe },
                     .{ .name = "zidl_rt", .module = zidl_rt_mod_llvm_safe },
+                    .{ .name = "test_domain", .module = test_domain_mod },
                 },
             }),
         });
         t.root_module.link_libc = true;
         emit_tests_llvm_step.dependOn(&b.addInstallArtifact(t, .{
             .dest_dir = .{ .override = .{ .custom = "tests-llvm" } },
         }).step);
+        test_release_small_step.dependOn(&addTestRun(b, t, &rs_dcps_test_domain).step);
     }
 
     // ── TSan test step ────────────────────────────────────────────────────────
@@ -1582,6 +1635,7 @@ pub fn build(b: *std.Build) void {
     }
 
     // DCPS tests (TSan) — WaitSet thread test, loopback
+    var tsan_dcps_test_domain: u32 = 1;
     for (dcps_test_files) |src| {
         const t = b.addTest(.{ .root_module = b.createModule(.{
             .root_source_file = b.path(src),
@@ -1591,10 +1645,11 @@ pub fn build(b: *std.Build) void {
                 .{ .name = "zzdds", .module = zzdds_mod_tsan },
                 .{ .name = "zzdds_generated", .module = generated_dcps_mod_tsan },
                 .{ .name = "zidl_rt", .module = zidl_rt_mod_tsan },
+                .{ .name = "test_domain", .module = test_domain_mod },
             },
         }), .use_llvm = true });
         t.root_module.link_libc = true;
-        tsan_step.dependOn(&b.addRunArtifact(t).step);
+        tsan_step.dependOn(&addTestRun(b, t, &tsan_dcps_test_domain).step);
     }
 
     // `zig build test-tsan-self-check` — regression guard proving TSan

docs/design/ci-platform-coverage-expansion.md

@@ -220,18 +220,29 @@ discover it as a confusing CI failure.
 `Test (ReleaseFast)` step is in `release.yml`'s `test` job on all four platforms. No
 safety-panic-dependent tests were found; the suite passes clean.
 
-**Outcome — `ReleaseSmall` NOT landed; root-caused to an upstream Zig bug (2026-08-29):**
-`zig build test -Doptimize=ReleaseSmall` produces 37 `panic: incorrect alignment` crashes
-(`bootstrap_test` / `typesupport_test`, via `zidl-rt`'s `entity_box.zig` `unboxAsView`
-`@alignCast(box.vtable)`). Traced to **Zig 0.16.0's self-hosted x86_64 backend emitting
-read-only global constants with no alignment, only at `-OReleaseSmall`**: `&SomeImpl.views`
-(an `extern struct` with `@alignOf` 8) and every `*_vtable` global land at odd addresses,
-byte-packed in `.rodata`; `unboxAsView`'s `@alignCast` correctly traps it. Not a zzdds/zidl
-defect — reproduces in ~10 lines with a bare `const val: u32` (`-OReleaseSmall -fno-llvm
--fno-lld` → 1-mod-4; `-OReleaseFast`, the LLVM backend, and Debug/ReleaseSafe all fine).
-Full trail + minimal repro: `zz-dev/releasesmall-misaligned-rodata-investigation.md`.
-Follow-ups: (1) file against `ziglang/zig`; (2) a ReleaseSmall lane, if wanted before the
-fix lands, must force `.use_llvm = true` (as `emit-tests-llvm` already does for Valgrind).
+**Outcome — `ReleaseSmall` landed 2026-08-29 via an LLVM-backend workaround (upstream bug
+root-caused + confirmed fixed on Zig master).** First finding: `zig build test
+-Doptimize=ReleaseSmall` produced 37 `panic: incorrect alignment` crashes (`bootstrap_test`
+/ `typesupport_test`, via `zidl-rt`'s `entity_box.zig` `unboxAsView` `@alignCast(box.vtable)`).
+Root cause: **Zig 0.16.0's self-hosted x86_64 backend emits read-only global constants with
+no alignment, only at `-OReleaseSmall`** — `&SomeImpl.views` (an `extern struct` with
+`@alignOf` 8) and every `*_vtable` global land at odd `.rodata` addresses; `@alignCast`
+correctly traps it. Not a zzdds/zidl defect — reproduces in ~10 lines with a bare
+`const val: u32` (`-OReleaseSmall -fno-llvm -fno-lld` → 1-mod-4; `-OReleaseFast`, the LLVM
+backend, and Debug/ReleaseSafe all fine). **Fixed on `zig-0.17.0-dev.1902+896bd9e15`**;
+still broken on 0.16.0 (the pinned toolchain and latest stable, tagged 2026-04-13; no
+matching upstream issue, nothing to file). Full trail + repro:
+`zz-dev/releasesmall-misaligned-rodata-investigation.md`.
+
+**What landed:** a new `zig build test-release-small` step (`build.zig`) that runs the whole
+unit suite at `-OReleaseSmall` **on the LLVM backend** — it reuses the baseline-CPU /
+`.use_llvm = true` module graph the `emit-tests-llvm` step already builds, and adds
+`b.addRunArtifact` for each test binary. Wired into `run_deterministic_matrix.py` as the
+`release-small` step (so `ci.yml`'s `test-linux` runs it) and into `release.yml`'s `test`
+job as `Test (ReleaseSmall, LLVM backend)`, gated to `ubuntu-latest` (the workaround path is
+Linux-exercised only). **At the Zig 0.17 bump:** delete `test-release-small` + its run-deps,
+replace the matrix/release.yml entries with a plain `zig build test -Doptimize=ReleaseSmall`
+on the normal self-hosted backend, and broaden the `release.yml` step to the whole matrix.
 
 ## Cross-cutting implementation notes
 

docs/roadmap.md

@@ -334,6 +334,15 @@ Audit of `build.zig` options, `scripts/run_deterministic_matrix.py`, `ci.yml`, a
   CMake package files) on each of the four release platforms, verifies completeness, and
   uploads a per-platform tarball that `publish` attaches to the GitHub release. Functional
   coverage of the bundled libraries is the `test` job's `test-bindings` step.
+- **`ReleaseSmall` lane** (2026-08-29) — new `zig build test-release-small` step runs the
+  whole unit suite at `-OReleaseSmall`, wired into `run_deterministic_matrix.py` (so
+  `ci.yml`'s `test-linux` covers it) and `release.yml`'s `test` job (Linux x86_64 only).
+  The step **forces the LLVM backend** to sidestep a Zig 0.16 self-hosted-x86_64 codegen bug
+  (misaligned read-only globals at `-OReleaseSmall` — see the Deferred note below and the
+  step's `build.zig` comment). **At the Zig 0.17 bump: delete `test-release-small` and
+  replace it with a plain `zig build test -Doptimize=ReleaseSmall` step on the normal
+  self-hosted backend, broadened to the whole `release.yml` matrix** — the bug is fixed on
+  0.17.
 
 ### Deferred (investigation trails exist)
 
@@ -346,19 +355,16 @@ Audit of `build.zig` options, `scripts/run_deterministic_matrix.py`, `ci.yml`, a
   drift). Revisit when Zig bundles a newer LLVM. Trail:
   `zz-dev/macos-tsan-crash-investigation.md`. (TSan on Windows: Clang/LLVM has no supported
   target. Extending `examples-tsan` to macOS is a separate follow-up.)
-- **`ReleaseSmall` gate — blocked on an upstream Zig codegen bug (root-caused 2026-08-29).**
-  `zig build test -Doptimize=ReleaseSmall` produces 37 `panic: incorrect alignment` crashes
-  (in `bootstrap_test` / `typesupport_test`, via `zidl-rt`'s `entity_box.zig` `unboxAsView`
-  `@alignCast(box.vtable)`). Root cause: **Zig 0.16.0's self-hosted x86_64 backend, only at
-  `-OReleaseSmall`, emits read-only global constants with no alignment** — `&SomeImpl.views`
-  (an `extern struct` `CAbiViews`, `@alignOf` 8) and every `*_vtable` global land at odd
-  addresses, packed byte-to-byte in `.rodata`. `unboxAsView`'s `@alignCast` correctly traps
-  it. Not a zzdds or zidl defect. Minimal repro (deterministic, ~10 lines): a bare
-  `const val: u32 = …` preceded by a 1-byte `const` lands 1-mod-4 under
-  `-OReleaseSmall -fno-llvm -fno-lld`; fine under `-OReleaseFast`, fine with the LLVM
-  backend, fine under Debug/ReleaseSafe. Full trail + repro in
-  `zz-dev/releasesmall-misaligned-rodata-investigation.md`. Next: file against `ziglang/zig`;
-  a ReleaseSmall CI lane would need `.use_llvm = true` (like `emit-tests-llvm`) until fixed.
+- **Self-hosted `-OReleaseSmall` on Zig 0.16 — upstream codegen bug, worked around above.**
+  Zig 0.16.0's self-hosted x86_64 backend, *only* at `-OReleaseSmall`, emits read-only
+  global constants with no alignment: `&SomeImpl.views` (an `extern struct` `CAbiViews`,
+  `@alignOf` 8) and every `*_vtable` global land at odd `.rodata` addresses, and `zidl-rt`'s
+  `@alignCast(box.vtable)` traps it (`panic: incorrect alignment`, ~37 C-ABI tests).
+  `-OReleaseFast`, the LLVM backend, and Debug/ReleaseSafe are all fine. Not a zzdds/zidl
+  defect; **fixed on `zig-0.17.0-dev.1902`**, not in any stable release (0.16.0, tagged
+  2026-04-13, is latest; no matching upstream issue found). The `ReleaseSmall` lane
+  (see *Landed*) sidesteps it by forcing the LLVM backend until the 0.17 bump. Minimal repro
+  + full trail: `zz-dev/releasesmall-misaligned-rodata-investigation.md`.
 
 ### Still open, ranked
 

scripts/run_deterministic_matrix.py

@@ -3,9 +3,14 @@
 
 This is a convenience wrapper around the checks that are useful before pushing:
 formatting, sleep guardrails, Debug tests, feature-minimal tests, ReleaseSafe
-tests, ReleaseFast tests, and fuzz harness compile-checks.  ThreadSanitizer is
-available as an opt-in because it is slower and can be noisy on some local
-systems.
+tests, ReleaseFast tests, ReleaseSmall tests, and fuzz harness compile-checks.
+ThreadSanitizer is available as an opt-in because it is slower and can be noisy
+on some local systems.
+
+The ReleaseSmall step runs via `zig build test-release-small`, which forces the
+LLVM backend: Zig 0.16's self-hosted x86_64 backend mis-aligns read-only globals
+at -OReleaseSmall (see that step's comment in build.zig). Drop back to a plain
+`zig build test -Doptimize=ReleaseSmall` step at the Zig 0.17 bump.
 """
 
 from __future__ import annotations
@@ -49,6 +54,7 @@ def parse_args() -> argparse.Namespace:
             "feature-minimal",
             "release-safe",
             "release-fast",
+            "release-small",
             "fuzz",
             "tsan-self-check",
             "tsan",
@@ -67,6 +73,10 @@ def steps(zig: str, include_tsan: bool) -> list[Step]:
         Step("feature-minimal", [zig, "build", "test", "-Dipv6=false", "-Dinterface-monitor=false"]),
         Step("release-safe", [zig, "build", "test", "-Doptimize=ReleaseSafe"]),
         Step("release-fast", [zig, "build", "test", "-Doptimize=ReleaseFast"]),
+        # LLVM backend forced by the `test-release-small` step itself -- see its
+        # build.zig comment. Switch to `["test", "-Doptimize=ReleaseSmall"]` at
+        # the Zig 0.17 bump.
+        Step("release-small", [zig, "build", "test-release-small", "-Doptimize=ReleaseSmall"]),
         Step("fuzz", [zig, "build", "test-fuzz"]),
     ]
     if include_tsan: