From 2b19cc33c69fa16bff4462dce708e13f9c6b17bb Mon Sep 17 00:00:00 2001 From: sqt <574914+sqt@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:00:23 +0000 Subject: [PATCH 1/3] Harden release artifacts and process ahead of a public release --- .github/workflows/release.yml | 100 +++++- CHANGELOG.md | 40 +++ build.zig | 8 +- docs/decisions.md | 22 ++ docs/roadmap.md | 36 +- scripts/extract_changelog.py | 100 ++++++ scripts/run_deterministic_matrix.py | 14 +- scripts/verify_release_bundle.py | 323 ++++++++++++++++++ .../cmake_consumer/CMakeLists.txt | 36 ++ test/release-bundle/consumer.c | 26 ++ 10 files changed, 684 insertions(+), 21 deletions(-) create mode 100755 scripts/extract_changelog.py create mode 100755 scripts/verify_release_bundle.py create mode 100644 test/release-bundle/cmake_consumer/CMakeLists.txt create mode 100644 test/release-bundle/consumer.c diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de6e7e33..3efb93d0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -121,6 +121,19 @@ jobs: if: matrix.os == 'ubuntu-latest' run: zig build test-release-small -Doptimize=ReleaseSmall + # musl / fully-static Linux target. `-Dtarget` was never actually + # cross-compiled anywhere in CI before -- every lane built the native + # glibc triple. A `-linux-musl` target links a static musl libc, and + # such a binary runs natively on this glibc x86_64 runner, so this is + # real execution coverage (1076/1076 tests), not just a build check. + # Proves zzdds is musl-clean for Alpine / static-binary / container + # consumers. aarch64-linux-musl would need qemu to run and is deferred; + # a static-archive libzzdds bundle variant is a separate follow-up + # (see docs/roadmap.md "CI / Release Platform Coverage"). + - name: Test (musl static, x86_64-linux) + if: matrix.os == 'ubuntu-latest' + run: zig build test -Dtarget=x86_64-linux-musl + # Binding smoke tests, mirroring ci.yml's test-other job: the release # gate previously ran only `zig build test` (+ReleaseFast +TSan) and # never built a single binding, so a PR breaking the C/C++/Java @@ -263,10 +276,14 @@ jobs: # pkgconfig + CMake package files) on each release platform and uploads it # as a per-platform tarball; `publish` attaches them to the GitHub release. # - # The *functional* smoke test of these libraries is the `test` job's - # `test-bindings` step, which compiles and runs real C/C++/Java programs - # against a freshly built libzzdds on every platform -- this job only has - # to confirm the install tree is complete before packaging it. + # Beyond confirming the tree is complete, this job now also extracts the + # finished tarball into an unrelated directory and drives a real downstream + # consume of it -- `find_package(ZZDDS)` + pkg-config, building + # examples/{c,cpp}/hello_world and a minimal consumer against the *relocated* + # prefix (scripts/verify_release_bundle.py). That catches broken CMake + # package files / pkg-config relocatability / rpath|install-name that the + # `test` job's in-tree `test-bindings` step (which points CMAKE_PREFIX_PATH + # straight at the live zig-out) cannot see. package-libs: name: package-libs (${{ matrix.name }}) needs: [prepare, test] @@ -330,6 +347,32 @@ jobs: mv zig-out "$dir" tar -czf "${dir}.tar.gz" "$dir" + - name: Install Python (for verify_release_bundle.py) + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + # Extract the tarball we just made into an unrelated directory and prove + # a downstream project can consume it from there. Linux: full path -- + # find_package + pkg-config, build examples/{c,cpp}/hello_world, run a + # pub/sub pair. macOS: same but --skip-example-run (the hello_world pair + # needs live UDP discovery, flaky on hosted macOS; cmake_consumer still + # links + runs against libzzdds.dylib, so install-name relocatability is + # still covered). Windows: --configure-only -- exercises the generated + # zzdds-config.cmake and the bundled zidl.exe; the CMake/compiler example + # build path on Windows is deferred (same as ci.yml's Windows Java). + - name: Verify prebuilt bundle is consumable + shell: bash + run: | + set -euo pipefail + tarball="zzdds-${{ needs.prepare.outputs.full_version }}-${{ matrix.name }}.tar.gz" + case "${{ runner.os }}" in + Linux) args="" ;; + macOS) args="--skip-example-run" ;; + Windows) args="--configure-only" ;; + esac + python3 scripts/verify_release_bundle.py --bundle "$tarball" $args + - name: Upload bundle uses: actions/upload-artifact@v4 with: @@ -380,21 +423,50 @@ jobs: git tag "${TAG}" git push origin "${TAG}" + # Release notes body: prefer the curated, date-headed CHANGELOG.md + # sections written since the previous release; fall back to raw commit + # subjects only if that yields nothing (e.g. CHANGELOG not updated). + # Either way, append a compare link to the full commit log. - name: Generate changelog id: changelog env: TAG: ${{ needs.prepare.outputs.tag }} run: | PREV_TAG=$(git describe --tags --abbrev=0 HEAD~1 2>/dev/null || echo "") + + BODY="" + SOURCE="" if [ -n "$PREV_TAG" ]; then - LOG=$(git log "${PREV_TAG}..HEAD~1" --pretty=format:"- %s" --no-merges) - else - LOG=$(git log HEAD~1 --pretty=format:"- %s" --no-merges) + SINCE_DATE=$(git log -1 --format=%as "$PREV_TAG" 2>/dev/null || echo "") + if [ -n "$SINCE_DATE" ] && \ + SLICE=$(python3 scripts/extract_changelog.py --changelog CHANGELOG.md --since-date "$SINCE_DATE"); then + # Demote CHANGELOG's own "## " headings so they nest under + # the "## Changelog" heading in the release-notes template. + BODY=$(printf '%s\n' "$SLICE" | sed 's/^## /### /') + SOURCE="CHANGELOG.md since ${PREV_TAG} (${SINCE_DATE})" + fi fi + + if [ -z "$BODY" ]; then + if [ -n "$PREV_TAG" ]; then + BODY=$(git log "${PREV_TAG}..HEAD~1" --pretty=format:"- %s" --no-merges) + else + BODY=$(git log HEAD~1 --pretty=format:"- %s" --no-merges) + fi + SOURCE="git log (CHANGELOG.md had no dated sections since the last release)" + fi + + if [ -n "$PREV_TAG" ]; then + BODY="${BODY} + + **Full commit log:** https://github.com/${{ github.repository }}/compare/${PREV_TAG}...${TAG}" + fi + + echo "Release-notes body sourced from: ${SOURCE}" { - echo "log<> $GITHUB_OUTPUT - name: Create GitHub release @@ -406,9 +478,15 @@ jobs: run: | gh release create "${TAG}" \ --title "${TAG}" \ - --notes "## Changes + --notes "## Changelog ${CHANGELOG} + ## Stability + Pre-1.0: **any release may break source and ABI compatibility** — the Zig API, the + C ABI, the QoS/config schema, and the bundle layout are all still in flux (see + \`docs/decisions.md\` → Versioning / Releases). Pin this exact tag / bundle; do not + track a branch or a version range. + ## Zig compatibility Built with and requires Zig \`${ZIG_VERSION}\`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 87ca1e58..58dec13e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,46 @@ Dated entries (no release tags past `v0.2.1-zig.0.16.0`; `build.zig.zon` is `TypeSupport.compute_key_hash` signature change (`is_key_only: bool`) rippling to the C ABI mirror and a further zidl release, so it is a follow-up beyond the v0.3.12 bump. Tracked in `docs/roadmap.md` "Selective CDR parse — deferred follow-ups". +- **Release prep — prebuilt-bundle consume check.** `release.yml`'s `package-libs` job used + to verify only the *contents* of the install tree it built, in place. It now also extracts + the finished per-platform tarball into an unrelated directory and drives a real downstream + consume of it (`scripts/verify_release_bundle.py` + committed fixtures under + `test/release-bundle/`): structural completeness, pkg-config / CMake-package + relocatability (no baked-in absolute build path), the bundled `bin/zidl` runs, and + `find_package(ZZDDS)` + `pkg-config` build `examples/{c,cpp}/hello_world` and a minimal + consumer against the *relocated* prefix — on Linux the hello_world pair also exchanges its + 10 samples. This is the consumption path `rmw_zzdds` (and any C/C++ CMake consumer) takes; + the in-tree `test-bindings` step never exercised a moved prefix. Linux: full; macOS: + `--skip-example-run` (skips only the live-UDP pair run — `cmake_consumer` still links and + runs against `libzzdds.dylib`); Windows: `--configure-only`. +- **Release prep — musl / static Linux target lane.** `-Dtarget` was never actually + cross-compiled anywhere in CI. New `zig build test -Dtarget=x86_64-linux-musl` step in + `run_deterministic_matrix.py` (so `ci.yml`'s `test-linux` covers it) and `release.yml`'s + `test` job (Linux x86_64 only). A `-linux-musl` binary is statically linked and runs + natively on the glibc runner, so this executes the full suite (1076/1076), proving zzdds + is musl-clean for Alpine / static-binary / container consumers. `aarch64-linux-musl` + (needs qemu) and a static-archive `libzzdds` bundle variant remain deferred — + `docs/roadmap.md` "CI / Release Platform Coverage". +- **Release prep — GitHub-release notes now come from `CHANGELOG.md`.** `release.yml`'s + `publish` job built its release body from raw `git log --pretty=%s` subjects. It now + quotes the curated, date-headed `CHANGELOG.md` sections written since the previous release + tag (`scripts/extract_changelog.py`, matching by the tag's own date), falls back to commit + subjects only if that yields nothing, and always appends a `compare` link to the full + commit log. +- **Decision recorded — pre-1.0 has no stability guarantee.** `docs/decisions.md` gains a + "Versioning / Releases" section: any release may break the Zig API, the C ABI, the + QoS/config schema, or the bundle layout, with no deprecation cycle; the C ABI stays in + flux until zzdds and Zig mature toward a distant 1.0; `--runtime-version ` stays + unimplemented until there is a tier worth pinning. Consumers pin an exact + `vX.Y.Z-zig.A.B.C` tag / bundle; downstream middleware (e.g. `rmw_zzdds`) owns its own + version mapping and absorbs zzdds churn behind its own boundary. `release.yml`'s release + notes now carry a matching "Stability" section. +- **Fix — the installed `zzdds.pc` / `zzdds-config.cmake` version now tracks `build.zig.zon`.** + `build.zig` carried a second, hand-maintained `zzdds_version` string (stuck at + `0.1.1-zig.0.16.0-dev`) that stamped the `Version:` field of the generated pkg-config and + CMake package files — so a consumer's `pkg-config --modversion zzdds` reported a version + two minors behind the actual package. It now reads `@import("build.zig.zon").version`, the + same field `release.yml` bumps at tag time. ## 2026-08-30 diff --git a/build.zig b/build.zig index 174c88b4..05f76d21 100644 --- a/build.zig +++ b/build.zig @@ -1,6 +1,12 @@ const std = @import("std"); const builtin = @import("builtin"); +/// Single source of truth for the package version. Everything that stamps a +/// version (the installed `zzdds.pc` and `zzdds-config.cmake` below) reads it +/// from here so it can never drift from the published package. `release.yml` +/// bumps `.version` in `build.zig.zon` at tag time. +const zzdds_version = @import("build.zig.zon").version; + /// Run a test binary, giving it a unique DDS domain via `ZZDDS_TEST_DOMAIN_BASE`. /// /// `zig build test` runs test binaries as parallel build-graph steps; the @@ -49,8 +55,6 @@ pub fn build(b: *std.Build) void { const sanitize_thread = b.option(bool, "sanitize-thread", "Enable ThreadSanitizer") orelse false; const debug_allocator = b.option(bool, "debug-allocator", "Route the default (allocator=NULL) factory allocation path through std.heap.DebugAllocator instead of std.heap.c_allocator, for fast attributable double-free/UAF diagnostics") orelse false; - const zzdds_version = "0.1.1-zig.0.16.0-dev"; - // ── Dependencies ────────────────────────────────────────────────────────── const zidl_dep = b.dependency("zidl", .{ diff --git a/docs/decisions.md b/docs/decisions.md index 48c8b561..a509c999 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -315,3 +315,25 @@ Wireshark correlation and deterministic tests. Both paths embed `ZZDDS_VENDOR_ID `-Dipv4`, `-Dipv6`, `-Dinterface-monitor`, `-Dwire-trace`, `-Dguid-filter`, `-Dxtypes`, `-Dcontent-subscription-profile`. Dead-code elimination removes unused paths at compile time — no runtime overhead, no `#ifdef`-style branching at call sites. + +--- + +## Versioning / Releases + +**Pre-1.0: no source- or ABI-compatibility guarantee across releases.** +Any release may change the Zig API, the C ABI (`zzdds_c.h` + the zidl-generated C/C++ +surface), the QoS/config schema, or the prebuilt-bundle layout — without a deprecation +cycle. The C ABI in particular is expected to stay in flux until both zzdds and Zig itself +mature toward a 1.0, which is a long way off. `--runtime-version ` (see +`language-bindings.md`) is deliberately unimplemented until there is a stable tier to pin; +there isn't one yet, and declaring one is not a near-term goal. + +**Consumers pin an exact release.** A tag is `vX.Y.Z-zig.A.B.C` (package version + the +exact Zig toolchain it was built with — enforced in `release.yml`). Pin the exact tag for +`zig fetch`, or the exact per-platform bundle tarball for C/C++ / CMake consumers. Do not +track a branch or a version range. + +**Downstream middleware owns its own compatibility mapping.** A consumer that re-exports +zzdds through its own stable-ish surface (e.g. an `rmw_zzdds`) is responsible for pinning a +specific zzdds release, carrying its own version/build metadata, and absorbing zzdds ABI +churn behind its own boundary — not for expecting zzdds to hold an interface for it. diff --git a/docs/roadmap.md b/docs/roadmap.md index 882f8460..7e04214a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -165,8 +165,10 @@ or an optimisation on an already-improved path): machinery works, but backends do not all generate a real `get_field_from_cdr` callback, so CFT/QueryCondition without a registered `TypeSupport.get_field` accessor passes all samples through (`raw_ops.zig:637`; see `decisions.md`). -- **`--runtime-version `** zidl flag for API-tier pinning is not implemented; relevant - once the first stable API tier is declared. +- **`--runtime-version `** zidl flag for API-tier pinning is not implemented, and + deliberately stays that way pre-1.0 — there is no stable API tier to pin and declaring one + is not a near-term goal (`decisions.md` → Versioning / Releases). Consumers pin an exact + release instead. - **Idiomatic Zig binding** — a future generated `dcps_zig.zig` (closure-based listeners, slice-friendly QoS builders) is not built; Zig callers use the native fat-pointer vtable directly. `language-bindings.md`. @@ -373,7 +375,9 @@ and shape are open — see `zidl/docs/roadmap.md` "Plugin architecture". Audit of `build.zig` options, `scripts/run_deterministic_matrix.py`, `ci.yml`, and `release.yml` against the platform/build-type matrix they exercise. Original ranking -2026-08-16; progress notes below from PR #65 (2026-08-18) and the 2026-08-28 CI pass. +2026-08-16; progress notes below from PR #65 (2026-08-18), the 2026-08-28 CI pass, and the +2026-09-02 release-prep pass (musl lane + prebuilt-bundle consume check + CHANGELOG-sourced +release notes). ### Landed @@ -392,6 +396,25 @@ Audit of `build.zig` options, `scripts/run_deterministic_matrix.py`, `ci.yml`, a CMake package files) on each of the four release platforms, verifies completeness, and uploads a per-platform tarball that `publish` attaches to the GitHub release. Functional coverage of the bundled libraries is the `test` job's `test-bindings` step. +- **Prebuilt-bundle consume check** (2026-09-02) — `package-libs` now also extracts the + finished tarball into an unrelated directory and drives a real downstream consume of it via + `scripts/verify_release_bundle.py`: `find_package(ZZDDS)` + pkg-config resolve from the + *relocated* prefix, the bundled `bin/zidl` runs, and `examples/{c,cpp}/hello_world` + a + minimal consumer compile, link and (Linux) exchange samples against it. Catches broken + CMake package files / pkg-config relocatability / rpath|install-name that the in-tree + `test-bindings` step (CMAKE_PREFIX_PATH pointed straight at the live `zig-out`) can't see. + Linux runs the full path; macOS skips only the live-UDP hello_world pair run + (`--skip-example-run`); Windows is `--configure-only` (CMake/compiler example build on + Windows deferred, as with the Java binding). +- **musl / static Linux target lane** (2026-09-02) — `zig build test -Dtarget=x86_64-linux-musl` + now runs in `run_deterministic_matrix.py` (so `ci.yml`'s `test-linux` covers it) and + `release.yml`'s `test` job (Linux x86_64 only). A `-linux-musl` binary is statically linked + and runs natively on the glibc runner, so this is full-suite execution coverage + (1076/1076), not just a build check — closes "`-Dtarget` is never actually cross-compiled". +- **Release notes sourced from `CHANGELOG.md`** (2026-09-02) — `release.yml`'s `publish` job + builds the GitHub-release body from the curated, date-headed `CHANGELOG.md` sections + written since the previous release tag (`scripts/extract_changelog.py`), falling back to + raw commit subjects only if that yields nothing, and always appending a `compare` link. - **`ReleaseSmall` lane** (2026-08-29) — new `zig build test-release-small` step runs the whole unit suite at `-OReleaseSmall`, wired into `run_deterministic_matrix.py` (so `ci.yml`'s `test-linux` covers it) and `release.yml`'s `test` job (Linux x86_64 only). @@ -429,8 +452,11 @@ Audit of `build.zig` options, `scripts/run_deterministic_matrix.py`, `ci.yml`, a 1. **Real vendor/self RTPS interop** (Connext / Cyclone / CoreDX / self) runs only on Linux x86_64 — no wire / discovery / CDR coverage on Windows, macOS, or ARM64. 2. **No Intel macOS coverage** — `macos-latest` is Apple Silicon only. -3. **No musl / static Linux target** — always glibc, always the native triple; `-Dtarget` - is never actually cross-compiled. +3. **musl coverage is x86_64-only, and there is no static-`libzzdds` bundle** — the new lane + (see *Landed*) cross-builds and runs `x86_64-linux-musl`; `aarch64-linux-musl` would need + qemu to execute. Separately, `build.zig` still only builds `libzzdds` as a shared library + (`.linkage = .dynamic`, no `-Dlinkage` option), so there's no static-archive/musl variant + in `package-libs`' bundle set — deferred until a concrete consumer asks for one. 4. **Valgrind has no viable non-Linux equivalent** — treat as Linux-only unless a specific non-Linux memory bug motivates revisiting. diff --git a/scripts/extract_changelog.py b/scripts/extract_changelog.py new file mode 100755 index 00000000..b203dfe2 --- /dev/null +++ b/scripts/extract_changelog.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Print the slice of CHANGELOG.md that belongs in a release's notes. + +`release.yml`'s publish job used to build its GitHub-release body from raw +`git log --pretty=%s` subjects between tags. Now that CHANGELOG.md is a curated, +date-headed log, the release notes should quote *it* instead. + +CHANGELOG.md is a sequence of `## ` sections, newest first, each +heading starting with an ISO date (`## 2026-09-02`, or a range +`## 2026-08-09 - 2026-08-10` -- the first date wins). This script walks from the +top and prints every section whose date is strictly after `--since-date` +(the previous release's date), stopping at the first section that is not +(or whose heading carries no parseable date -- the pre-dated-scheme tail). + +Exit status: + 0 one or more sections printed + 1 nothing matched (caller should fall back to a git-log body) + 2 bad usage / unreadable changelog +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import re +import sys +from pathlib import Path + + +DATE_RE = re.compile(r"(\d{4})-(\d{2})-(\d{2})") + + +def parse_heading_date(heading: str) -> dt.date | None: + m = DATE_RE.search(heading) + if not m: + return None + try: + return dt.date(int(m.group(1)), int(m.group(2)), int(m.group(3))) + except ValueError: + return None + + +def split_sections(text: str) -> list[tuple[str, list[str]]]: + """-> [(heading_line, [body_line, ...]), ...] in file order. Anything + before the first `## ` heading (the preamble) is dropped.""" + sections: list[tuple[str, list[str]]] = [] + cur_heading: str | None = None + cur_body: list[str] = [] + for line in text.splitlines(): + if line.startswith("## "): + if cur_heading is not None: + sections.append((cur_heading, cur_body)) + cur_heading = line + cur_body = [] + elif cur_heading is not None: + cur_body.append(line) + if cur_heading is not None: + sections.append((cur_heading, cur_body)) + return sections + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--changelog", type=Path, default=Path("CHANGELOG.md")) + ap.add_argument("--since-date", required=True, + help="ISO date (YYYY-MM-DD) of the previous release; sections on or before it are excluded") + args = ap.parse_args() + + try: + since = dt.date.fromisoformat(args.since_date) + except ValueError: + print(f"extract_changelog: --since-date is not an ISO date: {args.since_date!r}", file=sys.stderr) + return 2 + + try: + text = args.changelog.read_text() + except OSError as e: + print(f"extract_changelog: cannot read {args.changelog}: {e}", file=sys.stderr) + return 2 + + out: list[str] = [] + for heading, body in split_sections(text): + date = parse_heading_date(heading) + if date is None or date <= since: + break + out.append(heading) + out.extend(body) + + while out and not out[-1].strip(): + out.pop() + + if not out: + return 1 + + print("\n".join(out)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_deterministic_matrix.py b/scripts/run_deterministic_matrix.py index ccdf1a9f..ada155bd 100755 --- a/scripts/run_deterministic_matrix.py +++ b/scripts/run_deterministic_matrix.py @@ -3,9 +3,9 @@ This is a convenience wrapper around the checks that are useful before pushing: formatting, sleep guardrails, Debug tests, feature-minimal tests, ReleaseSafe -tests, ReleaseFast tests, ReleaseSmall tests, and fuzz harness compile-checks. -ThreadSanitizer is available as an opt-in because it is slower and can be noisy -on some local systems. +tests, ReleaseFast tests, ReleaseSmall tests, a musl static-target cross-build, +and fuzz harness compile-checks. ThreadSanitizer is available as an opt-in +because it is slower and can be noisy on some local systems. The ReleaseSmall step runs via `zig build test-release-small`, which forces the LLVM backend: Zig 0.16's self-hosted x86_64 backend mis-aligns read-only globals @@ -55,6 +55,7 @@ def parse_args() -> argparse.Namespace: "release-safe", "release-fast", "release-small", + "musl", "fuzz", "tsan-self-check", "tsan", @@ -77,6 +78,13 @@ def steps(zig: str, include_tsan: bool) -> list[Step]: # build.zig comment. Switch to `["test", "-Doptimize=ReleaseSmall"]` at # the Zig 0.17 bump. Step("release-small", [zig, "build", "test-release-small", "-Doptimize=ReleaseSmall"]), + # musl / fully-static Linux target. `-Dtarget` is otherwise never + # cross-compiled in the matrix. A `-linux-musl` binary is statically + # linked and runs natively on a glibc x86_64 host, so this executes + # the full suite (not just a build check) and proves zzdds is + # musl-clean for Alpine / static-binary consumers. Host-arch only; + # aarch64-linux-musl would need qemu. + Step("musl", [zig, "build", "test", "-Dtarget=x86_64-linux-musl"]), Step("fuzz", [zig, "build", "test-fuzz"]), ] if include_tsan: diff --git a/scripts/verify_release_bundle.py b/scripts/verify_release_bundle.py new file mode 100755 index 00000000..aa85d5d5 --- /dev/null +++ b/scripts/verify_release_bundle.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Verify a prebuilt zzdds C/C++ library bundle is consumable after relocation. + +`release.yml`'s `package-libs` job builds an install tree, renames it, and tars +it up. That job only checks the *contents* of the tree it just built, in place. +This script closes the gap: it takes the finished tarball, extracts it into a +fresh directory unrelated to the build tree, and confirms a downstream C/C++ +project can actually consume it from there. + +Steps: + + 1. Structural completeness -- every file `find_package(ZZDDS)` / pkg-config / + the C++ "three-artifact" build path needs is present (a superset of + `package-libs`' inline check: also `bin/zidl`, `src/*.cpp`, `zzdds_c.h`, + `zidl_allocator.h`, `libzidl_cdr.a`). + 2. Relocatability -- `zzdds.pc` and `zzdds-config.cmake` derive their prefix + from their own location, with no absolute build path baked in. + 3. The bundled `bin/zidl` code generator runs on this platform. + 4. `find_package(ZZDDS)` resolves from the relocated prefix and its imported + targets are usable -- `test/release-bundle/cmake_consumer` (fast, no + codegen) then the real `examples/c/hello_world` + `examples/cpp/hello_world` + downstream CMake projects, compiled and linked against the bundle. + 5. The linked binaries actually run -- a hello_world publisher/subscriber + pair exchanges its 10 samples (loads `libzzdds` from the relocated prefix: + catches broken rpath / install-name). + 6. pkg-config -- `pkg-config --cflags --libs zzdds` from the relocated prefix + compiles, links and runs `test/release-bundle/consumer.c`. + +`--configure-only` stops after step 4's `cmake` *configure* of the small +consumer (no compiler, no run, no examples). Used on Windows, where the +CMake/compiler example build path is not yet covered (same deferral as +`ci.yml`'s Windows Java binding) -- it still exercises the generated +`zzdds-config.cmake` and the bundled `zidl.exe`. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def log(msg: str) -> None: + print(f"[verify-bundle] {msg}", flush=True) + + +def fail(msg: str) -> "NoReturn": # type: ignore[name-defined] + print(f"::error::verify_release_bundle: {msg}" if os.environ.get("GITHUB_ACTIONS") else f"FAIL: {msg}", + file=sys.stderr, flush=True) + raise SystemExit(1) + + +def run(cmd: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None, + timeout: float | None = None, capture: bool = False) -> subprocess.CompletedProcess: + log(f"$ {' '.join(cmd)}" + (f" (cwd={cwd})" if cwd else "")) + return subprocess.run( + cmd, cwd=cwd, env=env, timeout=timeout, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None, + text=True, + ) + + +# ── step 1: structure ─────────────────────────────────────────────────────── + +def find_prefix(extract_root: Path) -> Path: + entries = [p for p in extract_root.iterdir() if p.is_dir()] + if len(entries) != 1: + fail(f"expected exactly one top-level directory in the bundle, found: {[p.name for p in entries]}") + return entries[0] + + +def check_structure(prefix: Path) -> None: + required = [ + "include/dcps.h", "include/zzdds.h", "include/zzdds_c.h", + "include/zidl_cdr.h", "include/zidl_allocator.h", + "include/dcps.hpp", "include/dcps_impl.hpp", + "include/zzdds.hpp", "include/zzdds_impl.hpp", + "lib/libzidl_cdr.a", + "lib/pkgconfig/zzdds.pc", + "lib/cmake/ZZDDS/zzdds-config.cmake", + "src/dcps_impl.cpp", "src/zzdds_impl.cpp", + ] + missing = [rel for rel in required if not (prefix / rel).is_file()] + if missing: + fail("bundle is missing required files:\n " + "\n ".join(missing)) + + if not any((prefix / rel).is_file() for rel in ("bin/zidl", "bin/zidl.exe")): + fail("bundle has no bin/zidl code generator") + + shared = ["lib/libzzdds.so", "lib/libzzdds.dylib", "bin/zzdds.dll", "lib/zzdds.dll"] + if not any((prefix / rel).is_file() for rel in shared): + fail("bundle has no dynamic libzzdds (looked for: " + ", ".join(shared) + ")") + + log("structure: all required files present") + + +def check_relocatable(prefix: Path) -> None: + pc = (prefix / "lib/pkgconfig/zzdds.pc").read_text() + prefix_lines = [ln for ln in pc.splitlines() if ln.startswith("prefix=")] + if not prefix_lines or not prefix_lines[0].startswith("prefix=${pcfiledir}"): + fail(f"zzdds.pc prefix= is not ${{pcfiledir}}-relative: {prefix_lines}") + + cmake_cfg = (prefix / "lib/cmake/ZZDDS/zzdds-config.cmake").read_text() + if "CMAKE_CURRENT_LIST_FILE" not in cmake_cfg: + fail("zzdds-config.cmake does not derive its prefix from CMAKE_CURRENT_LIST_FILE") + + # No absolute build-time path may survive into the shipped metadata. + needles = ["runner/work", "/home/", "/Users/", "\\Users\\", "zig-out"] + for rel in ("lib/pkgconfig/zzdds.pc", "lib/cmake/ZZDDS/zzdds-config.cmake"): + text = (prefix / rel).read_text() + hits = [n for n in needles if n in text] + if hits: + fail(f"{rel} contains a baked-in absolute build path (matched {hits}) -- bundle is not relocatable") + + log("relocatable: pkg-config + CMake metadata derive prefix from their own location") + + +# ── step 3: bundled zidl runs ─────────────────────────────────────────────── + +def check_zidl_runs(prefix: Path) -> None: + zidl = prefix / "bin" / ("zidl.exe" if (prefix / "bin/zidl.exe").is_file() else "zidl") + proc = run([str(zidl), "--version"], capture=True, timeout=60) + if proc.returncode != 0: + fail(f"bundled `{zidl.name} --version` exited {proc.returncode}:\n{proc.stdout}") + log(f"zidl: bundled generator runs -- {proc.stdout.strip()}") + + +# ── step 4/5: CMake consumers ─────────────────────────────────────────────── + +def cmake_configure(src: Path, build: Path, prefix: Path, extra: list[str] | None = None) -> None: + proc = run( + ["cmake", "-S", str(src), "-B", str(build), f"-DCMAKE_PREFIX_PATH={prefix}", *(extra or [])], + capture=True, timeout=600, + ) + if proc.returncode != 0: + fail(f"cmake configure of {src} failed:\n{proc.stdout}") + + +def cmake_build(build: Path) -> None: + proc = run(["cmake", "--build", str(build), "--parallel"], capture=True, timeout=1200) + if proc.returncode != 0: + fail(f"cmake build in {build} failed:\n{proc.stdout}") + + +def run_hello_world_pair(build_dir: Path, prefix: Path, domain: int) -> None: + """Start the hello_world sub + pub built in build_dir, on `domain`, and + require both to exit 0 with their success markers. The pair is designed + for a clean handshake shutdown (see the example sources), so this does + not need a settle timer -- just a generous ceiling.""" + def exe(name: str) -> str: + for cand in (build_dir / name, build_dir / f"{name}.exe"): + if cand.is_file(): + return str(cand) + fail(f"{name} not found under {build_dir}") + + env = dict(os.environ) + libdir = str(prefix / "lib") + for var in ("LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"): + env[var] = libdir + (os.pathsep + env[var] if env.get(var) else "") + + sub = subprocess.Popen([exe("hello_world_sub"), "--domain", str(domain)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env) + time.sleep(0.5) + pub = subprocess.Popen([exe("hello_world_pub"), "--domain", str(domain)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env) + + deadline = time.monotonic() + 60 + outs: dict[str, str] = {} + for name, proc in (("pub", pub), ("sub", sub)): + remaining = max(1.0, deadline - time.monotonic()) + try: + outs[name], _ = proc.communicate(timeout=remaining) + except subprocess.TimeoutExpired: + proc.kill() + outs[name], _ = proc.communicate() + fail(f"hello_world {name} (domain {domain}) did not exit within the deadline:\n{outs[name]}") + + markers = {"pub": "Publisher: done.", "sub": "Subscriber: received all 10 samples in order."} + for name, proc in (("pub", pub), ("sub", sub)): + if proc.returncode != 0: + fail(f"hello_world {name} (domain {domain}) exited {proc.returncode}:\n{outs[name]}") + if markers[name] not in outs[name]: + fail(f"hello_world {name} (domain {domain}) missing success marker {markers[name]!r}:\n{outs[name]}") + log(f"hello_world pair (domain {domain}): pub + sub exchanged 10 samples and exited cleanly") + + +# ── step 6: pkg-config ────────────────────────────────────────────────────── + +def check_pkgconfig(prefix: Path, work: Path) -> None: + if not shutil.which("pkg-config"): + log("pkg-config: not installed on this runner -- skipping (CMake path already covered)") + return + cc = os.environ.get("CC") or shutil.which("cc") or shutil.which("gcc") or shutil.which("clang") + if not cc: + log("pkg-config: no C compiler found -- skipping") + return + + env = dict(os.environ) + env["PKG_CONFIG_PATH"] = str(prefix / "lib/pkgconfig") + ( + os.pathsep + env["PKG_CONFIG_PATH"] if env.get("PKG_CONFIG_PATH") else "") + + def pc(*args: str) -> str: + proc = subprocess.run(["pkg-config", *args, "zzdds"], env=env, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if proc.returncode != 0: + fail(f"pkg-config {' '.join(args)} zzdds failed:\n{proc.stdout}") + return proc.stdout.strip() + + version = pc("--modversion") + cflags = pc("--cflags").split() + libs = pc("--libs").split() + log(f"pkg-config: zzdds {version}; cflags={cflags}; libs={libs}") + + out_bin = work / "pkgconfig_consumer" + src = REPO_ROOT / "test/release-bundle/consumer.c" + proc = run([cc, str(src), *cflags, *libs, "-o", str(out_bin)], capture=True, timeout=300) + if proc.returncode != 0: + fail(f"compiling consumer.c with pkg-config flags failed:\n{proc.stdout}") + + run_env = dict(os.environ) + libdir = str(prefix / "lib") + for var in ("LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"): + run_env[var] = libdir + (os.pathsep + run_env[var] if run_env.get(var) else "") + proc = subprocess.run([str(out_bin)], env=run_env, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60) + if proc.returncode != 0: + fail(f"pkg-config consumer binary exited {proc.returncode}:\n{proc.stdout}") + log(f"pkg-config: consumer binary ran -- {proc.stdout.strip()}") + + +# ── driver ───────────────────────────────────────────────────────────────── + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--bundle", required=True, type=Path, help="path to the zzdds--.tar.gz bundle") + ap.add_argument("--examples-dir", type=Path, default=REPO_ROOT / "examples", + help="path to the folded-in examples/ tree (default: /examples)") + ap.add_argument("--configure-only", action="store_true", + help="stop after cmake-configuring the small consumer (no compiler / no run / no examples)") + ap.add_argument("--skip-example-run", action="store_true", + help="build examples/{c,cpp}/hello_world against the bundle but do not run the " + "pub/sub pair (link coverage only; use where live UDP DDS discovery is flaky, " + "e.g. hosted macOS runners -- cmake_consumer still runs and loads libzzdds)") + ap.add_argument("--work", type=Path, default=None, help="working directory (default: a fresh temp dir)") + ap.add_argument("--keep", action="store_true", help="do not delete the working directory on exit") + ap.add_argument("--domain-base", type=int, default=58, + help="base DDS domain for the hello_world pairs (uses base and base+1)") + args = ap.parse_args() + + if not args.bundle.is_file(): + fail(f"bundle not found: {args.bundle}") + + work = args.work or Path(tempfile.mkdtemp(prefix="zzdds-verify-bundle-")) + work.mkdir(parents=True, exist_ok=True) + extract_root = work / "extracted" + extract_root.mkdir(exist_ok=True) + log(f"working directory: {work}") + + try: + log(f"extracting {args.bundle.name} -> {extract_root}") + with tarfile.open(args.bundle) as tf: + if sys.version_info >= (3, 12): + tf.extractall(extract_root, filter="data") + else: + tf.extractall(extract_root) # noqa: S202 -- our own release artifact + prefix = find_prefix(extract_root) + log(f"bundle prefix: {prefix}") + + check_structure(prefix) + check_relocatable(prefix) + check_zidl_runs(prefix) + + cc_build = work / "cmake_consumer_build" + cmake_configure(REPO_ROOT / "test/release-bundle/cmake_consumer", cc_build, prefix) + if args.configure_only: + log("--configure-only: stopping after the CMake consumer configure step") + log("PASS") + return 0 + cmake_build(cc_build) + proc = subprocess.run([str(next(p for p in (cc_build / "zzdds_bundle_cmake_consumer", + cc_build / "zzdds_bundle_cmake_consumer.exe") if p.is_file()))], + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60, + env={**os.environ, "LD_LIBRARY_PATH": str(prefix / "lib"), + "DYLD_LIBRARY_PATH": str(prefix / "lib")}) + if proc.returncode != 0: + fail(f"cmake_consumer binary exited {proc.returncode}:\n{proc.stdout}") + log(f"cmake_consumer: ran -- {proc.stdout.strip()}") + + for i, name in enumerate(("c", "cpp")): + example = args.examples_dir / name / "hello_world" + if not (example / "CMakeLists.txt").is_file(): + fail(f"example project not found: {example}") + build = work / f"hello_world_{name}_build" + cmake_configure(example, build, prefix) + cmake_build(build) + if args.skip_example_run: + log(f"hello_world ({name}): built against the bundle; --skip-example-run set, not running the pair") + else: + run_hello_world_pair(build, prefix, args.domain_base + i) + + check_pkgconfig(prefix, work) + + log("PASS") + return 0 + finally: + if args.keep: + log(f"--keep: left working directory in place at {work}") + else: + shutil.rmtree(work, ignore_errors=True) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/release-bundle/cmake_consumer/CMakeLists.txt b/test/release-bundle/cmake_consumer/CMakeLists.txt new file mode 100644 index 00000000..f6c107f3 --- /dev/null +++ b/test/release-bundle/cmake_consumer/CMakeLists.txt @@ -0,0 +1,36 @@ +# Minimal downstream CMake project that consumes a relocated zzdds bundle via +# find_package(ZZDDS). Driven by scripts/verify_release_bundle.py: +# +# cmake -S . -B build -DCMAKE_PREFIX_PATH= +# cmake --build build # skipped with --configure-only (Windows) +# ./build/zzdds_bundle_cmake_consumer +# +# It asserts the generated lib/cmake/ZZDDS/zzdds-config.cmake defines +# everything a real consumer (see examples/{c,cpp}/hello_world) depends on, +# then compiles/links/runs the shared consumer.c against the imported targets. +cmake_minimum_required(VERSION 3.14) +project(zzdds_bundle_cmake_consumer C) + +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) + +find_package(ZZDDS REQUIRED) + +foreach(_target ZZDDS::zzdds ZZDDS::zidl_cdr) + if(NOT TARGET ${_target}) + message(FATAL_ERROR "ZZDDS package config did not define imported target ${_target}") + endif() +endforeach() + +if(NOT DEFINED ZZDDS_ZIDL_EXECUTABLE OR NOT EXISTS "${ZZDDS_ZIDL_EXECUTABLE}") + message(FATAL_ERROR "ZZDDS_ZIDL_EXECUTABLE missing or does not exist: '${ZZDDS_ZIDL_EXECUTABLE}'") +endif() + +foreach(_var ZZDDS_DCPS_IMPL_CPP ZZDDS_ZZDDS_IMPL_CPP) + if(NOT DEFINED ${_var} OR NOT EXISTS "${${_var}}") + message(FATAL_ERROR "${_var} missing or does not exist: '${${_var}}'") + endif() +endforeach() + +add_executable(zzdds_bundle_cmake_consumer "${CMAKE_CURRENT_SOURCE_DIR}/../consumer.c") +target_link_libraries(zzdds_bundle_cmake_consumer PRIVATE ZZDDS::zzdds ZZDDS::zidl_cdr) diff --git a/test/release-bundle/consumer.c b/test/release-bundle/consumer.c new file mode 100644 index 00000000..c5675010 --- /dev/null +++ b/test/release-bundle/consumer.c @@ -0,0 +1,26 @@ +/* + * Minimal downstream consumer of a relocated zzdds C/C++ library bundle. + * + * Driven by scripts/verify_release_bundle.py through both supported + * consumption paths (pkg-config and CMake find_package(ZZDDS)). Deliberately + * tiny: it only has to force a real link against the bundled libzzdds + + * libzidl_cdr and execute one runtime call, proving the symbols resolve and + * the shared library actually loads from a prefix that was renamed and moved + * after the build. Functional DDS coverage is examples/{c,cpp}/hello_world + * (built by the same script against the same bundle) plus the test-bindings + * and examples CI jobs. + */ +#include "zzdds_c.h" + +#include + +int main(void) { + zzdds_DomainParticipantFactory factory = zzdds_create_factory(); + if (zzdds_factory_is_nil(factory)) { + fprintf(stderr, "FAIL: zzdds_create_factory() returned the nil sentinel\n"); + return 1; + } + zzdds_destroy_factory(factory); + printf("ok: zzdds bundle consumer linked against libzzdds and ran\n"); + return 0; +} From 8fab15877ed8ccbd0392de3c568913f0ecab1ed7 Mon Sep 17 00:00:00 2001 From: sqt <574914+sqt@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:13:31 +0000 Subject: [PATCH 2/3] resolving CI issues --- .github/workflows/release.yml | 50 ++++++------ CHANGELOG.md | 23 ++++-- docs/roadmap.md | 14 +++- scripts/extract_changelog.py | 115 +++++++++++++++++++--------- scripts/run_deterministic_matrix.py | 37 ++++++--- scripts/verify_release_bundle.py | 14 ++-- 6 files changed, 168 insertions(+), 85 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3efb93d0..a87c6932 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -276,14 +276,22 @@ jobs: # pkgconfig + CMake package files) on each release platform and uploads it # as a per-platform tarball; `publish` attaches them to the GitHub release. # - # Beyond confirming the tree is complete, this job now also extracts the - # finished tarball into an unrelated directory and drives a real downstream - # consume of it -- `find_package(ZZDDS)` + pkg-config, building + # Beyond confirming the tree is complete, this job (Linux + macOS) also + # extracts the finished tarball into an unrelated directory and drives a real + # downstream consume of it -- `find_package(ZZDDS)` + pkg-config, building # examples/{c,cpp}/hello_world and a minimal consumer against the *relocated* # prefix (scripts/verify_release_bundle.py). That catches broken CMake # package files / pkg-config relocatability / rpath|install-name that the # `test` job's in-tree `test-bindings` step (which points CMAKE_PREFIX_PATH # straight at the live zig-out) cannot see. + # + # Windows gets the structural check only: the generated zzdds-config.cmake / + # zzdds.pc are POSIX-shaped today (search `lib/` for the shared lib, no + # IMPORTED_IMPLIB, `bin/zidl` not `bin/zidl.exe`), so `find_package(ZZDDS)` + # can't configure there yet. Making the generated package Windows-correct is + # tracked in docs/roadmap.md "CI / Release Platform Coverage"; until then the + # Windows tarball ships as-is (its libraries are still functionally covered by + # the `test` job's `test-bindings` step). package-libs: name: package-libs (${{ matrix.name }}) needs: [prepare, test] @@ -348,29 +356,26 @@ jobs: tar -czf "${dir}.tar.gz" "$dir" - name: Install Python (for verify_release_bundle.py) + if: runner.os != 'Windows' uses: actions/setup-python@v5 with: python-version: "3.11" - # Extract the tarball we just made into an unrelated directory and prove - # a downstream project can consume it from there. Linux: full path -- + # Extract the tarball we just made into an unrelated directory and prove a + # downstream project can consume it from there. Linux: full path -- # find_package + pkg-config, build examples/{c,cpp}/hello_world, run a # pub/sub pair. macOS: same but --skip-example-run (the hello_world pair # needs live UDP discovery, flaky on hosted macOS; cmake_consumer still # links + runs against libzzdds.dylib, so install-name relocatability is - # still covered). Windows: --configure-only -- exercises the generated - # zzdds-config.cmake and the bundled zidl.exe; the CMake/compiler example - # build path on Windows is deferred (same as ci.yml's Windows Java). + # still covered). Windows is skipped -- see the job-header comment. - name: Verify prebuilt bundle is consumable + if: runner.os != 'Windows' shell: bash run: | set -euo pipefail tarball="zzdds-${{ needs.prepare.outputs.full_version }}-${{ matrix.name }}.tar.gz" - case "${{ runner.os }}" in - Linux) args="" ;; - macOS) args="--skip-example-run" ;; - Windows) args="--configure-only" ;; - esac + args="" + [ "${{ runner.os }}" = "macOS" ] && args="--skip-example-run" python3 scripts/verify_release_bundle.py --bundle "$tarball" $args - name: Upload bundle @@ -436,15 +441,16 @@ jobs: BODY="" SOURCE="" - if [ -n "$PREV_TAG" ]; then - SINCE_DATE=$(git log -1 --format=%as "$PREV_TAG" 2>/dev/null || echo "") - if [ -n "$SINCE_DATE" ] && \ - SLICE=$(python3 scripts/extract_changelog.py --changelog CHANGELOG.md --since-date "$SINCE_DATE"); then - # Demote CHANGELOG's own "## " headings so they nest under - # the "## Changelog" heading in the release-notes template. - BODY=$(printf '%s\n' "$SLICE" | sed 's/^## /### /') - SOURCE="CHANGELOG.md since ${PREV_TAG} (${SINCE_DATE})" - fi + # extract_changelog.py emits the CHANGELOG.md sections added since + # PREV_TAG (by heading-set diff against CHANGELOG.md as of that tag; + # falls back to the leading date-headed run if the tag predates the + # file). An empty PREV_TAG is passed through and lands on that same + # fallback. + if SLICE=$(python3 scripts/extract_changelog.py --changelog CHANGELOG.md --prev-ref "$PREV_TAG"); then + # Demote CHANGELOG's own "## " headings so they nest under + # the "## Changelog" heading in the release-notes template. + BODY=$(printf '%s\n' "$SLICE" | sed 's/^## /### /') + SOURCE="CHANGELOG.md since ${PREV_TAG:-}" fi if [ -z "$BODY" ]; then diff --git a/CHANGELOG.md b/CHANGELOG.md index 58dec13e..77f26cce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,21 +54,28 @@ Dated entries (no release tags past `v0.2.1-zig.0.16.0`; `build.zig.zon` is 10 samples. This is the consumption path `rmw_zzdds` (and any C/C++ CMake consumer) takes; the in-tree `test-bindings` step never exercised a moved prefix. Linux: full; macOS: `--skip-example-run` (skips only the live-UDP pair run — `cmake_consumer` still links and - runs against `libzzdds.dylib`); Windows: `--configure-only`. + runs against `libzzdds.dylib`). Windows keeps the structural check only: the generated + `zzdds-config.cmake` / `zzdds.pc` are POSIX-shaped (search `lib/` for the shared lib, no + `IMPORTED_IMPLIB`, `bin/zidl` not `bin/zidl.exe`), so `find_package(ZZDDS)` can't configure + a bundle there yet — tracked in `docs/roadmap.md` "CI / Release Platform Coverage". - **Release prep — musl / static Linux target lane.** `-Dtarget` was never actually cross-compiled anywhere in CI. New `zig build test -Dtarget=x86_64-linux-musl` step in `run_deterministic_matrix.py` (so `ci.yml`'s `test-linux` covers it) and `release.yml`'s `test` job (Linux x86_64 only). A `-linux-musl` binary is statically linked and runs natively on the glibc runner, so this executes the full suite (1076/1076), proving zzdds - is musl-clean for Alpine / static-binary / container consumers. `aarch64-linux-musl` - (needs qemu) and a static-archive `libzzdds` bundle variant remain deferred — - `docs/roadmap.md` "CI / Release Platform Coverage". + is musl-clean for Alpine / static-binary / container consumers. In + `run_deterministic_matrix.py` the step is gated to x86_64-Linux hosts (elsewhere the + cross-built binaries can't run, and Zig would silently skip them); CI's `ubuntu-latest` + runs it unconditionally. `aarch64-linux-musl` (needs qemu) and a static-archive `libzzdds` + bundle variant remain deferred — `docs/roadmap.md` "CI / Release Platform Coverage". - **Release prep — GitHub-release notes now come from `CHANGELOG.md`.** `release.yml`'s `publish` job built its release body from raw `git log --pretty=%s` subjects. It now - quotes the curated, date-headed `CHANGELOG.md` sections written since the previous release - tag (`scripts/extract_changelog.py`, matching by the tag's own date), falls back to commit - subjects only if that yields nothing, and always appends a `compare` link to the full - commit log. + quotes the `CHANGELOG.md` sections added since the previous release tag — + `scripts/extract_changelog.py` diffs the current section headings against `CHANGELOG.md` + as of that tag (so two releases on the same calendar day are handled: a plain date cutoff + would drop the second), falling back to the leading date-headed run when the tag predates + the file, and to raw commit subjects only if that yields nothing. Always appends a + `compare` link to the full commit log. - **Decision recorded — pre-1.0 has no stability guarantee.** `docs/decisions.md` gains a "Versioning / Releases" section: any release may break the Zig API, the C ABI, the QoS/config schema, or the bundle layout, with no deprecation cycle; the C ABI stays in diff --git a/docs/roadmap.md b/docs/roadmap.md index 7e04214a..1299eec7 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -404,8 +404,9 @@ release notes). CMake package files / pkg-config relocatability / rpath|install-name that the in-tree `test-bindings` step (CMAKE_PREFIX_PATH pointed straight at the live `zig-out`) can't see. Linux runs the full path; macOS skips only the live-UDP hello_world pair run - (`--skip-example-run`); Windows is `--configure-only` (CMake/compiler example build on - Windows deferred, as with the Java binding). + (`--skip-example-run`). Windows gets the structural check only — the generated + `zzdds-config.cmake` / `zzdds.pc` are POSIX-shaped, so `find_package(ZZDDS)` can't + configure there yet (see "Still open" below). - **musl / static Linux target lane** (2026-09-02) — `zig build test -Dtarget=x86_64-linux-musl` now runs in `run_deterministic_matrix.py` (so `ci.yml`'s `test-linux` covers it) and `release.yml`'s `test` job (Linux x86_64 only). A `-linux-musl` binary is statically linked @@ -457,7 +458,14 @@ release notes). qemu to execute. Separately, `build.zig` still only builds `libzzdds` as a shared library (`.linkage = .dynamic`, no `-Dlinkage` option), so there's no static-archive/musl variant in `package-libs`' bundle set — deferred until a concrete consumer asks for one. -4. **Valgrind has no viable non-Linux equivalent** — treat as Linux-only unless a specific +4. **The generated CMake/pkg-config package is POSIX-only** — `build.zig`'s + `zzdds-config.cmake` searches `lib/` for the shared library (the Windows DLL installs to + `bin/`), sets no `IMPORTED_IMPLIB` for the import lib, and hard-codes `bin/zidl` (not + `bin/zidl.exe`); `zzdds.pc` is likewise `-l`-style. So a Windows consumer can't + `find_package(ZZDDS)` a bundle yet — `package-libs` ships the Windows tarball with the + structural check only, and `verify_release_bundle.py` is not run there. Fix: platform-aware + generation in `build.zig` + turn the Windows arm of the consume check back on. +5. **Valgrind has no viable non-Linux equivalent** — treat as Linux-only unless a specific non-Linux memory bug motivates revisiting. --- diff --git a/scripts/extract_changelog.py b/scripts/extract_changelog.py index b203dfe2..3f924c69 100755 --- a/scripts/extract_changelog.py +++ b/scripts/extract_changelog.py @@ -3,41 +3,38 @@ `release.yml`'s publish job used to build its GitHub-release body from raw `git log --pretty=%s` subjects between tags. Now that CHANGELOG.md is a curated, -date-headed log, the release notes should quote *it* instead. - -CHANGELOG.md is a sequence of `## ` sections, newest first, each -heading starting with an ISO date (`## 2026-09-02`, or a range -`## 2026-08-09 - 2026-08-10` -- the first date wins). This script walks from the -top and prints every section whose date is strictly after `--since-date` -(the previous release's date), stopping at the first section that is not -(or whose heading carries no parseable date -- the pre-dated-scheme tail). +section-headed log, the release notes quote *it* instead. + +CHANGELOG.md is a sequence of `## ` sections, newest first. The slice +for a release is "every section added since the previous release tag". Given +`--prev-ref `, this compares the current CHANGELOG against the CHANGELOG as +it stood at that tag: it emits the leading run of sections whose headings are +new, and if there are none (two releases the same day, so the newest `## ` +heading already existed) it emits that section's heading plus only the body +lines added since. A plain date cutoff would drop the second same-day release. + +If the previous tag predates CHANGELOG.md (or `--prev-ref` is omitted / not a +resolvable ref), it falls back to emitting the leading run of date-headed +sections (`## 2026-09-02`, or a range `## 2026-08-09 - 2026-08-10`), stopping at +the first heading with no date -- correct for the first release cut under the +dated-entry scheme. Exit status: 0 one or more sections printed - 1 nothing matched (caller should fall back to a git-log body) + 1 nothing to emit (caller should fall back to a git-log body) 2 bad usage / unreadable changelog """ from __future__ import annotations import argparse -import datetime as dt import re +import subprocess import sys from pathlib import Path -DATE_RE = re.compile(r"(\d{4})-(\d{2})-(\d{2})") - - -def parse_heading_date(heading: str) -> dt.date | None: - m = DATE_RE.search(heading) - if not m: - return None - try: - return dt.date(int(m.group(1)), int(m.group(2)), int(m.group(3))) - except ValueError: - return None +DATE_RE = re.compile(r"\d{4}-\d{2}-\d{2}") def split_sections(text: str) -> list[tuple[str, list[str]]]: @@ -59,32 +56,78 @@ def split_sections(text: str) -> list[tuple[str, list[str]]]: return sections +def changelog_at_ref(ref: str, changelog: Path) -> str | None: + """The contents of `changelog` as of git `ref`, or None if that ref / + path pair does not resolve (e.g. the tag predates CHANGELOG.md).""" + # `:./` resolves the path relative to cwd, which is where the + # workflow invokes this script (repo root). + proc = subprocess.run( + ["git", "show", f"{ref}:./{changelog.name}"], + cwd=changelog.resolve().parent, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + ) + return proc.stdout if proc.returncode == 0 else None + + +def slice_by_prev_headings(sections: list[tuple[str, list[str]]], prev_text: str) -> list[str]: + prev_sections = split_sections(prev_text) + prev_headings = {h for h, _ in prev_sections} + out: list[str] = [] + for heading, body in sections: + if heading in prev_headings: + break + out.append(heading) + out.extend(body) + if out: + return out + + # Nothing under a brand-new heading. Handles two releases on the same day: + # the newest section's heading (e.g. "## 2026-09-02") already existed at the + # previous tag, but bullets were appended to it afterwards. Emit just the + # body lines that are new since then. + if sections: + heading, body = sections[0] + prev_body = next((b for h, b in prev_sections if h == heading), None) + if prev_body is not None: + prev_set = set(prev_body) + new_lines = [ln for ln in body if ln not in prev_set] + if any(ln.strip() for ln in new_lines): + return [heading, *new_lines] + return [] + + +def slice_by_dated_run(sections: list[tuple[str, list[str]]]) -> list[str]: + out: list[str] = [] + for heading, body in sections: + if not DATE_RE.search(heading): + break + out.append(heading) + out.extend(body) + return out + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--changelog", type=Path, default=Path("CHANGELOG.md")) - ap.add_argument("--since-date", required=True, - help="ISO date (YYYY-MM-DD) of the previous release; sections on or before it are excluded") + ap.add_argument("--prev-ref", default="", + help="git ref of the previous release tag; sections already present in " + "CHANGELOG.md at that ref are excluded. Omit / unresolvable => emit " + "the leading run of date-headed sections.") args = ap.parse_args() - try: - since = dt.date.fromisoformat(args.since_date) - except ValueError: - print(f"extract_changelog: --since-date is not an ISO date: {args.since_date!r}", file=sys.stderr) - return 2 - try: text = args.changelog.read_text() except OSError as e: print(f"extract_changelog: cannot read {args.changelog}: {e}", file=sys.stderr) return 2 - out: list[str] = [] - for heading, body in split_sections(text): - date = parse_heading_date(heading) - if date is None or date <= since: - break - out.append(heading) - out.extend(body) + sections = split_sections(text) + + prev_text = changelog_at_ref(args.prev_ref, args.changelog) if args.prev_ref else None + if prev_text is not None: + out = slice_by_prev_headings(sections, prev_text) + else: + out = slice_by_dated_run(sections) while out and not out[-1].strip(): out.pop() diff --git a/scripts/run_deterministic_matrix.py b/scripts/run_deterministic_matrix.py index ada155bd..4b4a24fb 100755 --- a/scripts/run_deterministic_matrix.py +++ b/scripts/run_deterministic_matrix.py @@ -3,9 +3,10 @@ This is a convenience wrapper around the checks that are useful before pushing: formatting, sleep guardrails, Debug tests, feature-minimal tests, ReleaseSafe -tests, ReleaseFast tests, ReleaseSmall tests, a musl static-target cross-build, -and fuzz harness compile-checks. ThreadSanitizer is available as an opt-in -because it is slower and can be noisy on some local systems. +tests, ReleaseFast tests, ReleaseSmall tests, a musl static-target build+run +(x86_64 Linux hosts only), and fuzz harness compile-checks. ThreadSanitizer is +available as an opt-in because it is slower and can be noisy on some local +systems. The ReleaseSmall step runs via `zig build test-release-small`, which forces the LLVM backend: Zig 0.16's self-hosted x86_64 backend mis-aligns read-only globals @@ -17,6 +18,7 @@ import argparse import os +import platform import subprocess import sys import time @@ -27,6 +29,16 @@ ROOT = Path(__file__).resolve().parents[1] +def musl_host_ok() -> bool: + """The musl step's `x86_64-linux-musl` test binaries are statically linked + and run natively only on an x86_64 Linux host. On any other host (macOS, + Windows, aarch64 Linux) `zig build test -Dtarget=...` cross-compiles + binaries the host cannot execute -- Zig then skips the run steps, so the + lane would report success having actually tested nothing. Omit it there; + CI runs it on `ubuntu-latest` regardless.""" + return sys.platform.startswith("linux") and platform.machine().lower() in ("x86_64", "amd64") + + @dataclass(frozen=True) class Step: name: str @@ -78,15 +90,16 @@ def steps(zig: str, include_tsan: bool) -> list[Step]: # build.zig comment. Switch to `["test", "-Doptimize=ReleaseSmall"]` at # the Zig 0.17 bump. Step("release-small", [zig, "build", "test-release-small", "-Doptimize=ReleaseSmall"]), - # musl / fully-static Linux target. `-Dtarget` is otherwise never - # cross-compiled in the matrix. A `-linux-musl` binary is statically - # linked and runs natively on a glibc x86_64 host, so this executes - # the full suite (not just a build check) and proves zzdds is - # musl-clean for Alpine / static-binary consumers. Host-arch only; - # aarch64-linux-musl would need qemu. - Step("musl", [zig, "build", "test", "-Dtarget=x86_64-linux-musl"]), - Step("fuzz", [zig, "build", "test-fuzz"]), ] + # musl / fully-static Linux target. `-Dtarget` is otherwise never + # cross-compiled in the matrix. A `-linux-musl` binary is statically linked + # and runs natively on a glibc x86_64 host, so this executes the full suite + # (not just a build check) and proves zzdds is musl-clean for Alpine / + # static-binary consumers. x86_64-Linux hosts only (see musl_host_ok); + # aarch64-linux-musl would need qemu. + if musl_host_ok(): + all_steps.append(Step("musl", [zig, "build", "test", "-Dtarget=x86_64-linux-musl"])) + all_steps.append(Step("fuzz", [zig, "build", "test-fuzz"])) if include_tsan: # Runs first: a fast fail-fast regression guard proving TSan can # still catch a real data race (see build.zig's @@ -123,6 +136,8 @@ def main() -> int: hint = "" if missing <= {"tsan", "tsan-self-check"}: hint = " (pass --include-tsan to enable the tsan/tsan-self-check steps)" + elif missing == {"musl"}: + hint = " (the musl step runs only on an x86_64 Linux host)" print( "Requested step(s) require additional flags: " + ", ".join(sorted(missing)) diff --git a/scripts/verify_release_bundle.py b/scripts/verify_release_bundle.py index aa85d5d5..2b6fedd0 100755 --- a/scripts/verify_release_bundle.py +++ b/scripts/verify_release_bundle.py @@ -27,10 +27,12 @@ compiles, links and runs `test/release-bundle/consumer.c`. `--configure-only` stops after step 4's `cmake` *configure* of the small -consumer (no compiler, no run, no examples). Used on Windows, where the -CMake/compiler example build path is not yet covered (same deferral as -`ci.yml`'s Windows Java binding) -- it still exercises the generated -`zzdds-config.cmake` and the bundled `zidl.exe`. +consumer (no compiler, no run, no examples). It is a local-debugging aid for a +host without a usable C/C++ toolchain; `release.yml` does not use it. In +particular it is NOT run on Windows -- the generated `zzdds-config.cmake` / +`zzdds.pc` are POSIX-shaped today (search `lib/` for the shared lib, no +`IMPORTED_IMPLIB`, `bin/zidl` not `bin/zidl.exe`), so `find_package(ZZDDS)` +does not configure there yet (tracked in `docs/roadmap.md`). """ from __future__ import annotations @@ -246,7 +248,9 @@ def main() -> int: ap.add_argument("--examples-dir", type=Path, default=REPO_ROOT / "examples", help="path to the folded-in examples/ tree (default: /examples)") ap.add_argument("--configure-only", action="store_true", - help="stop after cmake-configuring the small consumer (no compiler / no run / no examples)") + help="stop after cmake-configuring the small consumer (no compiler / no run / " + "no examples). Local-debugging aid for a host without a C/C++ toolchain; " + "not used by release.yml, and not usable on Windows yet (see module docstring)") ap.add_argument("--skip-example-run", action="store_true", help="build examples/{c,cpp}/hello_world against the bundle but do not run the " "pub/sub pair (link coverage only; use where live UDP DDS discovery is flaky, " From acd72d349a231ab27884851bdc08cce840f068a1 Mon Sep 17 00:00:00 2001 From: sqt <574914+sqt@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:27:04 +0000 Subject: [PATCH 3/3] resolving CI issues --- CHANGELOG.md | 11 ++++++----- scripts/extract_changelog.py | 36 +++++++++++++----------------------- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77f26cce..4652221d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,11 +71,12 @@ Dated entries (no release tags past `v0.2.1-zig.0.16.0`; `build.zig.zon` is - **Release prep — GitHub-release notes now come from `CHANGELOG.md`.** `release.yml`'s `publish` job built its release body from raw `git log --pretty=%s` subjects. It now quotes the `CHANGELOG.md` sections added since the previous release tag — - `scripts/extract_changelog.py` diffs the current section headings against `CHANGELOG.md` - as of that tag (so two releases on the same calendar day are handled: a plain date cutoff - would drop the second), falling back to the leading date-headed run when the tag predates - the file, and to raw commit subjects only if that yields nothing. Always appends a - `compare` link to the full commit log. + `scripts/extract_changelog.py` emits the leading run of sections whose heading is not + present in `CHANGELOG.md` as of that tag (whole-heading, not date, comparison), falling + back to the leading date-headed run when the tag predates the file, and to raw commit + subjects only if that yields nothing. Always appends a `compare` link. Two releases on + the *same calendar day* under one `## ` heading aren't distinguished — the second + gets the git-log fallback (fine for a hotfix; the notes are hand-editable). - **Decision recorded — pre-1.0 has no stability guarantee.** `docs/decisions.md` gains a "Versioning / Releases" section: any release may break the Zig API, the C ABI, the QoS/config schema, or the bundle layout, with no deprecation cycle; the C ABI stays in diff --git a/scripts/extract_changelog.py b/scripts/extract_changelog.py index 3f924c69..de2ed31c 100755 --- a/scripts/extract_changelog.py +++ b/scripts/extract_changelog.py @@ -7,11 +7,10 @@ CHANGELOG.md is a sequence of `## ` sections, newest first. The slice for a release is "every section added since the previous release tag". Given -`--prev-ref `, this compares the current CHANGELOG against the CHANGELOG as -it stood at that tag: it emits the leading run of sections whose headings are -new, and if there are none (two releases the same day, so the newest `## ` -heading already existed) it emits that section's heading plus only the body -lines added since. A plain date cutoff would drop the second same-day release. +`--prev-ref `, this emits the leading run of sections whose heading is not +present in CHANGELOG.md as it stood at that tag. Comparing whole headings (not +dates) means a section dated the same day as the previous tag is still emitted +as long as its heading text is new. If the previous tag predates CHANGELOG.md (or `--prev-ref` is omitted / not a resolvable ref), it falls back to emitting the leading run of date-headed @@ -19,6 +18,13 @@ the first heading with no date -- correct for the first release cut under the dated-entry scheme. +Two releases on the *same calendar day* that both write under one `## ` +heading are not distinguishable here: the second finds its heading already +present and emits nothing, so the caller falls back to a git-log body for it +(fine for a same-day hotfix; the maintainer can hand-edit the release notes). +Splitting bullets out of a shared dated section by line was tried and dropped -- +line-level diffing of prose fragments the entries. + Exit status: 0 one or more sections printed 1 nothing to emit (caller should fall back to a git-log body) @@ -70,30 +76,14 @@ def changelog_at_ref(ref: str, changelog: Path) -> str | None: def slice_by_prev_headings(sections: list[tuple[str, list[str]]], prev_text: str) -> list[str]: - prev_sections = split_sections(prev_text) - prev_headings = {h for h, _ in prev_sections} + prev_headings = {h for h, _ in split_sections(prev_text)} out: list[str] = [] for heading, body in sections: if heading in prev_headings: break out.append(heading) out.extend(body) - if out: - return out - - # Nothing under a brand-new heading. Handles two releases on the same day: - # the newest section's heading (e.g. "## 2026-09-02") already existed at the - # previous tag, but bullets were appended to it afterwards. Emit just the - # body lines that are new since then. - if sections: - heading, body = sections[0] - prev_body = next((b for h, b in prev_sections if h == heading), None) - if prev_body is not None: - prev_set = set(prev_body) - new_lines = [ln for ln in body if ln not in prev_set] - if any(ln.strip() for ln in new_lines): - return [heading, *new_lines] - return [] + return out def slice_by_dated_run(sections: list[tuple[str, list[str]]]) -> list[str]: