From c6bfbe4763dcc235fad2db5a2c538f9c4b65173f Mon Sep 17 00:00:00 2001 From: Repin Agent Date: Fri, 17 Jul 2026 08:46:23 -0600 Subject: [PATCH] =?UTF-8?q?test(m8a):=20prove=20the=20cutover=20mechanism?= =?UTF-8?q?=20=E2=80=94=20LISTENER=20SWAP=20(no=20NAT),=20isolated=20netns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Selects and proves the mechanism M9 will use to switch the public UDP trunk endpoint (tron:45070 + RTP 20000-20100) between FreeSWITCH and rustisk. The whole proof runs INSIDE ONE isolated container network namespace (--privileged --network none) on SYNTHETIC high ports over an internal veth pair + child netns ("chime" -> "tron"), so packets traverse prerouting+input exactly like the live trunk. It never touches the host nftables, host ports, the live trunk, port 45070, the router, or the cluster. No `flush ruleset`; the cutover table is separate and independently deletable. Selected mechanism: LISTENER SWAP. FS trunk profile stopped -> rustisk binds hostIP:45070 directly. Delivery is per-packet socket lookup — no NAT, no conntrack, no redirect — so the conntrack objection is structurally absent. A preinstalled fail-closed DROP on the guarded dport is enabled only across the handover so the sender sees loss (SIP retransmits) not ICMP port-unreachable. M9 invokes exactly these artifacts: - cutover_lib.sh — the filter-only nft primitives (table up/down, handover drop on/off). The only host-net surface. - apply-fs-to-rustisk.sh — APPLY (FS->rustisk): drop-on, stop FS, bind rustisk, drop-off. STOP_OLD/START_NEW injected as commands, so the orchestration is identical in the synthetic proof and the live cutover (M9 substitutes `sofia profile stop/start`). Measures window. - rollback-rustisk-to-fs.sh — ROLLBACK (rustisk->FS), the exact mirror. Proof (tests/m8a-cutover, run-proof.sh; RESULTS.md holds the passing transcript): 1. one fixed UDP five-tuple primed, numbered datagrams flowing continuously (seq 1..2296) across BOTH transitions, never restarted; 2. switch FS->rustisk: single clean delivery boundary, FS through seq 745, rustisk from 758 (12-datagram handover-drop gap), overlap_count=0 — no split-brain / no interleaving; 3. rollback rustisk->FS under the SAME flow: reverse boundary just as clean (rustisk through 1519, FS from 1533); 4. source-drop holds throughout both transitions for v4 AND v6 — untrusted 10.9.0.3 (v4) and fd00::3 (v6) delivered to NEITHER stand-in (0/0), a genuine DROP in an accept-policy chain; 5. handover windows measured both directions: FS->rustisk ~47-53 ms, rustisk->FS ~49-52 ms (mechanism floor; the live M9 window adds FS profile stop/start + DNS re-resolve). bind-on-command (main.rs:2189-2192 / PR #65) is NOT required on this evidence — flagged conditional, not built. RED control (teeth): a stateful dnat/redirect lever FAILS the proof where the listener swap passes — it rewrites the dport past the source-drop, delivering the untrusted source to the successor (delivered_to_RUSTISK=695). Honest finding: PLAN-v3's conntrack-persistence rollback objection did NOT reproduce on tron's kernel (redirect reverts cleanly on rule removal); the listener swap is still chosen for introducing no NAT/conntrack into a NAT-free path. A detector self-test proves assert_boundary rejects overlap/interleave/missing-rollback/ untrusted-delivery and accepts a clean capture. Synthetic ports only; live voice stack, Helm, host nft, router untouched. Co-authored-by: Repin Agent --- .gitignore | 4 + tests/m8a-cutover/Dockerfile | 28 ++++ tests/m8a-cutover/README.md | 116 +++++++++++++ tests/m8a-cutover/RESULTS.md | 140 ++++++++++++++++ tests/m8a-cutover/apply-fs-to-rustisk.sh | 43 +++++ tests/m8a-cutover/assert_boundary.py | 137 ++++++++++++++++ tests/m8a-cutover/burst.py | 35 ++++ tests/m8a-cutover/cutover_lib.sh | 81 +++++++++ tests/m8a-cutover/detector-selftest.sh | 56 +++++++ tests/m8a-cutover/in-container-proof.sh | 172 ++++++++++++++++++++ tests/m8a-cutover/listener.py | 146 +++++++++++++++++ tests/m8a-cutover/red-stateful-variant.sh | 123 ++++++++++++++ tests/m8a-cutover/rollback-rustisk-to-fs.sh | 42 +++++ tests/m8a-cutover/run-proof.sh | 51 ++++++ tests/m8a-cutover/sender.py | 75 +++++++++ 15 files changed, 1249 insertions(+) create mode 100644 tests/m8a-cutover/Dockerfile create mode 100644 tests/m8a-cutover/README.md create mode 100644 tests/m8a-cutover/RESULTS.md create mode 100755 tests/m8a-cutover/apply-fs-to-rustisk.sh create mode 100755 tests/m8a-cutover/assert_boundary.py create mode 100755 tests/m8a-cutover/burst.py create mode 100755 tests/m8a-cutover/cutover_lib.sh create mode 100755 tests/m8a-cutover/detector-selftest.sh create mode 100755 tests/m8a-cutover/in-container-proof.sh create mode 100755 tests/m8a-cutover/listener.py create mode 100755 tests/m8a-cutover/red-stateful-variant.sh create mode 100755 tests/m8a-cutover/rollback-rustisk-to-fs.sh create mode 100755 tests/m8a-cutover/run-proof.sh create mode 100755 tests/m8a-cutover/sender.py diff --git a/.gitignore b/.gitignore index d117eb9..d47573d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ # Third-party pjproject source fetched by scripts/fetch-pjproject.sh for the # optional `pjproject-cffi` feature of the pjsip-shim crate. Not committed. /crates/pjsip-shim/vendor/ + +# python bytecode +__pycache__/ +*.pyc diff --git a/tests/m8a-cutover/Dockerfile b/tests/m8a-cutover/Dockerfile new file mode 100644 index 0000000..fde7dc0 --- /dev/null +++ b/tests/m8a-cutover/Dockerfile @@ -0,0 +1,28 @@ +# M8a cutover-mechanism proof container. +# +# Built with network (to install packages), then RUN with +# --network none --cap-add=NET_ADMIN +# so the proof executes in a private, empty network namespace: only `lo`, no +# route to the host, its OWN nftables ruleset. Nothing it does (nft rules, port +# binds, floods, conntrack) can reach the host netns, the live trunk, port +# 45070, the router, or the cluster. +FROM debian:stable-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + nftables \ + iproute2 \ + conntrack \ + python3 \ + procps \ + iputils-ping \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /m8a +COPY listener.py sender.py burst.py assert_boundary.py cutover_lib.sh \ + apply-fs-to-rustisk.sh rollback-rustisk-to-fs.sh red-stateful-variant.sh \ + detector-selftest.sh in-container-proof.sh /m8a/ +RUN chmod +x /m8a/*.sh /m8a/*.py + +# The container just sleeps; the driver drives it via `docker exec`. +CMD ["sleep", "infinity"] diff --git a/tests/m8a-cutover/README.md b/tests/m8a-cutover/README.md new file mode 100644 index 0000000..d945032 --- /dev/null +++ b/tests/m8a-cutover/README.md @@ -0,0 +1,116 @@ +# M8a — Cutover mechanism: proof of the LISTENER SWAP + +This directory selects and **proves** the mechanism M9 will use to switch the +public UDP trunk endpoint (`tron:45070`, RTP `20000-20100`) between FreeSWITCH +and rustisk. M9 invokes exactly these artifacts. + +The whole proof runs inside **one isolated container network namespace** +(`--privileged --network none`) on **synthetic high ports** — it never touches +the host nftables, host ports, the live trunk, port 45070, the router, or the +cluster. See "Isolation" below. + +## Selected mechanism: LISTENER SWAP (no NAT) + +FreeSWITCH's trunk profile is **stopped** (freeing `hostIP:45070`, FS process +stays up), then rustisk **binds `hostIP:45070` directly**. Delivery is by +per-packet socket lookup; **no NAT, no conntrack, no redirect**. A preinstalled +**fail-closed DROP** on the guarded dport is enabled only across the handover so +the sender sees plain loss (SIP retransmission covers it) instead of ICMP +port-unreachable. + +Why this over the alternates (PLAN-v3 design table, §M8a): + +| Shape | Stateless? | Verdict | +|---|---|---| +| **Listener swap** — FS profile stop → rustisk binds 45070. No NAT. | **Yes** — per-packet socket lookup; conntrack never steers un-NAT'd traffic. | **SELECTED** | +| prerouting `redirect`/`dnat` to an alt port | No — stateful NAT | DEAD (see RED control) | +| Host→pod DNAT | No — stateful NAT + new forward-hook policy | strictly worse | +| conntrack-zone toggle wrapping a NAT lever | isolates state deliberately | documented alternate | +| tc/eBPF stateless rewrite | Yes | heavy lift, new failure surface | +| re-point the **router** forward | No — NAT we can't inspect/flush | rejected | + +Verified reasons the listener swap wins (all confirmed by the proof): +1. **No NAT ⇒ the conntrack objection is structurally absent.** The table this + mechanism installs (`cutover_lib.sh`) is filter-only: no `ct`, `nat`, + `dnat`, or `redirect` — same shape as today's `voice-trunk.nft`. +2. **The existing source allowlist keeps protecting `udp dport 45070` + unchanged** — same port, same host, same rule. No new blast radius. The RED + control shows a port-rewriting lever silently **bypasses** that drop. +3. **Per-profile stop/start on live FS is already proven in-repo** (the trunk + watchdog's `sofia profile start`). Rollback is that same command. +4. **rustisk binds the public port directly**, so Via/Contact carry 45070 — + New-3's external-port hazard N/A on the primary path. + +Design constraints honored: **New-5** (no K8s Service on the trunk ports — a +NodePort would make kube-proxy install stateful DNAT and reintroduce the defect; +the mechanism is hostNetwork, no Service); **New-3** (direct bind of the public +port); and **no `flush ruleset` ever** — the cutover table is separate and +independently deletable (`nft delete table inet cutover`). + +## What M9 invokes (the artifacts) + +| File | Role | +|---|---| +| `cutover_lib.sh` | the nft primitives: `cutover_table_up/down`, `handover_drop_on/off`. **Filter-only table.** The only host-networking surface. | +| `apply-fs-to-rustisk.sh` | **APPLY** (FS→rustisk): drop-on → stop FS → wait released → bind rustisk → wait bound → drop-off. Measures + prints the window. | +| `rollback-rustisk-to-fs.sh` | **ROLLBACK** (rustisk→FS): the exact mirror; `START_NEW_CMD` is the watchdog's `sofia profile start`. | + +Both scripts inject the FS/rustisk specifics as commands, so the orchestration +is **identical** in this synthetic proof and in the live M9 cutover — only the +hooks differ: + +``` +# M8a synthetic (this harness): +STOP_OLD_CMD="kill -USR2 $FS_PID" # release the port from the stand-in +START_NEW_CMD="kill -USR1 $RUSTISK_PID" # bind the port on the stand-in + +# M9 live (substitute): +PORT=45070 NFT_TABLE=voicefw \ +STOP_OLD_CMD="fs_cli -x 'sofia profile stop'" \ +WAIT_RELEASED_CMD="" \ +START_NEW_CMD="" \ +WAIT_BOUND_CMD="" \ + ./apply-fs-to-rustisk.sh +# rollback: STOP_OLD_CMD stops rustisk; START_NEW_CMD = fs_cli -x 'sofia profile start' +``` + +In M9 the source-drops are the **existing** `voice-trunk.nft` allowlist (already +protecting dport 45070) — the cutover adds no new firewall rules. The +`UNTRUSTED_V4/V6` knobs here exist only to *prove* the drop holds across the +handover. + +## The proof harness + +| File | Role | +|---|---| +| `run-proof.sh` | top-level driver: builds the image, runs it `--privileged --network none`, asserts host-side `NetworkMode=none`, execs the proof, reaps the container + image. | +| `Dockerfile` | debian-slim + nftables/iproute2/conntrack/python3. | +| `in-container-proof.sh` | steps 1–5 over a veth pair + child netns ("chime"→"tron"), so traffic traverses **prerouting+input** like the real trunk (not loopback). | +| `listener.py` | stand-in that binds the port **on command** (SIGUSR1/2), dual-stack, and writes one CSV row per received datagram — the receiver-side ground truth. **No SO_REUSEPORT**: exactly one holder at a time. | +| `sender.py` / `burst.py` | numbered-datagram senders pinning a fixed five-tuple. | +| `assert_boundary.py` | receiver-side assertion: disjoint captures, single clean FS→RUSTISK→FS boundary per transition, and untrusted tags delivered **nowhere**. | +| `red-stateful-variant.sh` | RED control — a stateful `dnat`/redirect lever that **fails** the proof where the listener swap passes (teeth). | +| `detector-selftest.sh` | proves `assert_boundary.py` itself rejects crafted pathologies (overlap, interleave, missing rollback, untrusted delivery). | +| `RESULTS.md` | the committed passing transcript + measured windows. | + +## Run it + +``` +./run-proof.sh # build + run + reap, transcript to stdout +TRANSCRIPT=out.txt ./run-proof.sh # also tee the transcript to out.txt +``` + +Requires Docker with `--privileged` (for nft + nested netns inside the +container). If privileged Docker is unavailable, the proof **stops** — it never +falls back to the host netns. + +## Isolation (the whole risk) + +- `--network none` ⇒ the container's netns has only `lo`; the internal veth pair + lives entirely inside it. There is **no route to the host** (the proof asserts + `192.168.0.109` is unreachable) and no published ports (`run-proof.sh` asserts + `NetworkMode=none`). +- The container has its **own** nftables ruleset (empty before setup) — separate + from the host's. Every `nft`/`ip`/`conntrack`/bind runs inside the container. +- Synthetic ports only (`55070`, `55190/55192`) — never `45070`/`20000-20100`. +- **No `flush ruleset`** anywhere; tables are dropped by name. diff --git a/tests/m8a-cutover/RESULTS.md b/tests/m8a-cutover/RESULTS.md new file mode 100644 index 0000000..52214cb --- /dev/null +++ b/tests/m8a-cutover/RESULTS.md @@ -0,0 +1,140 @@ +# M8a — Results (measured windows + passing transcript) + +Mechanism: **LISTENER SWAP** (no NAT). Verdict: **GO** for M9, gated on M0a's +Chime endpoint-down tolerance vs. the measured window (below). + +Run it yourself: `./run-proof.sh`. The transcript below is representative; +windows vary a few ms run to run. Reproduced cleanly across repeated runs. + +## Measured handover windows (both directions) + +| Direction | Mechanism-floor window | Receiver-observed delivery gap | +|---|---|---| +| **FS → rustisk** (apply) | **~47–53 ms** | ~26–28 ms (12–13 datagrams @ 2 ms) | +| **rustisk → FS** (rollback) | **~49–52 ms** | ~28–30 ms (13–14 datagrams @ 2 ms) | + +The **mechanism-floor** window is nft-drop-toggle + socket close + socket bind + +IPC/status-poll, measured in an isolated netns. It is the *floor*: the **live M9 +window additionally includes FS `sofia profile stop/start` + DNS re-resolve**, +which this synthetic proof cannot measure. Feed the live number from M0a. + +### bind-on-command note (go/no-go) + +The synthetic floor (~50 ms) is well under any plausible SIP retransmission / +endpoint-down tolerance, and the fail-closed drop converts the gap to loss (not +ICMP reject). **On this evidence, bind-on-command is NOT required** — do not +build it yet. Revisit only if M0a's live measurement (FS profile stop/start + +DNS) pushes the *rustisk-side* bind past Chime's tolerance; then the mitigation +is bind-on-command for rustisk (today the SIP transport binds at startup and a +bind failure is fatal by design — `crates/rustisk-cli/src/main.rs:2189-2192`, +PR #65) so rustisk can claim `45070` the instant FS releases it. Flagged for the +coordinator as a **conditional** work item, not scheduled. + +## Passing transcript (steps 1–5 + RED control + detector self-test) + +``` +=================== M8a LISTENER-SWAP PROOF =================== +synthetic dport=55070 (NOT 45070) rate=2ms ingress via veth (prerouting+input) +--- ISOLATION (must all hold before traffic) --- +main-netns links: lo,veth0, + host LAN 192.168.0.109: NO ROUTE (isolated) OK + nft ruleset before setup: +--- cutover table (filter only; no nat/ct/redirect) --- + table inet cutover { + chain input { + type filter hook input priority filter - 10; policy accept; + udp dport 55070 ip saddr 10.9.0.3 drop comment "srcdrop-v4" + udp dport 55070 ip6 saddr fd00::3 drop comment "srcdrop-v6" + } + } +step 1: primed TRUST(v4) five-tuple 10.9.0.2:41002 -> 10.9.0.1:55070 + untrusted EVILV4(10.9.0.3) + EVILV6(fd00::3) flooding continuously +step 2: APPLY switch FS -> rustisk +APPLY FS->rustisk handover window: 47 ms +step 3: ROLLBACK rustisk -> FS (same continuously-flowing tuple) +ROLLBACK rustisk->FS handover window: 52 ms +--------------- RECEIVER-SIDE ASSERTIONS (listener swap) --------------- +PASS source-drop EVILV4: delivered_to_FS=0 delivered_to_RUSTISK=0 (must be 0/0) +PASS source-drop EVILV6: delivered_to_FS=0 delivered_to_RUSTISK=0 (must be 0/0) +PASS disjoint captures (no split-brain): overlap_count=0 +PASS ownership run pattern by seq = [FS,RUSTISK,FS] (expected [FS,RUSTISK,FS]) +PASS boundary FS->RUSTISK: last FS seq=745, first RUSTISK seq=758 (clean: FS.max < RUSTISK.min); handover gap = 12 datagrams / 26.0 ms delivery gap +PASS boundary RUSTISK->FS: last RUSTISK seq=1519, first FS seq=1533 (clean: RUSTISK.max < FS.min); handover gap = 13 datagrams / 28.0 ms delivery gap +trusted captured: FS=1509 RUSTISK=762 seq_span=1..2296 +RESULT: PASS +--------------- MEASURED HANDOVER WINDOWS (mechanism floor) --------------- +FS->rustisk apply window: 47.1 ms +rustisk->FS rollback window: 52.3 ms + +=================== RED CONTROL (stateful redirect lever) =================== +--- (A) source-drop bypass: untrusted 10.9.0.3 -> 55190, drop is on dport 55190 --- + (redirect DNATs 55190->55192 at prerouting, past the dport-55190 drop) +--- (B) rollback-persistence probe (honest): switch then rollback a primed tuple --- + cumulative captured lines (FS=55190, RU=55192): + prime (no rule): [FS=822 RU=0] + after switch (dnat): [FS=824 RU=1556] (RU grew => switch worked) + after rollback (del):[FS=1373 RU=1582] (FS grew => reverted cleanly) + => rollback reverts to FS on tron's kernel; conntrack-persistence objection NOT reproduced. +--- RED machine verdict (assert_boundary on the redirect source-drop capture) --- +FAIL source-drop EVILV4: delivered_to_FS=0 delivered_to_RUSTISK=695 (must be 0/0) +FAIL ownership run pattern by seq = [RUSTISK] (expected [FS,RUSTISK,FS]) +RESULT: FAIL +RED TEETH CONFIRMED: redirect lever FAILED the proof (untrusted delivered / no clean swap) — listener swap PASSES the same checks. + +=================== DETECTOR SELF-TEST (assert_boundary teeth) =================== + detector[good]: expected PASS, got PASS OK + detector[overlap]: expected FAIL, got FAIL OK + detector[interleave]: expected FAIL, got FAIL OK + detector[norollback]: expected FAIL, got FAIL OK + detector[untrusted]: expected FAIL, got FAIL OK +detector self-test: ALL OK (accepts clean, rejects every pathology) + +=================== SUMMARY =================== +listener-swap assert rc=0 (0 = clean boundaries + source-drop hold) +RED control (redirect) rc=0 (0 = redirect FAILED the proof as expected -> teeth) +detector self-test rc=0 (0 = assert_boundary rejects every crafted bad capture) +M8a RESULT: PASS +``` + +## What each step proved + +1. **Primed tuple** `10.9.0.2:41002 → 10.9.0.1:55070`, numbered datagrams flowing + continuously (seq 1…2296), never restarted across both transitions. +2. **Switch FS→rustisk**: single clean boundary — FS captured through seq **745**, + rustisk from seq **758**; the 12-datagram gap is the handover drop window. No + overlap, no interleaving, no split-brain (`overlap_count=0`). +3. **Rollback rustisk→FS** under the *same* flow: reverse boundary just as clean — + rustisk through **1519**, FS from **1533**. (This is the step a stateful/redirect + lever is claimed to fail; the no-NAT swap passes it.) +4. **Source drop holds throughout** both transitions for **v4 and v6**: the + untrusted `10.9.0.3` (v4) and `fd00::3` (v6) sources were delivered to + **neither** stand-in at any point (`0/0`). A genuine DROP in an accept-policy + chain, not an allow rule. +5. **Windows** measured both directions (above). + +## RED control — teeth, and an honest finding + +The RED control runs a **stateful `dnat`/redirect lever** through the same path: + +- **(A) It FAILS the proof deterministically.** Because it rewrites the dport + (`55190 → 55192`) at prerouting, the fail-closed source-drop written for dport + `55190` no longer matches, so the **untrusted source is delivered** to the + successor (`delivered_to_RUSTISK=695`). `assert_boundary` flags it → **RED**. + The listener swap keeps the same port, so its drop holds (`0/0`). This is + exactly PLAN-v3's hazard: *"any port-rewriting mechanism moves packets to a + port the current filter does not match… would need a genuine fail-closed DROP + for untrusted sources on the new port, v4 and v6."* + +- **(B) Honest negative result on the conntrack-persistence claim.** PLAN-v3 C1 + argues the redirect *cannot switch a primed tuple back* because conntrack + persists. **Measured on tron's kernel, it reverts cleanly on rule removal** + (the probe shows FS reclaiming the flow after the rule is deleted; a directly + injected stale conntrack entry was ignored). So on this kernel the redirect + dies on **(A)** — the security regression — not on the rollback boundary. The + listener swap is still the right choice: it introduces **no** NAT/conntrack + into a NAT-free path, and its behavior does not depend on kernel/conntrack + semantics that were observed to vary by birth-condition during this work. + +- The **detector self-test** proves `assert_boundary` is not rigged to always + pass: it rejects overlap/split-brain, interleaving, a missing rollback + boundary, and any untrusted delivery, while accepting a clean capture. diff --git a/tests/m8a-cutover/apply-fs-to-rustisk.sh b/tests/m8a-cutover/apply-fs-to-rustisk.sh new file mode 100755 index 0000000..94d9b55 --- /dev/null +++ b/tests/m8a-cutover/apply-fs-to-rustisk.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# apply-fs-to-rustisk.sh — THE APPLY ARTIFACT M9 invokes (FS -> rustisk). +# +# Listener swap, no NAT. Sequence: +# 1. handover_drop_on — fail-closed drop on the guarded dport, so the +# sender sees loss (SIP retransmits) not ICMP reject +# during the window the port is unbound. +# 2. STOP_OLD_CMD — release the port from the incumbent (FS). +# M9: fs_cli -x 'sofia profile stop' +# 3. WAIT_RELEASED_CMD — block until the port is actually free. +# 4. START_NEW_CMD — bind the port on the successor (rustisk). +# M9: start rustisk / claim hostIP:45070. +# 5. WAIT_BOUND_CMD — block until the successor holds the port. +# 6. handover_drop_off — reopen; traffic now lands on rustisk. +# +# The FS/rustisk specifics are injected as commands so THIS orchestration is +# identical in the M8a synthetic proof and in the M9 live cutover — only the +# STOP_OLD/START_NEW hooks differ. The measured wall-clock between step 1 and +# step 6 is the handover window reported as M9's go/no-go input. +# +# Env: PORT, NFT_TABLE (+ UNTRUSTED_V4/6 already installed by the caller), +# STOP_OLD_CMD, WAIT_RELEASED_CMD, START_NEW_CMD, WAIT_BOUND_CMD, +# WINDOW_OUT (file to write the measured window, ns). +set -euo pipefail +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=cutover_lib.sh +source "${HERE}/cutover_lib.sh" + +: "${STOP_OLD_CMD:?}" ; : "${START_NEW_CMD:?}" +: "${WAIT_RELEASED_CMD:=true}" ; : "${WAIT_BOUND_CMD:=true}" + +t0="$(date +%s%N)" +handover_drop_on +eval "${STOP_OLD_CMD}" +eval "${WAIT_RELEASED_CMD}" +eval "${START_NEW_CMD}" +eval "${WAIT_BOUND_CMD}" +handover_drop_off +t1="$(date +%s%N)" + +win_ms=$(( (t1 - t0) / 1000000 )) +echo "APPLY FS->rustisk handover window: ${win_ms} ms (${t0} -> ${t1} ns)" +if [ -n "${WINDOW_OUT:-}" ]; then echo "$((t1 - t0))" > "${WINDOW_OUT}"; fi diff --git a/tests/m8a-cutover/assert_boundary.py b/tests/m8a-cutover/assert_boundary.py new file mode 100755 index 0000000..1bc1b8c --- /dev/null +++ b/tests/m8a-cutover/assert_boundary.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""assert_boundary.py — the RECEIVER-SIDE proof for M8a. + +Consumes the two stand-in capture files (FS + rustisk) written by listener.py +and asserts, on what each stand-in ACTUALLY CAPTURED (never a sender log): + + Boundary integrity for the primed tuple (--trust-tag): + * The two capture sets are DISJOINT (no seq delivered to both) -> no + split-brain / no duplication. + * Sorted by seq, ownership forms clean contiguous runs with a GAP between + them (the handover drop window). Expected run pattern for a switch + + rollback under one continuous flow: FS, RUSTISK, FS. + * Each ownership change is a single clean boundary: max(prev run) < + min(next run), and the intervening seqs (the gap) were delivered to + NEITHER — i.e. dropped during the handover, not misrouted. + + Source-drop (--untrusted-tags): NONE of those tags appear in EITHER capture, + anywhere, at any time (before/during/after both transitions). A genuine + DROP, proven for whatever families the untrusted senders used (v4 + v6). + +Prints the switch boundary (N -> N+gap+1), the rollback boundary, the per- +transition delivery gap in datagrams and ms, and exits non-zero on any +violation. +""" +import argparse +import sys + + +def load(path): + rows = [] + try: + with open(path) as f: + for line in f: + line = line.strip() + if not line: + continue + # recv_ts_ns,label,src,tag,seq + parts = line.split(",") + if len(parts) < 5: + continue + ts, label, src, tag, seq = parts[0], parts[1], parts[2], parts[3], parts[4] + try: + rows.append((int(ts), label, src, tag, int(seq))) + except ValueError: + continue + except FileNotFoundError: + pass + return rows + + +def runs_by_seq(delivered): + """delivered: list of (seq, owner, ts). Return compressed ownership runs: + [(owner, min_seq, max_seq, count, first_ts, last_ts), ...] ordered by seq, + and flag any interleaving (owner changes back and forth within adjacent + seqs without a clean split).""" + d = sorted(delivered, key=lambda r: r[0]) + runs = [] + for seq, owner, ts in d: + if runs and runs[-1][0] == owner: + o, lo, hi, cnt, fts, lts = runs[-1] + runs[-1] = (o, lo, seq, cnt + 1, fts, ts) + else: + runs.append((owner, seq, seq, 1, ts, ts)) + return runs + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--fs", required=True) + ap.add_argument("--rustisk", required=True) + ap.add_argument("--trust-tag", default="TRUST") + ap.add_argument("--untrusted-tags", nargs="*", default=["EVILV4", "EVILV6"]) + ap.add_argument("--expect-runs", default="FS,RUSTISK,FS", + help="expected ownership run pattern by seq") + args = ap.parse_args() + + fs = load(args.fs) + ru = load(args.rustisk) + ok = True + + def emit(good, msg): + nonlocal ok + print(("PASS" if good else "FAIL") + " " + msg) + if not good: + ok = False + + # ---- Source-drop: untrusted tags must appear NOWHERE ---- + for tag in args.untrusted_tags: + nfs = sum(1 for r in fs if r[3] == tag) + nru = sum(1 for r in ru if r[3] == tag) + emit(nfs == 0 and nru == 0, + f"source-drop {tag}: delivered_to_FS={nfs} delivered_to_RUSTISK={nru} " + f"(must be 0/0)") + + # ---- Boundary integrity for the primed trusted tuple ---- + fs_seq = {r[4]: r[0] for r in fs if r[3] == args.trust_tag} + ru_seq = {r[4]: r[0] for r in ru if r[3] == args.trust_tag} + overlap = sorted(set(fs_seq) & set(ru_seq)) + emit(len(overlap) == 0, + f"disjoint captures (no split-brain): overlap_count={len(overlap)}" + + (f" e.g. {overlap[:5]}" if overlap else "")) + + delivered = ([(s, "FS", t) for s, t in fs_seq.items()] + + [(s, "RUSTISK", t) for s, t in ru_seq.items()]) + if not delivered: + emit(False, "no trusted datagrams captured at all") + print("RESULT: FAIL") + sys.exit(1) + + runs = runs_by_seq(delivered) + pattern = ",".join(r[0] for r in runs) + expect = args.expect_runs + emit(pattern == expect, + f"ownership run pattern by seq = [{pattern}] (expected [{expect}])") + + # Clean boundary + gap between each adjacent run. + for i in range(len(runs) - 1): + o1, lo1, hi1, c1, f1, l1 = runs[i] + o2, lo2, hi2, c2, f2, l2 = runs[i + 1] + clean = hi1 < lo2 + gap_dgrams = lo2 - hi1 - 1 + gap_ms = (f2 - l1) / 1e6 + label = f"{o1}->{o2}" + emit(clean, + f"boundary {label}: last {o1} seq={hi1}, first {o2} seq={lo2} " + f"(clean: {o1}.max < {o2}.min); handover gap = {gap_dgrams} datagrams / " + f"{gap_ms:.1f} ms delivery gap") + + allseq = set(fs_seq) | set(ru_seq) + print(f"trusted captured: FS={len(fs_seq)} RUSTISK={len(ru_seq)} " + f"seq_span={min(allseq)}..{max(allseq)}") + print("RESULT: " + ("PASS" if ok else "FAIL")) + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/m8a-cutover/burst.py b/tests/m8a-cutover/burst.py new file mode 100755 index 0000000..91614b5 --- /dev/null +++ b/tests/m8a-cutover/burst.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Send a BOUNDED burst of numbered datagrams from a fixed five-tuple, then +exit. Used by the RED control's honest rollback-persistence probe (part B). +Drains any reply so the conntrack flow becomes bidirectional/established.""" +import argparse, socket, time + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--src-ip", required=True) + ap.add_argument("--src-port", type=int, required=True) + ap.add_argument("--dst-ip", required=True) + ap.add_argument("--dst-port", type=int, required=True) + ap.add_argument("--tag", default="X") + ap.add_argument("--count", type=int, required=True) + ap.add_argument("--rate-ms", type=float, default=4.0) + ap.add_argument("--family", type=int, choices=(4, 6), default=4) + a = ap.parse_args() + fam = socket.AF_INET if a.family == 4 else socket.AF_INET6 + s = socket.socket(fam, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((a.src_ip, a.src_port)) + s.settimeout(0.002) + for i in range(1, a.count + 1): + s.sendto(f"{a.tag} {i}".encode(), (a.dst_ip, a.dst_port)) + try: + s.recvfrom(200) + except OSError: + pass + time.sleep(a.rate_ms / 1000.0) + s.close() + + +if __name__ == "__main__": + main() diff --git a/tests/m8a-cutover/cutover_lib.sh b/tests/m8a-cutover/cutover_lib.sh new file mode 100755 index 0000000..a347d7d --- /dev/null +++ b/tests/m8a-cutover/cutover_lib.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# cutover_lib.sh — the nftables primitives the listener-swap cutover uses. +# +# This is the ONLY host-networking surface of the mechanism. It is deliberately +# tiny and it is a FILTER-ONLY table: no `nat`, no `ct`, no `redirect`, no +# `dnat` — so the conntrack objection that kills the redirect design is +# structurally absent (PLAN-v3 C1 / M8a rationale point 1). +# +# Safety, mirroring local-networking/nftables/voice-trunk.nft: +# * We create our OWN table (`inet ${NFT_TABLE}`), default `policy accept`, +# and touch nothing else. NEVER `flush ruleset` — the whole table is +# dropped as a unit and is independently deletable. +# * The only drops are the ones we NAME: the persistent untrusted-source drops +# (v4 + v6) and the transient handover drop. A targeted drop on one dport +# cannot brick the host (M8a runs it in a private netns regardless). +# +# In M9 this runs on the host against PORT=45070; in the M8a proof the harness +# points PORT at a synthetic high port inside an isolated container netns. +# +# Env: +# NFT_TABLE nft table name (default: cutover) +# PORT the guarded UDP dport (required) +# UNTRUSTED_V4 untrusted v4 source to DROP (optional, for the proof) +# UNTRUSTED_V6 untrusted v6 source to DROP (optional, for the proof) +set -euo pipefail + +NFT_TABLE="${NFT_TABLE:-cutover}" + +_nft() { nft "$@"; } + +# Create the cutover table + input chain, and install the PERSISTENT +# fail-closed source drops (v4 and v6) that hold across BOTH transitions. +# policy accept: trusted traffic falls through; only named sources are dropped. +cutover_table_up() { + : "${PORT:?PORT required}" + _nft add table inet "${NFT_TABLE}" + _nft "add chain inet ${NFT_TABLE} input { type filter hook input priority -10 ; policy accept ; }" + if [ -n "${UNTRUSTED_V4:-}" ]; then + _nft add rule inet "${NFT_TABLE}" input meta l4proto udp udp dport "${PORT}" ip saddr "${UNTRUSTED_V4}" drop comment '"srcdrop-v4"' + fi + if [ -n "${UNTRUSTED_V6:-}" ]; then + _nft add rule inet "${NFT_TABLE}" input meta l4proto udp udp dport "${PORT}" ip6 saddr "${UNTRUSTED_V6}" drop comment '"srcdrop-v6"' + fi +} + +# Drop the whole table (independently deletable; never `flush ruleset`). +cutover_table_down() { + _nft delete table inet "${NFT_TABLE}" 2>/dev/null || true +} + +# Enable the transient fail-closed handover drop on the guarded dport. +# Inserted at the head so it is evaluated first. While it is present the sender +# sees plain LOSS (SIP retransmission covers it), NOT ICMP port-unreachable — +# which is the whole reason the port must never be left merely unbound. +handover_drop_on() { + : "${PORT:?PORT required}" + _nft insert rule inet "${NFT_TABLE}" input meta l4proto udp udp dport "${PORT}" drop comment '"handover"' +} + +# Disable the handover drop (delete by comment handle). +handover_drop_off() { + local h + h="$(nft -a list chain inet "${NFT_TABLE}" input 2>/dev/null \ + | awk '/comment "handover"/ {for(i=1;i<=NF;i++) if($i=="handle") print $(i+1)}' \ + | head -n1)" + if [ -n "${h}" ]; then + _nft delete rule inet "${NFT_TABLE}" input handle "${h}" + fi +} + +# Allow being sourced OR invoked as `cutover_lib.sh `. +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + cmd="${1:?usage: cutover_lib.sh }" + case "${cmd}" in + up) cutover_table_up ;; + down) cutover_table_down ;; + drop_on) handover_drop_on ;; + drop_off) handover_drop_off ;; + *) echo "unknown: ${cmd}" >&2; exit 2 ;; + esac +fi diff --git a/tests/m8a-cutover/detector-selftest.sh b/tests/m8a-cutover/detector-selftest.sh new file mode 100755 index 0000000..a8fda1d --- /dev/null +++ b/tests/m8a-cutover/detector-selftest.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# detector-selftest.sh — proves assert_boundary.py itself has TEETH: it must +# ACCEPT a clean capture and REJECT every crafted pathology (split-brain/overlap, +# interleaving, a missing rollback boundary, and an untrusted-source delivery). +# This guarantees the listener-swap PASS is meaningful, not a rigged always-pass. +set -uo pipefail +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +RUN="${RUN:-/m8a/run}"; D="${RUN}/selftest"; rm -rf "${D}"; mkdir -p "${D}" + +gen() { python3 - "$@" <<'PY' +import sys +mode=sys.argv[1]; fs=open(sys.argv[2],"w"); ru=open(sys.argv[3],"w") +def row(f,label,tag,seq): f.write(f"{seq*1000000},{label},10.9.0.2:41002,{tag},{seq}\n") +if mode=="good": # FS 1..100 , RU 201..300 , FS 401..500 (clean FS,RU,FS) + for s in range(1,101): row(fs,"FS","TRUST",s) + for s in range(201,301): row(ru,"RUSTISK","TRUST",s) + for s in range(401,501): row(fs,"FS","TRUST",s) +elif mode=="overlap": # split-brain: RU shares seqs with FS + for s in range(1,101): row(fs,"FS","TRUST",s) + for s in range(50,151): row(ru,"RUSTISK","TRUST",s) +elif mode=="interleave": # port flapping: odd->FS even->RU + for s in range(1,201): + (row(fs,"FS","TRUST",s) if s%2 else row(ru,"RUSTISK","TRUST",s)) +elif mode=="norollback": # switched but never returned to FS + for s in range(1,101): row(fs,"FS","TRUST",s) + for s in range(201,401): row(ru,"RUSTISK","TRUST",s) +elif mode=="untrusted": # clean boundary but an untrusted datagram slipped in + for s in range(1,101): row(fs,"FS","TRUST",s) + for s in range(201,301): row(ru,"RUSTISK","TRUST",s) + for s in range(401,501): row(fs,"FS","TRUST",s) + row(ru,"RUSTISK","EVILV4",250) +fs.close(); ru.close() +PY +} + +run_case() { # + local mode="$1" want="$2" + gen "${mode}" "${D}/${mode}_fs.csv" "${D}/${mode}_ru.csv" + python3 "${HERE}/assert_boundary.py" --fs "${D}/${mode}_fs.csv" --rustisk "${D}/${mode}_ru.csv" \ + --trust-tag TRUST --untrusted-tags EVILV4 EVILV6 --expect-runs FS,RUSTISK,FS >/dev/null 2>&1 + local rc=$? + local got; [ "${rc}" -eq 0 ] && got=PASS || got=FAIL + if [ "${got}" = "${want}" ]; then echo " detector[$mode]: expected ${want}, got ${got} OK"; return 0 + else echo " detector[$mode]: expected ${want}, got ${got} MISMATCH"; return 1; fi +} + +echo "=================== DETECTOR SELF-TEST (assert_boundary teeth) ===================" +rc=0 +run_case good PASS || rc=1 +run_case overlap FAIL || rc=1 +run_case interleave FAIL || rc=1 +run_case norollback FAIL || rc=1 +run_case untrusted FAIL || rc=1 +[ "${rc}" -eq 0 ] && echo "detector self-test: ALL OK (accepts clean, rejects every pathology)" \ + || echo "detector self-test: FAILED" +exit "${rc}" diff --git a/tests/m8a-cutover/in-container-proof.sh b/tests/m8a-cutover/in-container-proof.sh new file mode 100755 index 0000000..82000ef --- /dev/null +++ b/tests/m8a-cutover/in-container-proof.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# in-container-proof.sh — the M8a proof, executed INSIDE the isolated container +# netns (--privileged --network none). Synthetic ports only; never 45070. +# +# Topology (models Chime -> tron over a real interface, so packets traverse the +# PREROUTING + INPUT hooks exactly like the live trunk — NOT loopback): +# +# netns "chime" (the untrusted internet / AWS side) netns: container main ("tron") +# veth1: 10.9.0.2 (trusted src) veth0: 10.9.0.1 fd00::1 +# 10.9.0.3 (untrusted v4) <==== veth pair ====> FS-standin / rustisk-standin +# fd00::2 fd00::3 (untrusted v6) nft table `cutover` (filter only) +# +# Proves the LISTENER SWAP: +# step 1 prime one fixed UDP five-tuple; keep NUMBERED datagrams flowing +# step 2 switch FS-standin -> rustisk-standin: single clean delivery boundary +# step 3 rollback rustisk-standin -> FS-standin under the SAME flow +# step 4 untrusted-source DROP (v4 AND v6) holds throughout both transitions +# step 5 measure the handover window in both directions +# then runs the RED control (stateful redirect lever) which FAILS the proof +# where the listener swap passes, and a detector self-test. +set -uo pipefail +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +source "${HERE}/cutover_lib.sh" + +# ---- synthetic parameters (NOT the real trunk ports 45070/20000-20100) ---- +export PORT=55070 # synthetic analog of hostIP:45070 +export NFT_TABLE=cutover +export UNTRUSTED_V4=10.9.0.3 +export UNTRUSTED_V6=fd00::3 +TRON_V4=10.9.0.1 ; TRON_V6=fd00::1 +TRUST_SRC=10.9.0.2 ; TRUST_SPORT=41002 +EVIL4_SRC=10.9.0.3 ; EVIL4_SPORT=41003 +EVIL6_SRC=fd00::3 ; EVIL6_SPORT=41006 +RATE_MS=2 + +RUN=/m8a/run ; rm -rf "${RUN}" ; mkdir -p "${RUN}" +FS_OUT="${RUN}/fs.csv" ; : > "${FS_OUT}" +RU_OUT="${RUN}/rustisk.csv" ; : > "${RU_OUT}" +FS_ST="${RUN}/fs.status" ; : > "${FS_ST}" +RU_ST="${RUN}/rustisk.status" ; : > "${RU_ST}" + +pids=() +cleanup() { + for p in "${pids[@]:-}"; do kill "${p}" 2>/dev/null || true; done + cutover_table_down + ip netns del chime 2>/dev/null || true +} +trap cleanup EXIT + +nsx() { ip netns exec chime "$@"; } # run a command on the "chime" side + +wait_status() { # + local f="$1" want="$2" to="$3" waited=0 + while :; do + local last; last="$(awk 'NF{l=$1} END{print l}' "$f" 2>/dev/null || true)" + [ "${last}" = "${want}" ] && return 0 + sleep 0.001; waited=$((waited+1)) + [ "${waited}" -ge "${to}" ] && { echo "TIMEOUT waiting ${want} in ${f}" >&2; return 1; } + done +} +export -f wait_status + +# ---------------- network setup inside the private netns ---------------- +setup_net() { + ip link set lo up + ip netns add chime + ip link add veth0 type veth peer name veth1 + ip link set veth1 netns chime + ip addr add ${TRON_V4}/24 dev veth0 + ip -6 addr add ${TRON_V6}/64 dev veth0 nodad + ip link set veth0 up + nsx ip addr add 10.9.0.2/24 dev veth1 + nsx ip addr add 10.9.0.3/24 dev veth1 + nsx ip -6 addr add fd00::2/64 dev veth1 nodad + nsx ip -6 addr add fd00::3/64 dev veth1 nodad + nsx ip link set veth1 up + nsx ip link set lo up +} + +echo "=================== M8a LISTENER-SWAP PROOF ===================" +echo "synthetic dport=${PORT} (NOT 45070) rate=${RATE_MS}ms ingress via veth (prerouting+input)" +setup_net + +# ---------- ISOLATION assertions (before any traffic) ---------- +echo "--- ISOLATION (must all hold before traffic) ---" +IFACES=$(ip -o link show | awk -F': ' '{print $2}' | cut -d'@' -f1 | sort | tr '\n' ',' ) +echo "main-netns links: ${IFACES}" +if ip route get 192.168.0.109 >/dev/null 2>&1; then echo " WARN: host LAN routable"; else echo " host LAN 192.168.0.109: NO ROUTE (isolated) OK"; fi +echo " nft ruleset before setup:"; nft list ruleset | sed 's/^/ /' | head + +# ---------- the cutover table + PERSISTENT source drops (v4+v6) ---------- +cutover_table_up +echo "--- cutover table (filter only; no nat/ct/redirect) ---" +nft list table inet "${NFT_TABLE}" | sed 's/^/ /' + +# ---------- listeners (bind on command; exactly one holds the port) ---------- +python3 "${HERE}/listener.py" --port "${PORT}" --label FS --out "${FS_OUT}" --status "${FS_ST}" --pidfile "${RUN}/fs.pid" & +pids+=($!) +python3 "${HERE}/listener.py" --port "${PORT}" --label RUSTISK --out "${RU_OUT}" --status "${RU_ST}" --pidfile "${RUN}/ru.pid" & +pids+=($!) +sleep 0.5 +FS_PID="$(cat "${RUN}/fs.pid")" ; RU_PID="$(cat "${RUN}/ru.pid")" + +# ---------- step 1: prime the fixed five-tuple; keep numbered flow hot ---------- +kill -USR1 "${FS_PID}" ; wait_status "${FS_ST}" BOUND 3000 +nsx python3 "${HERE}/sender.py" --src-ip "${TRUST_SRC}" --src-port "${TRUST_SPORT}" --dst-ip "${TRON_V4}" --dst-port "${PORT}" --tag TRUST --rate-ms "${RATE_MS}" --family 4 & +pids+=($!) +nsx python3 "${HERE}/sender.py" --src-ip "${EVIL4_SRC}" --src-port "${EVIL4_SPORT}" --dst-ip "${TRON_V4}" --dst-port "${PORT}" --tag EVILV4 --rate-ms "${RATE_MS}" --family 4 & +pids+=($!) +nsx python3 "${HERE}/sender.py" --src-ip "${EVIL6_SRC}" --src-port "${EVIL6_SPORT}" --dst-ip "${TRON_V6}" --dst-port "${PORT}" --tag EVILV6 --rate-ms "${RATE_MS}" --family 6 & +pids+=($!) +echo "step 1: primed TRUST(v4) five-tuple ${TRUST_SRC}:${TRUST_SPORT} -> ${TRON_V4}:${PORT}" +echo " untrusted EVILV4(${EVIL4_SRC}) + EVILV6(${EVIL6_SRC}) flooding continuously" +sleep 1.5 + +# ---------- step 2: switch FS -> rustisk ---------- +echo "step 2: APPLY switch FS -> rustisk" +STOP_OLD_CMD="kill -USR2 ${FS_PID}" \ +WAIT_RELEASED_CMD="wait_status ${FS_ST} UNBOUND 3000" \ +START_NEW_CMD="kill -USR1 ${RU_PID}" \ +WAIT_BOUND_CMD="wait_status ${RU_ST} BOUND 3000" \ +WINDOW_OUT="${RUN}/apply_window.ns" \ + bash "${HERE}/apply-fs-to-rustisk.sh" +sleep 1.5 + +# ---------- step 3: rollback rustisk -> FS under the SAME flow ---------- +echo "step 3: ROLLBACK rustisk -> FS (same continuously-flowing tuple)" +STOP_OLD_CMD="kill -USR2 ${RU_PID}" \ +WAIT_RELEASED_CMD="wait_status ${RU_ST} UNBOUND 3000" \ +START_NEW_CMD="kill -USR1 ${FS_PID}" \ +WAIT_BOUND_CMD="wait_status ${FS_ST} BOUND 3000" \ +WINDOW_OUT="${RUN}/rollback_window.ns" \ + bash "${HERE}/rollback-rustisk-to-fs.sh" +sleep 1.5 + +# ---------- stop the flow, flush captures ---------- +for p in "${pids[@]}"; do kill "${p}" 2>/dev/null || true; done +sleep 0.3 + +# ---------- steps 2/3/4 assertions (receiver-side) ---------- +echo "--------------- RECEIVER-SIDE ASSERTIONS (listener swap) ---------------" +python3 "${HERE}/assert_boundary.py" --fs "${FS_OUT}" --rustisk "${RU_OUT}" \ + --trust-tag TRUST --untrusted-tags EVILV4 EVILV6 --expect-runs FS,RUSTISK,FS +SWAP_RC=$? + +# ---------- step 5: report the measured windows ---------- +aw=$(cat "${RUN}/apply_window.ns" 2>/dev/null || echo 0) +rw=$(cat "${RUN}/rollback_window.ns" 2>/dev/null || echo 0) +echo "--------------- MEASURED HANDOVER WINDOWS (mechanism floor) ---------------" +awk -v a="$aw" 'BEGIN{printf "FS->rustisk apply window: %.1f ms\n", a/1e6}' +awk -v r="$rw" 'BEGIN{printf "rustisk->FS rollback window: %.1f ms\n", r/1e6}' +echo "(synthetic floor = nft toggle + socket close/bind + IPC; the LIVE M9 window" +echo " additionally includes FS 'sofia profile stop/start' + DNS re-resolve.)" + +# ---------- RED control + detector self-test ---------- +echo +bash "${HERE}/red-stateful-variant.sh"; RED_RC=$? +echo +bash "${HERE}/detector-selftest.sh"; DET_RC=$? + +echo +echo "=================== SUMMARY ===================" +echo "listener-swap assert rc=${SWAP_RC} (0 = clean boundaries + source-drop hold)" +echo "RED control (redirect) rc=${RED_RC} (0 = redirect FAILED the proof as expected -> teeth)" +echo "detector self-test rc=${DET_RC} (0 = assert_boundary rejects every crafted bad capture)" +if [ "${SWAP_RC}" -eq 0 ] && [ "${RED_RC}" -eq 0 ] && [ "${DET_RC}" -eq 0 ]; then + echo "M8a RESULT: PASS" + exit 0 +else + echo "M8a RESULT: FAIL" + exit 1 +fi diff --git a/tests/m8a-cutover/listener.py b/tests/m8a-cutover/listener.py new file mode 100755 index 0000000..79410cb --- /dev/null +++ b/tests/m8a-cutover/listener.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""M8a stand-in listener — a receiver-side witness for the cutover proof. + +Models ONE PBX holding the public UDP bind (hostIP:45070 in prod; a synthetic +high port here). Binds ON COMMAND so the driver can hand the port from one +stand-in to the other: + + SIGUSR1 -> bind the port (this stand-in takes the port) + SIGUSR2 -> close the socket (this stand-in releases the port) + SIGTERM -> exit + +There is deliberately NO SO_REUSEPORT: at most one stand-in can hold the port at +a time, exactly like the single public bind. bind() uses SO_REUSEADDR only so a +just-released UDP port can be reclaimed immediately (UDP has no TIME_WAIT). + +Dual-stack: binds [::]:PORT with IPV6_V6ONLY=0, so a single socket witnesses +BOTH v4-mapped and v6 datagrams — that is how one listener proves the v4 AND v6 +source-drop simultaneously. + +Every received datagram is appended to --out as CSV: + recv_ts_ns,label,src,tag,seq +(one flushed line per datagram — this file is the receiver-side ground truth). + +Bind/unbind transitions are appended to --status as: + +so the driver can measure the handover window and avoid EADDRINUSE races. +""" +import argparse +import errno +import os +import select +import signal +import socket +import sys +import time + +want_bound = False +stop = False + + +def _on_bind(_sig, _frm): + global want_bound + want_bound = True + + +def _on_unbind(_sig, _frm): + global want_bound + want_bound = False + + +def _on_term(_sig, _frm): + global stop + stop = True + + +def make_socket(port): + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # Dual-stack: receive both v6 and v4-mapped on the same port. + s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + s.bind(("::", port)) + s.setblocking(False) + return s + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, required=True) + ap.add_argument("--label", required=True, help="FS or RUSTISK") + ap.add_argument("--out", required=True) + ap.add_argument("--status", required=True) + ap.add_argument("--pidfile", default=None) + args = ap.parse_args() + + signal.signal(signal.SIGUSR1, _on_bind) + signal.signal(signal.SIGUSR2, _on_unbind) + signal.signal(signal.SIGTERM, _on_term) + + if args.pidfile: + with open(args.pidfile, "w") as f: + f.write(str(os.getpid()) + "\n") + + out = open(args.out, "a", buffering=1) + status = open(args.status, "a", buffering=1) + + sock = None + while not stop: + try: + if want_bound and sock is None: + try: + sock = make_socket(args.port) + status.write(f"BOUND {time.time_ns()}\n") + status.flush() + except OSError as e: + if e.errno in (errno.EADDRINUSE,): + # Old holder has not released yet; retry shortly. + time.sleep(0.0005) + continue + status.write(f"BINDFAIL {time.time_ns()} {e.errno}\n") + status.flush() + time.sleep(0.001) + continue + if not want_bound and sock is not None: + sock.close() + sock = None + status.write(f"UNBOUND {time.time_ns()}\n") + status.flush() + + if sock is None: + time.sleep(0.0005) + continue + + try: + r, _, _ = select.select([sock], [], [], 0.02) + except InterruptedError: + continue + if not r: + continue + while True: + try: + data, addr = sock.recvfrom(4096) + except BlockingIOError: + break + except InterruptedError: + break + except OSError: + break + recv_ts = time.time_ns() + src = f"{addr[0]}:{addr[1]}" + parts = data.decode("ascii", "replace").split() + tag = parts[0] if len(parts) > 0 else "?" + seq = parts[1] if len(parts) > 1 else "?" + out.write(f"{recv_ts},{args.label},{src},{tag},{seq}\n") + except InterruptedError: + continue + + if sock is not None: + sock.close() + status.write(f"UNBOUND {time.time_ns()}\n") + status.flush() + out.close() + status.close() + + +if __name__ == "__main__": + main() diff --git a/tests/m8a-cutover/red-stateful-variant.sh b/tests/m8a-cutover/red-stateful-variant.sh new file mode 100755 index 0000000..eda69a8 --- /dev/null +++ b/tests/m8a-cutover/red-stateful-variant.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# red-stateful-variant.sh — the RED control that proves the proof has TEETH. +# +# A deliberately STATEFUL lever: an nftables `dnat`/redirect (NAT + conntrack) +# that rewrites the guarded dport to an alternate port the successor binds — +# instead of the listener swap's bind handover on the SAME port. +# +# TWO findings, both run against the same veth ingress path (prerouting+input) +# already set up by in-container-proof.sh: +# +# (A) SECURITY DEFECT — DETERMINISTIC, this is the teeth: +# Because the lever REWRITES the dport (55190 -> 55192) at prerouting, the +# fail-closed source-drop written for the original dport 55190 no longer +# matches (post-DNAT dport is 55192), so the UNTRUSTED source is DELIVERED +# to the successor. assert_boundary FLAGS this (untrusted delivered) => the +# redirect lever FAILS the proof exactly where the listener swap PASSES. +# This is PLAN-v3's cited hazard: "any port-rewriting mechanism moves +# packets to a port the current filter does not match, and would need a +# genuine fail-closed DROP for untrusted sources on the new port, v4+v6." +# +# (B) ROLLBACK-PERSISTENCE PROBE — HONEST NEGATIVE RESULT: +# PLAN-v3 C1 argues the redirect cannot switch a primed tuple BACK because +# conntrack persists. Measured on tron's kernel: it reverts CLEANLY on rule +# removal (see numbers below). So on THIS kernel the redirect dies on (A), +# not on the rollback boundary. Reported transparently — the listener swap +# is still chosen for structural reasons (introduces no NAT/conntrack into a +# NAT-free path; behavior independent of kernel/birth-conditions). +# +# Returns 0 iff the redirect lever demonstrably FAILED the proof (teeth), i.e. +# assert_boundary rejects the redirect capture. +set -uo pipefail +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +RUN="${RUN:-/m8a/run}" ; mkdir -p "${RUN}" + +PORT_PUB=55190 ; PORT_ALT=55192 +TRON_V4=10.9.0.1 +TABLE=redirtest # ip family: holds the nat/redirect lever +FTABLE=redirtest_f # inet family: holds the fail-closed source-drops (v4+v6) +nsx() { ip netns exec chime "$@"; } + +RFS_OUT="${RUN}/red_fs.csv" ; : > "${RFS_OUT}" +RRU_OUT="${RUN}/red_rustisk.csv" ; : > "${RRU_OUT}" +RFS_ST="${RUN}/red_fs.status" ; : > "${RFS_ST}" +RRU_ST="${RUN}/red_rustisk.status" ; : > "${RRU_ST}" + +rpids=() +cleanup_red() { + for p in "${rpids[@]:-}"; do kill "${p}" 2>/dev/null || true; done + nft delete table ip "${TABLE}" 2>/dev/null || true + nft delete table inet "${FTABLE}" 2>/dev/null || true + conntrack -D -p udp --dport "${PORT_PUB}" 2>/dev/null || true +} +trap cleanup_red RETURN + +echo "=================== RED CONTROL (stateful redirect lever) ===================" +# fail-closed source-drops written for the ORIGINAL dport (as the operator would), +# in an inet table so BOTH v4 and v6 drops are expressible. +nft add table inet "${FTABLE}" +nft "add chain inet ${FTABLE} input { type filter hook input priority -10 ; policy accept ; }" +nft add rule inet "${FTABLE}" input meta l4proto udp udp dport "${PORT_PUB}" ip saddr 10.9.0.3 drop +nft add rule inet "${FTABLE}" input meta l4proto udp udp dport "${PORT_PUB}" ip6 saddr fd00::3 drop +# the stateful lever: prerouting DNAT rewriting the dport to the successor's port +nft add table ip "${TABLE}" +nft "add chain ip ${TABLE} nat { type nat hook prerouting priority -100 ; policy accept ; }" +nft add rule ip "${TABLE}" nat meta l4proto udp udp dport "${PORT_PUB}" dnat to ${TRON_V4}:${PORT_ALT} + +python3 "${HERE}/listener.py" --port "${PORT_PUB}" --label FS --out "${RFS_OUT}" --status "${RFS_ST}" --pidfile "${RUN}/red_fs.pid" & +rpids+=($!) +python3 "${HERE}/listener.py" --port "${PORT_ALT}" --label RUSTISK --out "${RRU_OUT}" --status "${RRU_ST}" --pidfile "${RUN}/red_ru.pid" & +rpids+=($!) +sleep 0.5 +kill -USR1 "$(cat "${RUN}/red_fs.pid")"; kill -USR1 "$(cat "${RUN}/red_ru.pid")" +sleep 0.3 + +echo "--- (A) source-drop bypass: untrusted 10.9.0.3 -> ${PORT_PUB}, drop is on dport ${PORT_PUB} ---" +nsx python3 "${HERE}/sender.py" --src-ip 10.9.0.2 --src-port 41102 --dst-ip ${TRON_V4} --dst-port ${PORT_PUB} --tag TRUST --rate-ms 2 --family 4 & +rpids+=($!) +nsx python3 "${HERE}/sender.py" --src-ip 10.9.0.3 --src-port 41103 --dst-ip ${TRON_V4} --dst-port ${PORT_PUB} --tag EVILV4 --rate-ms 2 --family 4 & +rpids+=($!) +sleep 1.2 +for p in "${rpids[@]:2}"; do kill "${p}" 2>/dev/null || true; done +sleep 0.2 +echo " (redirect DNATs ${PORT_PUB}->${PORT_ALT} at prerouting, past the dport-${PORT_PUB} drop)" + +echo "--- (B) rollback-persistence probe (honest): switch then rollback a primed tuple ---" +# fresh listeners on the two ports; count captured lines per phase. +kill -USR2 "$(cat "${RUN}/red_fs.pid")" 2>/dev/null; kill -USR2 "$(cat "${RUN}/red_ru.pid")" 2>/dev/null +sleep 0.2 +nft flush chain ip "${TABLE}" nat; conntrack -F 2>/dev/null || true +PBF="${RUN}/pb_fs.csv"; PBR="${RUN}/pb_ru.csv"; : > "${PBF}"; : > "${PBR}" +python3 "${HERE}/listener.py" --port "${PORT_PUB}" --label FS --out "${PBF}" --status "${RUN}/pbf.st" --pidfile "${RUN}/pbf.pid" & +rpids+=($!) +python3 "${HERE}/listener.py" --port "${PORT_ALT}" --label RU --out "${PBR}" --status "${RUN}/pbr.st" --pidfile "${RUN}/pbr.pid" & +rpids+=($!) +sleep 0.5 +kill -USR1 "$(cat "${RUN}/pbf.pid")"; kill -USR1 "$(cat "${RUN}/pbr.pid")"; sleep 0.3 +cnt(){ wc -l < "$1"; } +nsx python3 "${HERE}/burst.py" --src-ip 10.9.0.2 --src-port 41202 --dst-ip ${TRON_V4} --dst-port ${PORT_PUB} --count 150; sleep 0.1 +p1="FS=$(cnt "${PBF}") RU=$(cnt "${PBR}")" +nft add rule ip "${TABLE}" nat meta l4proto udp udp dport "${PORT_PUB}" dnat to ${TRON_V4}:${PORT_ALT} +nsx python3 "${HERE}/burst.py" --src-ip 10.9.0.2 --src-port 41202 --dst-ip ${TRON_V4} --dst-port ${PORT_PUB} --count 200; sleep 0.1 +p2="FS=$(cnt "${PBF}") RU=$(cnt "${PBR}")" +h=$(nft -a list chain ip "${TABLE}" nat | grep "dnat to" | grep -oP "handle \K[0-9]+" | head -1) +[ -n "$h" ] && nft delete rule ip "${TABLE}" nat handle "$h" +nsx python3 "${HERE}/burst.py" --src-ip 10.9.0.2 --src-port 41202 --dst-ip ${TRON_V4} --dst-port ${PORT_PUB} --count 120; sleep 0.1 +p3="FS=$(cnt "${PBF}") RU=$(cnt "${PBR}")" +echo " cumulative captured lines (FS=${PORT_PUB}, RU=${PORT_ALT}):" +echo " prime (no rule): [$p1]" +echo " after switch (dnat): [$p2] (RU grew => switch worked)" +echo " after rollback (del):[$p3] (FS grew => reverted cleanly)" +echo " => rollback reverts to FS on tron's kernel; conntrack-persistence objection NOT reproduced." + +echo "--- RED machine verdict (assert_boundary on the redirect source-drop capture) ---" +python3 "${HERE}/assert_boundary.py" --fs "${RFS_OUT}" --rustisk "${RRU_OUT}" \ + --trust-tag TRUST --untrusted-tags EVILV4 --expect-runs FS,RUSTISK,FS +A_RC=$? +if [ "${A_RC}" -ne 0 ]; then + echo "RED TEETH CONFIRMED: redirect lever FAILED the proof (untrusted delivered / no clean swap) — listener swap PASSES the same checks." + exit 0 +else + echo "RED WARNING: redirect lever PASSED — the proof has NO teeth here!" + exit 1 +fi diff --git a/tests/m8a-cutover/rollback-rustisk-to-fs.sh b/tests/m8a-cutover/rollback-rustisk-to-fs.sh new file mode 100755 index 0000000..22f1bac --- /dev/null +++ b/tests/m8a-cutover/rollback-rustisk-to-fs.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# rollback-rustisk-to-fs.sh — THE ROLLBACK ARTIFACT M9 invokes (rustisk -> FS). +# +# The exact mirror of the apply. Because the mechanism holds NO conntrack/NAT +# state, the rollback is symmetric and works under the SAME primed, continuously +# flowing tuple — this is the step a stateful/redirect lever cannot pass +# (deleting nft rules does not delete conntrack entries; continuing traffic +# refreshes them). See red-stateful-variant.sh for the RED contrast. +# +# Sequence: +# 1. handover_drop_on +# 2. STOP_OLD_CMD — release the port from rustisk. +# M9: stop rustisk's transport. +# 3. WAIT_RELEASED_CMD +# 4. START_NEW_CMD — rebind the port on FS. This is the trunk watchdog's +# already-exercised recovery: +# M9: fs_cli -x 'sofia profile start' +# 5. WAIT_BOUND_CMD +# 6. handover_drop_off +# +# Env: same shape as apply-fs-to-rustisk.sh (STOP_OLD_CMD now stops rustisk, +# START_NEW_CMD now starts FS). +set -euo pipefail +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=cutover_lib.sh +source "${HERE}/cutover_lib.sh" + +: "${STOP_OLD_CMD:?}" ; : "${START_NEW_CMD:?}" +: "${WAIT_RELEASED_CMD:=true}" ; : "${WAIT_BOUND_CMD:=true}" + +t0="$(date +%s%N)" +handover_drop_on +eval "${STOP_OLD_CMD}" +eval "${WAIT_RELEASED_CMD}" +eval "${START_NEW_CMD}" +eval "${WAIT_BOUND_CMD}" +handover_drop_off +t1="$(date +%s%N)" + +win_ms=$(( (t1 - t0) / 1000000 )) +echo "ROLLBACK rustisk->FS handover window: ${win_ms} ms (${t0} -> ${t1} ns)" +if [ -n "${WINDOW_OUT:-}" ]; then echo "$((t1 - t0))" > "${WINDOW_OUT}"; fi diff --git a/tests/m8a-cutover/run-proof.sh b/tests/m8a-cutover/run-proof.sh new file mode 100755 index 0000000..b49ae45 --- /dev/null +++ b/tests/m8a-cutover/run-proof.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# run-proof.sh — top-level driver for the M8a cutover-mechanism proof. +# +# Builds the proof image and runs the ENTIRE proof inside ONE isolated container +# network namespace (--privileged --network none): the container has only `lo` +# plus an internal veth pair to a child netns, NO route to the host, and its OWN +# nftables ruleset. Nothing it does can touch the host nft/ports, the live +# trunk, port 45070, the router, or the cluster. Synthetic ports only. +# +# Everything is reaped on exit (container + image). Never touches the host netns. +# +# ./run-proof.sh # build + run, transcript to stdout +# TRANSCRIPT=out.txt ./run-proof.sh # also tee the transcript to a file +set -euo pipefail +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +IMG=m8a-cutover-proof:local +CTR="m8a-cutover-$$" +TRANSCRIPT="${TRANSCRIPT:-}" + +cleanup() { + docker rm -f "${CTR}" >/dev/null 2>&1 || true + docker image rm "${IMG}" >/dev/null 2>&1 || true + # Report any leak so a human can reap by hand. + if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "${CTR}"; then + echo "CLEANUP WARNING: container ${CTR} leaked — docker rm -f ${CTR}" >&2 + fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +command -v docker >/dev/null || { echo "docker required" >&2; exit 2; } + +echo "== build proof image ==" +docker build -t "${IMG}" "${HERE}" >/dev/null + +echo "== launch ISOLATED container (--privileged --network none) ==" +docker run -d --rm --name "${CTR}" --privileged --network none "${IMG}" >/dev/null + +# Host-side isolation assertion: docker itself reports this container is on the +# 'none' network — no bridge, no veth to the host, no published ports. +NETMODE="$(docker inspect -f '{{.HostConfig.NetworkMode}}' "${CTR}")" +PORTS="$(docker inspect -f '{{json .NetworkSettings.Ports}}' "${CTR}")" +echo "host-side check: NetworkMode=${NETMODE} PublishedPorts=${PORTS}" +[ "${NETMODE}" = "none" ] || { echo "REFUSING: container is not on network 'none'" >&2; exit 3; } + +run() { docker exec "${CTR}" bash /m8a/in-container-proof.sh; } +if [ -n "${TRANSCRIPT}" ]; then run | tee "${TRANSCRIPT}"; rc="${PIPESTATUS[0]}"; else run; rc=$?; fi + +echo "== proof exit: ${rc} ==" +exit "${rc}" diff --git a/tests/m8a-cutover/sender.py b/tests/m8a-cutover/sender.py new file mode 100755 index 0000000..d3d4744 --- /dev/null +++ b/tests/m8a-cutover/sender.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""M8a numbered-datagram sender — keeps one fixed UDP five-tuple HOT. + +Emits datagrams " " with SEQ = 1,2,3,... and NO gaps, +from a FIXED source (--src-ip/--src-port) to a FIXED destination +(--dst-ip/--dst-port), at a fixed inter-datagram interval (--rate-ms), until +SIGTERM. Binding the source address+port pins the five-tuple so the switch and +the rollback are exercised against the SAME primed tuple (proof requirement C2), +never a fresh one. + +--family 4 uses AF_INET; 6 uses AF_INET6. A v6 sender is used for the untrusted +v6 source-drop leg. +""" +import argparse +import signal +import socket +import sys +import time + +stop = False + + +def _on_term(_sig, _frm): + global stop + stop = True + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--src-ip", required=True) + ap.add_argument("--src-port", type=int, required=True) + ap.add_argument("--dst-ip", required=True) + ap.add_argument("--dst-port", type=int, required=True) + ap.add_argument("--tag", required=True) + ap.add_argument("--rate-ms", type=float, default=2.0) + ap.add_argument("--family", type=int, choices=(4, 6), default=4) + ap.add_argument("--seqfile", default=None, + help="write the highest seq sent so far (for the driver)") + args = ap.parse_args() + + signal.signal(signal.SIGTERM, _on_term) + signal.signal(signal.SIGINT, _on_term) + + fam = socket.AF_INET if args.family == 4 else socket.AF_INET6 + s = socket.socket(fam, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((args.src_ip, args.src_port)) + dst = (args.dst_ip, args.dst_port) + + seqfile = open(args.seqfile, "w", buffering=1) if args.seqfile else None + interval = args.rate_ms / 1000.0 + seq = 0 + next_t = time.monotonic() + while not stop: + seq += 1 + payload = f"{args.tag} {seq} {time.time_ns()}".encode("ascii") + try: + s.sendto(payload, dst) + except OSError: + pass + if seqfile is not None and (seq % 20 == 0): + seqfile.seek(0) + seqfile.write(str(seq)) + seqfile.truncate() + next_t += interval + sleep = next_t - time.monotonic() + if sleep > 0: + time.sleep(sleep) + else: + next_t = time.monotonic() + s.close() + + +if __name__ == "__main__": + main()