release: 3.6.1

.github/workflows/ci.yml

@@ -31,7 +31,22 @@ jobs:
       # It is deliberately NOT the measured 78.96%: a floor set exactly at
       # today's number turns any unrelated refactor that moves one region into a
       # failed build.
-      coverage-threshold: 75
+      # 79, not the org standard's 90, and not because podup is short of it.
+      #
+      # Measured 2026-08-03: the full suite covers **91.52%** of lines. This job
+      # cannot run it — the integration tests need a live Podman and skip
+      # themselves on a runner without one — so what it measures is `--lib
+      # --bins` alone, which is **79.39%**. The two numbers describe different
+      # things and only one of them is checkable here.
+      #
+      # 75 left four points of silent slack on a figure that is already a
+      # subset: coverage could fall from 79 to 75 with nothing saying so. This
+      # locks in what holds today rather than demanding new work, which is what
+      # the testing standard asks for when adopting a floor.
+      #
+      # The 90 gate belongs where the tests can actually run — the `podman-vm`
+      # lane — and that is tracked separately rather than papered over here.
+      coverage-threshold: 79
       msrv: "1.85"
       package-check: true
       semver-check: true

.github/workflows/podman-lane.yml

@@ -23,7 +23,16 @@ jobs:
     runs-on: ubuntu-24.04
     # Raised for the one-thread experiment: serialising the suite costs wall
     # clock, and a run that dies on the timeout answers nothing (#1039).
-    timeout-minutes: 75
+    #
+    # Raised again for the coverage path. Capping the coverage run to the same
+    # thread count as the gating run (#1338) means the Podman 6 leg now runs two
+    # serialised suites, and it came in at 66m27s of the 75 on run 30879785232 —
+    # 89% of the budget. That is not a failure yet, which is the only useful
+    # moment to change it: a slower runner or a heavier rawhide image tips it
+    # over, and the run that dies tells you nothing about the coverage it was
+    # measuring. Costs nothing on the pull-request path, where coverage is off
+    # and the leg finishes in about half this.
+    timeout-minutes: 95
     strategy:
       fail-fast: false
       matrix:
@@ -95,6 +104,15 @@ jobs:
                 # flawless 155-pass run and clear the floor comfortably.
                 export PODUP_REQUIRE_PODMAN=1
                 cd "$HOME/podup" || exit 90
+                # Coverage runs on the scheduled and manual paths only — never on a
+                # pull request. The number is worth having where the integration
+                # tests can actually run (#1326: the CI coverage job sees 79% because
+                # every integration test skips itself without Podman, while the full
+                # suite covers 91.5%), but instrumenting doubles nothing here for
+                # free: it is a second full build inside the VM. Gating on it is
+                # deliberately NOT done yet — a threshold set from a number never
+                # observed in this environment is how a phantom check gets born.
+                COVERAGE=__COVERAGE__
                 # Capture the FULL output: the per-failure detail (where the flake
                 # marker lives) precedes the summary, so a `tail` on the console
                 # would truncate it and hide why a test failed.
@@ -129,11 +147,18 @@ jobs:
                 # this hypothesis dies too and the search moves elsewhere.
                 #
                 # `--test-threads` caps how many streaming tests hit the libpod
-                # socket at once. The nested-virt (VM-in-runner) transport is what
-                # drops connections mid-stream — measured: on a single-level VM
-                # (Podman 5.8.1 and 6.0.1) the same suite never drops a byte, so the
-                # flake is the double-virt socket under concurrent load, not a podup
-                # or libpod bug. Fewer concurrent streams means less peak pressure,
+                # socket at once.
+                #
+                # This used to say the drops were specific to the nested-virt
+                # (VM-in-runner) transport, and that a single-level VM "never
+                # drops a byte". **That is measured false** (2026-08-03, #1322):
+                # on a plain KVM guest with Podman 6.0.1, the same suite fails
+                # 17 of 27 at eight test threads and 3 at one — a clean
+                # dose-response curve in thread count, reproducible across
+                # repetitions. Nesting makes it worse; it does not cause it.
+                #
+                # What holds is the lever: fewer concurrent streams means less
+                # peak pressure,
                 # which is the lever to shrink Podman 6's variable flaky tail toward
                 # a small, stable set that an identity list can gate (#1039). Kept at
                 # 2 rather than 1 so the suite still fits the VM's time budget.
@@ -156,6 +181,67 @@ jobs:
                 FLAKY=$(grep -cE "$DROPPED" /tmp/cargo.log)
                 echo "PODUP_TESTS_RC=$RC"
                 echo "PODUP_SUMMARY pass=${PASS:-0} fail=${FAIL:-0} flaky=${FLAKY:-0}"
+                # Coverage, measured where the integration tests can actually run.
+                # Reported, never gated — see the note where COVERAGE is set. A
+                # failure here must not redden the leg: this is an observation, and
+                # the suite's own result above is what gates.
+                if [ "$COVERAGE" = "1" ]; then
+                  # Nothing here redirects to /dev/console. This script runs as
+                  # `tester`, which may not open that device, and a denied
+                  # redirection aborts the command it was attached to and is
+                  # reported as that command's own failure. The first run said
+                  # `install-failed` for exactly that reason: cargo never ran.
+                  # stdout and stderr already point at the console, inherited
+                  # from the root caller that launched this script.
+                  # cargo-llvm-cov looks for the `llvm-tools-preview` rustup
+                  # component, and there is no rustup here — Rust comes from dnf.
+                  # It accepts the two tools directly instead, and Fedora's `llvm`
+                  # package (already installed above) carries both. Taking them
+                  # from the distribution is also what keeps the versions in step:
+                  # rustc and llvm are built for the same Fedora release, so the
+                  # profile format matches, which a separately-pinned LLVM would
+                  # not guarantee.
+                  #
+                  # Exactly ONE PODUP_COVERAGE line is emitted on every path. Two
+                  # would be worse than none: whoever greps the log gets an answer
+                  # either way and no reason to suspect there was another.
+                  export LLVM_COV=/usr/bin/llvm-cov
+                  export LLVM_PROFDATA=/usr/bin/llvm-profdata
+                  # Print both versions unconditionally. A profile-format mismatch
+                  # is the next thing likely to go wrong here, and it is only
+                  # diagnosable against the pair of numbers that produced it.
+                  echo "llvm tools: $("$LLVM_COV" --version 2>&1 | grep -i version | head -1)"
+                  echo "rustc llvm: $(rustc --version --verbose | grep -i '^LLVM')"
+                  if [ ! -x "$LLVM_COV" ] || [ ! -x "$LLVM_PROFDATA" ]; then
+                    # Say so here rather than letting cargo-llvm-cov fail with its
+                    # own wording several minutes and one full build later.
+                    echo "PODUP_COVERAGE=llvm-tools-missing"
+                  elif cargo install cargo-llvm-cov --locked; then
+                    # Same thread cap as the gating run above. cargo-llvm-cov
+                    # drives its own `cargo test` and does NOT inherit it, which
+                    # is measurable rather than theoretical: on the Podman 6 leg
+                    # the gating run took 1743s and passed 178/178, and the
+                    # coverage run of the same code in the same boot took 441s —
+                    # four times faster because it was parallel — and failed 5
+                    # with hyper IncompleteMessage. Uncapped, this step measures
+                    # a different suite than the one that gates.
+                    cargo llvm-cov --all-features --summary-only \
+                      -- --test-threads=__THREADS__ >/tmp/cov.log 2>&1
+                    PCT=$(awk '$1=="TOTAL" { for (i=1;i<=NF;i++) if ($i ~ /%$/) { print $i; exit } }' /tmp/cov.log)
+                    if [ -n "$PCT" ]; then
+                      echo "PODUP_COVERAGE=$PCT"
+                    else
+                      # Ran, produced nothing parseable. Print what it actually
+                      # said: a second 15-minute boot is an expensive way to
+                      # find out, and each distinct marker below names one
+                      # cause instead of collapsing every failure into one word.
+                      echo "PODUP_COVERAGE=no-total-line"
+                      tail -30 /tmp/cov.log
+                    fi
+                  else
+                    echo "PODUP_COVERAGE=install-failed"
+                  fi
+                fi
                 # Emit the IDENTITIES of the failing tests, not just the count.
                 # The floor gate can only tighten to a per-test known-flaky
                 # allowlist once the flaky set is small and stable (#1039), and
@@ -169,7 +255,7 @@ jobs:
                 echo "PODUP_FAILED_TESTS=${FAILED_TESTS}"
           runcmd:
             - bash -c 'F=$(findmnt -no FSTYPE /); case $F in btrfs) btrfs filesystem resize max / ;; xfs) xfs_growfs / ;; ext4) resize2fs $(findmnt -no SOURCE /) ;; esac >/dev/console 2>&1; echo "DISK=$(df -h / | tail -1)" >/dev/console'
-            - bash -c 'dnf install -y podman rust cargo gcc git >/dev/console 2>&1'
+            - bash -c 'dnf install -y podman rust cargo gcc git llvm >/dev/console 2>&1'
             # Prove the driver took, so a silently-ignored config cannot put the
             # logs tests back to passing without reading anything.
             - bash -c 'echo "LOGDRIVER=$(podman info --format {{.Host.LogDriver}} 2>/dev/null)" >/dev/console'
@@ -186,6 +272,13 @@ jobs:
           [ "${{ matrix.podman }}" = "6" ] && THREADS=1
           echo "test-threads for Podman ${{ matrix.podman }}: $THREADS"
           sed -i "s/__THREADS__/$THREADS/" "$RUNNER_TEMP/user-data"
+          # Coverage on the scheduled and manual paths only. A pull request must
+          # not pay for a second instrumented build, and must not be able to fail
+          # for a reason that has nothing to do with its own change.
+          COVERAGE=0
+          case "${{ github.event_name }}" in schedule|workflow_dispatch) COVERAGE=1 ;; esac
+          echo "coverage pass for Podman ${{ matrix.podman }}: $COVERAGE"
+          sed -i "s/__COVERAGE__/$COVERAGE/" "$RUNNER_TEMP/user-data"
           cloud-localds "$RUNNER_TEMP/seed.iso" "$RUNNER_TEMP/user-data"
       - name: Boot VM (9p = repo only) + capture
         run: |
@@ -228,6 +321,16 @@ jobs:
           # log (the console here is tail-truncated, so don't recount from it).
           SUM=$(grep -oE 'PODUP_SUMMARY pass=[0-9]+ fail=[0-9]+ flaky=[0-9]+' "$L" | tail -1)
           [ -n "$SUM" ] || { echo "::error::no PODUP_SUMMARY from VM on Podman $VER — suite did not report"; exit 1; }
+          # Coverage, when this run asked for it. Reported into the job summary and
+          # never gated: the org standard's 90% floor belongs here rather than on
+          # the Podman-less job that can only see 79% (#1326), but a threshold set
+          # from a number never observed in THIS environment would be a phantom
+          # check. Observe first, then gate, in a separate change.
+          COV=$(grep -oE 'PODUP_COVERAGE=[^ ]+' "$L" | tail -1 | cut -d= -f2)
+          if [ -n "$COV" ]; then
+            echo "coverage on Podman $VER (full suite, integration tests included): $COV"
+            echo "- **Podman $VER coverage:** $COV" >> "$GITHUB_STEP_SUMMARY"
+          fi
           PASS=$(echo "$SUM" | grep -oE 'pass=[0-9]+' | cut -d= -f2)
           FAIL=$(echo "$SUM" | grep -oE 'fail=[0-9]+' | cut -d= -f2)
           FLAKY=$(echo "$SUM" | grep -oE 'flaky=[0-9]+' | cut -d= -f2)

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "podup"
-version = "3.6.0"
+version = "3.6.1"
 edition = "2021"
 rust-version = "1.85"
 description = "Translate and run docker-compose files on rootless Podman"

debian/changelog

@@ -1,3 +1,22 @@
+podup (3.6.1) unstable; urgency=medium
+
+  Fixed
+
+  * A lifecycle operation whose response the daemon drops is no longer reported
+    as a failure when it actually completed. `restart`, `start`, `kill`, `stop`
+    and `rm` now confirm the container reached the state the command was for,
+    and only then report success; a container that did not reach it, or a state
+    that cannot be re-read, still fails. Podman 6 severs these responses under
+    concurrency, which made `up`, `down` and `restart` fail on multi-service
+    projects for a reason that was never the command.
+  * `exec` retries once when the daemon drops the response to its session
+    create. That request is where the drops land most, and it cannot be resolved
+    by re-checking the container, because what is lost is the session id.
+    Retrying is safe: an exec that is created and never started allocates no
+    process and is discarded with the container.
+
+ -- Jaro-c <75870284+Jaro-c@users.noreply.github.com>  Tue, 04 Aug 2026 04:35:55 -0500
+
 podup (3.6.0) unstable; urgency=medium
 
   Incompatible

internal/engine/fake_podman.rs

@@ -42,6 +42,19 @@ pub(super) enum FakeReply {
 	/// closes. The other place a severed stream can land, and — measured — hyper
 	/// classifies the two differently, which is why both exist here.
 	ChunkedCutMidPayload(String),
+	/// The request is read and accepted, and then the connection closes with **no
+	/// response at all** — not even a status line.
+	///
+	/// This is the shape `PodmanError::is_incomplete_message` names: hyper's
+	/// `IncompleteMessage` is about the message *head*, so it is the one reply
+	/// here that produces it, and the severed-body variants above do not.
+	///
+	/// It is what libpod does on Podman 6 to the container-archive PUT (#1097,
+	/// applies the archive then hangs up) and to state-changing POSTs under
+	/// concurrency (#1339). Both are handled by re-checking the observable out of
+	/// band, and until this existed neither discriminator had a test that could
+	/// reach it.
+	ClosedWithoutResponse,
 }
 
 /// A test's routing rule: `(method, target) -> reply`, where `target` is the
@@ -181,6 +194,9 @@ async fn serve_one(
 				.await?;
 			stream.flush().await?;
 		}
+		FakeReply::ClosedWithoutResponse => {
+			// Write nothing. The shutdown below is the entire reply.
+		}
 	}
 	stream.shutdown().await?;
 	Ok(())

internal/engine/lifecycle/commands.rs

@@ -3,6 +3,7 @@
 use crate::compose::types::ComposeFile;
 use crate::error::{ComposeError, Result};
 
+use super::drop_recheck::LifecycleGoal;
 use super::filter_services;
 use super::parallel::{
 	filter_levels, first_error, join_bounded, restart_service_set, retain_levels,
@@ -81,6 +82,7 @@ impl Engine {
 		path: &str,
 		container: &str,
 		done: &str,
+		goal: LifecycleGoal,
 	) -> Result<bool> {
 		match self.client.post_empty_ok(path).await {
 			Ok(()) => {
@@ -91,6 +93,21 @@ impl Engine {
 				tracing::debug!("{container}: {done} skipped ({e})");
 				Ok(false)
 			}
+			// The server closed before completing the response. That is not an
+			// answer: the operation may have run to completion and lost only its
+			// reply. Measured on Podman 6 under concurrency, where the drops land
+			// on exactly these state-changing POSTs and follow a slow one — a
+			// restart that burned its full stop grace, then a drop on the next
+			// (#1339). It is not a client deadline (READ_TIMEOUT is 120s) and not
+			// a pooled-connection race (there is no pool; every request gets a
+			// fresh socket), so the transport genuinely cannot say.
+			//
+			// Resolve it the way `cp` and `stats` already do: ask the observable
+			// the transport cannot see. If the container reached the state the
+			// operation was for, it succeeded.
+			Err(e) if e.is_incomplete_message() => {
+				self.confirm_lost_response(container, done, goal, e).await
+			}
 			Err(e) => Err(ComposeError::Podman(e)),
 		}
 	}
@@ -148,6 +165,13 @@ impl Engine {
 				tracing::debug!("{container}: stop skipped ({e})");
 				Ok(())
 			}
+			// `stop` is one of the four state-changing calls the drops were
+			// measured on (#1339), and it does not go through `run_lifecycle_op`,
+			// so it needs the re-check on its own.
+			Err(e) if e.is_incomplete_message() => self
+				.confirm_lost_response(container, "Stopped", LifecycleGoal::NotRunning, e)
+				.await
+				.map(|_| ()),
 			Err(e) if e.is_timeout() => {
 				tracing::warn!(
 					"{container}: stop did not complete within the grace window; escalating to SIGKILL"