From 4f878bdd31253c9e2f4cbab0526268d78d6f0094 Mon Sep 17 00:00:00 2001 From: Mlanawo MBECHEZI Date: Fri, 3 Jul 2026 15:07:15 +0300 Subject: [PATCH 1/4] Honor per-check start_period grace before the readiness rollout deadline --- src/models/health_check.rs | 27 +++++++ src/runtime/docker/container.rs | 31 +++++++- src/scheduler/health_checker.rs | 9 +++ src/scheduler/scheduler.rs | 121 ++++++++++++++++++++++++++++++-- 4 files changed, 183 insertions(+), 5 deletions(-) diff --git a/src/models/health_check.rs b/src/models/health_check.rs index 4b2a55f2..cd798880 100644 --- a/src/models/health_check.rs +++ b/src/models/health_check.rs @@ -36,6 +36,17 @@ pub(crate) enum HealthCheck { /// to the scheduler's built-in window when unset. #[serde(default, skip_serializing_if = "Option::is_none")] min_healthy_time: Option, + /// Grace period before this check's failures count against the + /// deployment. A deployment that builds at boot (e.g. `bun run + /// build` behind Caddy) isn't ready for many seconds, and a + /// readiness probe running during that window would fail purely + /// because the app hasn't started serving yet. The scheduler's + /// readiness deadline (`rollout_deadline_exceeded`) starts counting + /// only after this period, and it is forwarded to Docker's native + /// `HEALTHCHECK start_period`. Parsed by `parse_duration`. Ignored + /// when `readiness = false`. + #[serde(default, skip_serializing_if = "Option::is_none")] + start_period: Option, }, #[serde(rename = "http")] Http { @@ -49,6 +60,8 @@ pub(crate) enum HealthCheck { readiness: bool, #[serde(default, skip_serializing_if = "Option::is_none")] min_healthy_time: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + start_period: Option, }, #[serde(rename = "command")] Command { @@ -62,6 +75,8 @@ pub(crate) enum HealthCheck { readiness: bool, #[serde(default, skip_serializing_if = "Option::is_none")] min_healthy_time: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + start_period: Option, }, } @@ -174,6 +189,17 @@ impl HealthCheck { } => min_healthy_time.as_deref(), } } + + /// Returns the configured build/boot grace period for this check, if any. + /// Only meaningful when `is_readiness() == true`; the scheduler takes the + /// maximum across readiness checks so the slowest-starting one wins. + pub(crate) fn start_period(&self) -> Option<&str> { + match self { + HealthCheck::Tcp { start_period, .. } + | HealthCheck::Http { start_period, .. } + | HealthCheck::Command { start_period, .. } => start_period.as_deref(), + } + } } impl Default for HealthCheck { @@ -186,6 +212,7 @@ impl Default for HealthCheck { on_failure: FailureAction::Restart, readiness: false, min_healthy_time: None, + start_period: None, } } } diff --git a/src/runtime/docker/container.rs b/src/runtime/docker/container.rs index 5783b447..e8358f21 100644 --- a/src/runtime/docker/container.rs +++ b/src/runtime/docker/container.rs @@ -110,7 +110,11 @@ fn build_health_config(health_checks: &[HealthCheck]) -> Option { interval: Some(to_nanos(hc.interval())), timeout: Some(to_nanos(hc.timeout())), retries: Some(hc.threshold() as i64), - start_period: None, + // Grace period before Docker's native HEALTHCHECK counts a failure — + // covers a slow in-container build (e.g. `bun run build` behind Caddy) + // that isn't serving yet. Mirrors the scheduler-side readiness deadline + // grace (`start_period_for`). None (Docker's default) when unset. + start_period: hc.start_period().map(to_nanos), start_interval: None, }) } @@ -993,6 +997,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: false, min_healthy_time: None, + start_period: None, }]; assert!(build_health_config(&hcs).is_none()); } @@ -1009,6 +1014,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: true, min_healthy_time: None, + start_period: None, }]; assert!(build_health_config(&hcs).is_none()); } @@ -1023,6 +1029,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: true, min_healthy_time: None, + start_period: None, }]; assert!(build_health_config(&hcs).is_none()); } @@ -1037,6 +1044,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: true, min_healthy_time: None, + start_period: None, }]; let cfg = build_health_config(&hcs).expect("should translate"); assert_eq!( @@ -1049,6 +1057,25 @@ mod tests { assert_eq!(cfg.interval, Some(10_000_000_000)); assert_eq!(cfg.timeout, Some(5_000_000_000)); assert_eq!(cfg.retries, Some(3)); + // No start_period declared → Docker's default (None). + assert_eq!(cfg.start_period, None); + } + + #[test] + fn build_health_config_forwards_start_period() { + let hcs = vec![HealthCheck::Command { + command: "test -f /var/run/kemeter/ready".to_string(), + interval: "10s".to_string(), + timeout: "5s".to_string(), + threshold: 3, + on_failure: FailureAction::Alert, + readiness: true, + min_healthy_time: None, + start_period: Some("180s".to_string()), + }]; + let cfg = build_health_config(&hcs).expect("should translate"); + // The grace period reaches Docker's native HEALTHCHECK in nanoseconds. + assert_eq!(cfg.start_period, Some(180_000_000_000)); } #[test] @@ -1062,6 +1089,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: true, min_healthy_time: None, + start_period: None, }, HealthCheck::Command { command: "/usr/local/bin/ready.sh".to_string(), @@ -1071,6 +1099,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: true, min_healthy_time: None, + start_period: None, }, ]; let cfg = build_health_config(&hcs).expect("should translate the command HC"); diff --git a/src/scheduler/health_checker.rs b/src/scheduler/health_checker.rs index 97b20770..8a2023b1 100644 --- a/src/scheduler/health_checker.rs +++ b/src/scheduler/health_checker.rs @@ -414,6 +414,7 @@ mod tests { on_failure: FailureAction::Restart, readiness: false, min_healthy_time: None, + start_period: None, }], ); @@ -447,6 +448,7 @@ mod tests { on_failure: FailureAction::Restart, readiness: false, min_healthy_time: None, + start_period: None, }], ); @@ -491,6 +493,7 @@ mod tests { on_failure: FailureAction::Restart, readiness: false, min_healthy_time: None, + start_period: None, }], ); @@ -523,6 +526,7 @@ mod tests { on_failure: FailureAction::Restart, readiness: false, min_healthy_time: None, + start_period: None, }], ); @@ -555,6 +559,7 @@ mod tests { on_failure: FailureAction::Restart, readiness: false, min_healthy_time: None, + start_period: None, }], ); @@ -602,6 +607,7 @@ mod tests { on_failure: FailureAction::Stop, readiness: false, min_healthy_time: None, + start_period: None, }], ); @@ -636,6 +642,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: false, min_healthy_time: None, + start_period: None, }], ); @@ -697,6 +704,7 @@ mod tests { on_failure: FailureAction::Restart, readiness: true, min_healthy_time: None, + start_period: None, } } @@ -709,6 +717,7 @@ mod tests { on_failure: FailureAction::Restart, readiness: false, min_healthy_time: None, + start_period: None, } } diff --git a/src/scheduler/scheduler.rs b/src/scheduler/scheduler.rs index 3d398af5..babbc842 100644 --- a/src/scheduler/scheduler.rs +++ b/src/scheduler/scheduler.rs @@ -4,7 +4,7 @@ use crate::models::config; use crate::models::config::Config; use crate::models::deployment_event; use crate::models::deployments::{self, Deployment, DeploymentStatus, EnvValue}; -use crate::models::health_check::HealthCheckStatus; +use crate::models::health_check::{HealthCheck, HealthCheckStatus}; use crate::models::health_check_logs; use crate::models::secret as SecretModel; use crate::models::volume::ResolvedMount; @@ -515,8 +515,6 @@ const DEFAULT_MIN_HEALTHY_TIME: Duration = Duration::from_secs(10); /// nothing is set. Malformed values are logged at warn and ignored — the /// rollout shouldn't stall just because someone typo'd a duration. fn min_healthy_time_for(child: &Deployment) -> Duration { - use crate::models::health_check::HealthCheck; - let mut window = DEFAULT_MIN_HEALTHY_TIME; let mut overridden = false; @@ -542,6 +540,38 @@ fn min_healthy_time_for(child: &Deployment) -> Duration { window } +/// Resolve the build/boot grace period for a deployment: the max of the per-HC +/// `start_period` across readiness checks. Zero when nothing is set. Malformed +/// values are logged at warn and ignored — a typo'd duration shouldn't shorten +/// the grace to zero and fail a healthy slow-building app. +/// +/// This is the window during which a readiness probe is expected to fail (the +/// app is still building / booting and not serving yet), so the readiness +/// deadline only starts counting *after* it. +fn start_period_for(child: &Deployment) -> Duration { + let mut window = Duration::ZERO; + + for hc in child.health_checks.iter().filter(|hc| hc.is_readiness()) { + if let Some(raw) = hc.start_period() { + match HealthCheck::parse_duration(raw) { + Ok(d) => { + if d > window { + window = d; + } + } + Err(e) => { + warn!( + "Invalid start_period '{}' on readiness HC for deployment {}: {} — ignoring", + raw, child.id, e + ); + } + } + } + } + + window +} + /// True when the child has been alive longer than the rollout deadline. /// /// Used as a safety valve: if a child's readiness probe never turns green, the @@ -550,6 +580,13 @@ fn min_healthy_time_for(child: &Deployment) -> Duration { /// via RING_ROLLOUT_DEADLINE (seconds, default 600). A child with an /// unparseable created_at is treated as not-yet-expired (fail safe: never force /// a drain on bad data). +/// +/// The deadline starts counting only *after* the readiness checks' grace period +/// (`start_period`), not at `created_at`. A runtime that builds at boot (e.g. +/// `bun run build` behind Caddy) legitimately fails readiness for the whole +/// build; without honouring the grace, a short `RING_ROLLOUT_DEADLINE` fails a +/// healthy app the instant its build outlasts the deadline. The effective +/// budget is therefore `start_period + RING_ROLLOUT_DEADLINE`. fn rollout_deadline_exceeded(child: &Deployment) -> bool { let deadline_secs: i64 = std::env::var("RING_ROLLOUT_DEADLINE") .ok() @@ -567,7 +604,9 @@ fn rollout_deadline_exceeded(child: &Deployment) -> bool { Err(_) => return false, }; - (chrono::Utc::now() - created).num_seconds() >= deadline_secs + let grace_secs = start_period_for(child).as_secs() as i64; + + (chrono::Utc::now() - created).num_seconds() >= grace_secs + deadline_secs } /// Evaluate the readiness state of a deployment from its recorded health-check @@ -1648,6 +1687,75 @@ mod tests { assert!(!rollout_deadline_exceeded(&child)); } + fn readiness_command_with_start_period(start_period: &str) -> HealthCheck { + HealthCheck::Command { + command: "test -f /var/run/kemeter/ready".to_string(), + interval: "10s".to_string(), + timeout: "5s".to_string(), + threshold: 3, + on_failure: FailureAction::Alert, + readiness: true, + min_healthy_time: None, + start_period: Some(start_period.to_string()), + } + } + + #[test] + fn start_period_zero_without_readiness_start_period() { + // No readiness HC declares a start_period → no grace. + let child = child_with_health_checks("no-grace", vec![readiness_command("ready")]); + assert_eq!(start_period_for(&child), Duration::ZERO); + } + + #[test] + fn start_period_uses_max_across_readiness_checks() { + let child = child_with_health_checks( + "max-grace", + vec![ + readiness_command_with_start_period("60s"), + readiness_command_with_start_period("180s"), + ], + ); + assert_eq!(start_period_for(&child), Duration::from_secs(180)); + } + + #[test] + fn start_period_ignores_malformed_value() { + // A typo'd duration must not shorten the grace to zero. + let child = child_with_health_checks( + "bad-grace", + vec![readiness_command_with_start_period("nope")], + ); + assert_eq!(start_period_for(&child), Duration::ZERO); + } + + #[test] + fn rollout_deadline_defers_by_start_period() { + // A slow-building app: 700s old, 180s build grace, default 600s + // deadline. Effective budget is 180 + 600 = 780s, so 700s is NOT yet + // past the deadline — without honouring start_period it would have + // failed at 600s. + let mut child = child_with_health_checks( + "slow-build", + vec![readiness_command_with_start_period("180s")], + ); + child.created_at = (chrono::Utc::now() - chrono::Duration::seconds(700)).to_string(); + assert!(!rollout_deadline_exceeded(&child)); + } + + #[test] + fn rollout_deadline_still_fires_past_grace_plus_deadline() { + // Same 180s grace, but now 800s old → past the 780s effective budget. + // The safety valve still fires so a genuinely broken probe can't pin + // the deployment forever. + let mut child = child_with_health_checks( + "stuck-build", + vec![readiness_command_with_start_period("180s")], + ); + child.created_at = (chrono::Utc::now() - chrono::Duration::seconds(800)).to_string(); + assert!(rollout_deadline_exceeded(&child)); + } + /// Insert a health_check row directly so the gate has something to read. /// `seconds_ago` is the offset from now() applied to all timestamps — /// makes the anti-flap window deterministic without sleeping. @@ -1685,6 +1793,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: true, min_healthy_time: None, + start_period: None, } } @@ -1697,6 +1806,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: true, min_healthy_time: None, + start_period: None, } } @@ -1709,6 +1819,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: false, min_healthy_time: None, + start_period: None, } } @@ -1723,6 +1834,7 @@ mod tests { on_failure: FailureAction::Alert, readiness, min_healthy_time: raw.map(|s| s.to_string()), + start_period: None, }) .collect(); child_with_health_checks(id, hcs) @@ -1860,6 +1972,7 @@ mod tests { on_failure: FailureAction::Alert, readiness: false, // not a readiness HC min_healthy_time: None, + start_period: None, }); let child = child_with_health_checks("child-7", hcs); insert_hc_result(&pool, "child-7", "command", "success", 30).await; From e8d3f111ef16c6597b307c22685fd6d2581e23d3 Mon Sep 17 00:00:00 2001 From: Mlanawo MBECHEZI Date: Fri, 3 Jul 2026 15:07:15 +0300 Subject: [PATCH 2/4] Forward start_period from apply manifests to the API --- src/commands/apply.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/commands/apply.rs b/src/commands/apply.rs index 10de9bc2..dc444adb 100644 --- a/src/commands/apply.rs +++ b/src/commands/apply.rs @@ -188,6 +188,8 @@ enum HealthCheck { readiness: bool, #[serde(default, skip_serializing_if = "Option::is_none")] min_healthy_time: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + start_period: Option, }, Http { url: String, @@ -200,6 +202,8 @@ enum HealthCheck { readiness: bool, #[serde(default, skip_serializing_if = "Option::is_none")] min_healthy_time: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + start_period: Option, }, Command { command: String, @@ -212,6 +216,8 @@ enum HealthCheck { readiness: bool, #[serde(default, skip_serializing_if = "Option::is_none")] min_healthy_time: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + start_period: Option, }, } From 79f6079a28023fce2094d25bc1ca27ba865b0b64 Mon Sep 17 00:00:00 2001 From: Mlanawo MBECHEZI Date: Fri, 3 Jul 2026 15:07:15 +0300 Subject: [PATCH 3/4] Document the start_period readiness grace window --- CHANGELOG.md | 1 + .../how-to/configure-health-checks.md | 24 +++++++++++++++++++ documentation/reference/manifest.md | 1 + 3 files changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b67e0229..99bb53ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Readiness health checks accept a `start_period` grace window: readiness failures during a slow build/boot no longer trip the rollout deadline (effective budget becomes `start_period + RING_ROLLOUT_DEADLINE`), and the value is forwarded to Docker's native `HEALTHCHECK start_period` (#183) - containerd runtime over native gRPC: index/entrypoint resolution for multi-arch images, CNI networking, logs, command health checks (#144) - Podman runtime, opt-in under `[server.runtime.podman]` (#139) - Firecracker microVM runtime (experimental): boot, networking, outbound NAT, restart reconciliation (#142, #146, #147) diff --git a/documentation/how-to/configure-health-checks.md b/documentation/how-to/configure-health-checks.md index 885d3d44..a8d0060e 100644 --- a/documentation/how-to/configure-health-checks.md +++ b/documentation/how-to/configure-health-checks.md @@ -153,6 +153,30 @@ For services that need longer than 10 s to be truly ready (JVM cold start, big i - When several readiness checks declare different values, the **maximum** wins, so the gate waits on the slowest probe. - A typo (unparseable value) logs a warning and falls back to the 10 s default; rollouts aren't blocked. +### Services that build at boot: tune `start_period` + +Some containers build their app *after* they start (e.g. `bun run build` behind Caddy). Their readiness probe legitimately fails for the whole build, which would trip the rollout deadline (`RING_ROLLOUT_DEADLINE`, default `600s`) and fail a perfectly healthy deployment. Set `start_period` on the readiness check to grant a grace window before failures count: + +```yaml +- type: command + command: test -f /var/run/kemeter/ready + interval: 5s + timeout: 2s + threshold: 3 + on_failure: alert + readiness: true + start_period: "180s" # 3 min of build/boot before failures count +``` + +- The rollout deadline only starts counting **after** the grace period. Effective budget = `start_period + RING_ROLLOUT_DEADLINE`. +- Same duration syntax as `interval` / `timeout` (`"500ms"`, `"180s"`). +- Only matters when `readiness: true`. +- When several readiness checks declare different values, the **maximum** wins. +- For `command` checks on Docker, the value is also forwarded to the native `HEALTHCHECK start_period`, so Docker won't mark the container `unhealthy` during the build. +- A typo (unparseable value) logs a warning and is ignored; the grace falls back to zero rather than blocking a slow build. + +`start_period` differs from `min_healthy_time`: `start_period` is a grace *before* failures count (build/boot), while `min_healthy_time` is how long the check must stay green *after* it first succeeds (anti-flap). + **Proxy bonus:** a `readiness: true` check of `type: command` is **also** translated into a native Docker `HEALTHCHECK`. [Sozune](https://sozune.kemeter.io) (the companion proxy) and other label-aware proxies gate on `Status: healthy`, so they won't route traffic to the new container while the readiness command fails. See [how-to: expose HTTP traffic](/documentation/how-to/expose-http-traffic) for the integration, or [Health checks design → proxy integration](/documentation/concepts/health-checks-design#proxy-integration) for the why. For HTTP readiness with proxy gating, wrap the probe in a shell command: diff --git a/documentation/reference/manifest.md b/documentation/reference/manifest.md index 54a7d764..0808b209 100644 --- a/documentation/reference/manifest.md +++ b/documentation/reference/manifest.md @@ -391,6 +391,7 @@ health_checks: | `on_failure` | yes | `restart` (recreate the instance), `stop` (delete the deployment), or `alert` (log only). | | `readiness` | no (default `false`) | When `true`, this check gates rolling updates and (for `command` on Docker) is translated into a native `HEALTHCHECK`. See [health checks (design) → the readiness gate](/documentation/concepts/health-checks-design#the-readiness-gate). | | `min_healthy_time` | no (default `10s`) | Anti-flap window for the readiness gate: the check must be green for this long before the parent is drained. Per-check; the scheduler takes the maximum across readiness checks. Ignored when `readiness: false`. Same syntax as `interval` / `timeout`. | +| `start_period` | no (default none) | Build/boot grace period during which readiness failures don't count. The rollout deadline (`RING_ROLLOUT_DEADLINE`) only starts *after* this window, and it is forwarded to Docker's native `HEALTHCHECK start_period`. Per-check; the scheduler takes the maximum across readiness checks. A malformed value logs a warning and is ignored. Ignored when `readiness: false`. Same syntax as `interval` / `timeout`. | ### Type-specific fields From 7471b3990e7b38be3829dbb069b989a5fb3230ee Mon Sep 17 00:00:00 2001 From: Mlanawo MBECHEZI Date: Fri, 3 Jul 2026 15:07:29 +0300 Subject: [PATCH 4/4] Add e2e coverage for the start_period readiness grace --- .../e2e/docker/t37_readiness_start_period.sh | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100755 tests/e2e/docker/t37_readiness_start_period.sh diff --git a/tests/e2e/docker/t37_readiness_start_period.sh b/tests/e2e/docker/t37_readiness_start_period.sh new file mode 100755 index 00000000..386f4be7 --- /dev/null +++ b/tests/e2e/docker/t37_readiness_start_period.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# T37: the readiness deadline honours a per-check `start_period` grace window. +# +# A deployment whose readiness probe never turns green is a safety-valve case: +# after RING_ROLLOUT_DEADLINE the scheduler marks it `failed` so a broken probe +# can't pin it forever. But some containers legitimately fail readiness for the +# whole boot — they *build* their app after starting (e.g. `bun run build` +# behind Caddy). Without a grace, a short deadline fails a perfectly healthy +# slow-building app the instant its build outlasts the deadline. +# +# `start_period` defers that deadline: the effective budget becomes +# `start_period + RING_ROLLOUT_DEADLINE`. This test proves it end-to-end against +# the real compiled binary and a real Docker container, using a readiness probe +# that never succeeds (the file it checks for is never created). +# +# Deadline is pinned short via RING_ROLLOUT_DEADLINE so the test runs in +# seconds instead of the 600s default. +# +# Invariants: +# 1. WITHOUT start_period: a never-ready deployment is marked `failed` shortly +# after the (short) deadline — the existing safety valve still fires. +# 2. WITH start_period > deadline: the same never-ready deployment is STILL +# `creating` well past the bare deadline (the grace defers the verdict). +# 3. WITH start_period: it does eventually fail once start_period + deadline +# elapses — the safety valve is deferred, not disabled. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../lib.sh +source "$SCRIPT_DIR/../lib.sh" + +# Short deadline so the test doesn't wait 600s. `start_ring` spawns the server +# as a child of this shell, so the export is inherited. +export RING_ROLLOUT_DEADLINE=10 + +log "== T37: readiness deadline honours start_period ==" + +start_ring +ring_login + +NS="ring-e2e-start-period" + +# A readiness probe that never succeeds: the file is never created inside the +# container, so `test -f` always exits non-zero and readiness stays red. +write_fixture() { + local file="$1" name="$2" start_period_line="$3" + cat > "$file" </dev/null \ + | jq -r --arg ns "$NS" --arg n "with-grace" \ + '.[] | select(.namespace==$ns and .name==$n) | .status' | head -n1) + if [ "$status" = "failed" ]; then + fail "Invariant 2: with-grace failed at second $i — start_period did not defer the deadline" + fi + sleep 1 +done +log "Invariant 2: PASS (with-grace survived 25s past the bare deadline — grace honoured)" + +############################################################################### +# Invariant 3 — the deferred safety valve still fires once the budget elapses. +############################################################################### +log "-- Invariant 3: with-grace eventually fails once start_period + deadline elapses" + +# Effective budget is 40 + 10 = 50s from created_at. We've already burned ~35s+ +# above; wait up to a further 60s for the verdict so the safety valve proves it +# is deferred, not disabled. +wait_deployment_status "$NS" "with-grace" "failed" 60 +log "Invariant 3: PASS (with-grace failed after the deferred budget — valve deferred, not disabled)" + +log "== T37: all invariants passed =="