Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions src/hypervisor/classifier.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,16 +93,23 @@ pub(crate) fn classify_create_error(err: &RuntimeError) -> Disposition {
/// exception because they are *unambiguously* permanent under the standard shell
/// convention, so a restart can never succeed:
///
/// * `0` — the process ran to completion successfully. A worker that exits 0
/// has *finished*, not crashed: it must converge to `Completed`, never be
/// recreated. Treating it as retryable recreates the container every tick
/// forever (re-pulling the image each time under the default `Always`
/// policy) — a one-shot/`pg_dump`-style container declared as a worker would
/// otherwise loop endlessly and starve the reconcile cycle.
/// * `127` — command not found (the entrypoint/binary doesn't exist);
/// * `126` — found but not executable (bad perms / not a binary).
///
/// Both mean the container can never start its program, so we fail fast onto
/// 126/127 mean the container can never start its program, so we fail fast onto
/// `CreateContainerError` rather than burning the whole restart budget. Every
/// other code (including `0`, generic `1`, and signal-kill `128+n`) stays
/// retryable — those can be transient, and mislabelling them terminal would
/// wrongly give up on a recoverable worker.
/// other code (generic `1`, signal-kill `128+n`) stays retryable — those can be
/// transient, and mislabelling them terminal would wrongly give up on a
/// recoverable worker.
pub(crate) fn classify_exit_code(exit_code: Option<i64>) -> Disposition {
match exit_code {
Some(0) => Disposition::Terminal(DeploymentStatus::Completed),
Some(126) | Some(127) => Disposition::Terminal(DeploymentStatus::CreateContainerError),
_ => Disposition::Retry,
}
Expand Down Expand Up @@ -183,9 +190,19 @@ mod tests {
);
}

#[test]
fn clean_exit_completes() {
// A successful exit (code 0) is terminal-Completed, never retried — a
// worker that finished must not be recreated in a loop.
assert_eq!(
classify_exit_code(Some(0)),
Disposition::Terminal(DeploymentStatus::Completed)
);
}

#[test]
fn other_exit_codes_retry() {
assert_eq!(classify_exit_code(Some(0)), Disposition::Retry);
// Generic failures and signal kills stay retryable (could be transient).
assert_eq!(classify_exit_code(Some(1)), Disposition::Retry);
assert_eq!(classify_exit_code(Some(137)), Disposition::Retry);
assert_eq!(classify_exit_code(None), Disposition::Retry);
Expand Down
45 changes: 42 additions & 3 deletions src/runtime/docker/lifecycle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,26 @@ pub(crate) async fn apply(
/// daemon — see the tests in this module.
fn apply_unexpected_exits(deployment: &mut Deployment, exited: &[(String, Option<i64>)]) -> bool {
for (container_id, exit_code) in exited {
let disposition = crate::hypervisor::classifier::classify_exit_code(*exit_code);

// A clean exit (code 0) is a *success*, not a crash: the worker finished
// its work. Converge to Completed without touching restart_count, so it
// is never recreated — recreating an exit-0 container every tick is the
// infinite pull/recreate loop this guards against.
if let Disposition::Terminal(status @ DeploymentStatus::Completed) = disposition {
deployment.emit_event(
"info",
format!(
"Container {} exited cleanly (code 0); marking completed",
&container_id[..container_id.len().min(12)]
),
"docker",
Some("container_completed"),
);
deployment.status = status;
return true;
}

deployment.restart_count += 1;
deployment.emit_event(
"error",
Expand All @@ -76,9 +96,7 @@ fn apply_unexpected_exits(deployment: &mut Deployment, exited: &[(String, Option
// not-executable): the container can never start its program, so
// retrying it up to MAX_RESTART_COUNT only delays the inevitable. Land
// on the terminal status now.
if let Disposition::Terminal(status) =
crate::hypervisor::classifier::classify_exit_code(*exit_code)
{
if let Disposition::Terminal(status) = disposition {
deployment.emit_event(
"error",
format!(
Expand Down Expand Up @@ -743,6 +761,27 @@ mod tests {
assert_eq!(deployment.status, DeploymentStatus::Running);
}

/// A clean exit (code 0) is a success, not a crash: the worker is marked
/// Completed and `restart_count` is left untouched, so it is never recreated.
/// This is the production loop guard — a one-shot/`pg_dump`-style container
/// declared as a worker used to exit 0, get recreated, re-pull its image, and
/// loop forever, starving the reconcile cycle.
#[test]
fn clean_exit_completes_without_restart() {
let mut deployment = worker_running();
let exited = vec![("container-done".to_string(), Some(0))];
let stop = apply_unexpected_exits(&mut deployment, &exited);
assert!(
stop,
"a clean exit is terminal, reconciling stops this tick"
);
assert_eq!(deployment.status, DeploymentStatus::Completed);
assert_eq!(
deployment.restart_count, 0,
"a successful exit must not count as a crash"
);
}

/// Liveness gate: a container still running right after start may be
/// promoted to Running...
#[test]
Expand Down