From a783fae3c05f5b79840b7e7d988926b9c38df833 Mon Sep 17 00:00:00 2001 From: "E.FU" Date: Mon, 1 Jun 2026 09:15:43 +0800 Subject: [PATCH] feat: auto-retry-on-flaky for mix test.json (v0.5.0) --- .version | 2 +- CHANGELOG.md | 27 +++ README.md | 18 ++ ROADMAP.md | 5 +- lib/ex_unit_json.ex | 62 +++++- lib/ex_unit_json/config.ex | 22 +- lib/ex_unit_json/retry.ex | 151 +++++++++++++ lib/mix/tasks/test_json.ex | 345 +++++++++++++++++++++++++++--- roadmap/data.json | 23 ++ roadmap/tasks.toml | 14 ++ test/ex_unit_json/config_test.exs | 34 +++ test/ex_unit_json/retry_test.exs | 234 ++++++++++++++++++++ test/mix/tasks/test_json_test.exs | 135 ++++++++++++ 13 files changed, 1031 insertions(+), 41 deletions(-) create mode 100644 lib/ex_unit_json/retry.ex create mode 100644 test/ex_unit_json/retry_test.exs diff --git a/.version b/.version index 17b2ccd..8f0916f 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -0.4.3 +0.5.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 11f25ab..f4f9f1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,33 @@ Completed roadmap tasks. For upcoming work, see [ROADMAP.md](ROADMAP.md). --- +## v0.5.0 (2026-06-01) + +### New Features + +**Automatic retry-on-flaky (default ON)** + +When a run has failures, `mix test.json` now automatically re-runs only the previously-failed tests once (in a subprocess, via ExUnit's native `--failed`) and merges the two runs: + +- **confirmed** — failed both runs → stays in `tests`, stays red, exits non-zero. +- **flaky** — failed run 1, passed run 2 → moved to a new top-level `flaky` array (named, never hidden) and no longer blocks the run. + +When every first-run failure heals, `summary.result` becomes `"passed"` and the exit code is `0`, so an AI agent isn't blocked by an intermittent async/GenServer/LiveView/Port flake — while each flaky test is still surfaced. A test that fails both runs stays a hard failure. + +This is **default behavior** because the motivating problem is that agents run the bare `mix test.json` command and can't be forced to pass `--failed` themselves. Opt out with `--no-retry` or `config :ex_unit_json, retry: false`. + +The merged output adds (only when a retry ran): a `flaky` array, a `summary.flaky` count, and a `retry` metadata object (`ran`/`passes`/`retried`/`confirmed`/`flaky`). The schema `version` stays `1` (additive) — default output for green suites is byte-compatible. + +Retry is automatically skipped when it would be meaningless or unsupported: `--no-retry`, `config :ex_unit_json, retry: false`, `--failed` (already iterating; also prevents the retry subprocess recursing), `--summary-only`, `--first-failure`, `--compact`, `--group-by-error`, `--filter-out`, a `file:line` target, or umbrella projects. A green suite never triggers a second run (one extra temp-file round-trip, no second test run). + +### Internal + +- New module `ExUnitJSON.Retry` — pure `merge/2` overlay classifying flaky vs confirmed, matching tests across runs by `{module, name}` +- `ExUnitJSON.Config` gains `retry?/0` (reads `config :ex_unit_json, :retry`, default `true`) and a `:retry` option +- `Mix.Tasks.Test.Json` generalizes the temp-output buffer (cover/quiet/retry), adds the retry orchestration, and uses `System.halt(0)` only on the heal-to-green path (overrides ExUnit's at_exit failure status) + +--- + ## v0.4.3 (2026-04-18) > Note: v0.4.2 was published to Hex on 2026-02-28 from an out-of-tree state and diff --git a/README.md b/README.md index 7f7c16e..845845d 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ ExUnitJSON provides structured JSON output from `mix test` for use with AI edito - Drop-in replacement for `mix test` with JSON output - **AI-optimized default**: Shows only failures (use `--all` for all tests) +- **Automatic retry-on-flaky** (default): re-runs failed tests once; failures that heal are reported as `flaky` instead of blocking (`--no-retry` to opt out) - **Code coverage** with `--cover` and **coverage gating** with `--cover-threshold N` - Detailed failure information with assertion values and stacktraces - Filtering: `--summary-only`, `--first-failure`, `--filter-out`, `--group-by-error` @@ -70,9 +71,26 @@ mix test.json --quiet --all | `--compact` | Output JSONL with minimal keys (compact format) | | `--cover-threshold N` | Fail if coverage below N% (requires `--cover`) | | `--no-warn` | Suppress the "use --failed" tip | +| `--no-retry` | Disable automatic retry of failed tests | All standard `mix test` flags are passed through (`--failed`, `--only`, `--exclude`, `--seed`, etc.). +### Automatic Retry (Flaky Healing) + +When a run has failures, `mix test.json` re-runs only the previously-failed tests once and merges the results: + +- **confirmed** — failed both runs → stays red (`tests`), exits non-zero. +- **flaky** — failed then passed → moved to a top-level `flaky` array (named, never hidden) and no longer blocks the run. + +When every first-run failure heals, `summary.result` is `"passed"` and the exit code is `0`, so an AI agent isn't blocked by an intermittent async/GenServer/LiveView failure — while each flaky test is still surfaced. A `retry` metadata object (`retried`/`confirmed`/`flaky`) is added when a retry runs. + +Retry is skipped for `--no-retry`, `config :ex_unit_json, retry: false`, `--failed`, `--summary-only`, `--first-failure`, `--compact`, `--group-by-error`, `--filter-out`, a `file:line` target, or umbrella projects. A green suite never triggers a second run. + +```elixir +# Disable globally in config/test.exs +config :ex_unit_json, retry: false +``` + ### Code Coverage ```bash diff --git a/ROADMAP.md b/ROADMAP.md index 3252bd4..8b84d9d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,9 +28,9 @@ - No external dependencies for core functionality -**Focus phase:** 2 — Future Enhancements (8 of 18 done · 0 in progress) +**Focus phase:** 2 — Future Enhancements (9 of 19 done · 0 in progress) -**Last shipped:** no recent shipments +**Last shipped:** Task 30 — Automatic retry-on-flaky (default ON) on 2026-06-01 **Up next:** Task 20 — Consistent failure_message field (Schema v2) [D:2/B:7/U:7 → Eff:3.5] 🎯 @@ -85,6 +85,7 @@ shipped across the v0.3.x–v0.4.x line; pending features sorted by efficiency. | Task 27 | ⬜ | 🎁 **future** · CI systems integration [D:4/B:5/U:5 → Eff:1.25] 📋 | | Task 28 | ⬜ | 🎁 **future** · Custom output templates [D:6/B:4/U:4 → Eff:0.67] ⚠️ | | Task 29 | ⬜ | 🎁 **future** · Optional Jason fallback [D:4/B:5/U:5 → Eff:1.25] 📋 | +| Task 30 | ✅ | 🎁 **failed_iteration** · Automatic retry-on-flaky (default ON) [D:7/B:9/U:9 → Eff:1.29] 📋 | --- diff --git a/lib/ex_unit_json.ex b/lib/ex_unit_json.ex index c81683b..b584199 100644 --- a/lib/ex_unit_json.ex +++ b/lib/ex_unit_json.ex @@ -9,6 +9,8 @@ defmodule ExUnitJSON do - Drop-in replacement for `mix test` with JSON output - **AI-optimized default**: Shows only failures (use `--all` for all tests) + - **Automatic retry-on-flaky** (default): failed tests are re-run once; failures + that heal are surfaced as `flaky` instead of blocking (opt out with `--no-retry`) - **Code coverage** available with `--cover` flag - **Coverage gating** with `--cover-threshold N` (fails if overall coverage drops below N) - All test states: passed, failed, skipped, excluded @@ -95,8 +97,34 @@ defmodule ExUnitJSON do # Output JSONL with minimal keys (compact format) mix test.json --quiet --compact + # Disable automatic retry of failed tests + mix test.json --quiet --no-retry + All standard `mix test` options are also supported (file paths, line numbers, etc.). + ## Automatic Retry (Flaky Healing) + + By **default**, when a run has failures, `mix test.json` re-runs only the + previously-failed tests once (in a subprocess, using ExUnit's `--failed`) and + merges the results: + + - **confirmed** — failed both runs → stays in `tests`, stays red, exits non-zero. + - **flaky** — failed then passed → moved to a top-level `flaky` array (never + hidden) and no longer blocks the run. + + When every first-run failure heals, the result goes `"passed"` and the exit code + is `0`, so an AI agent isn't blocked by a flake — while each flaky test is still + named in the output. This is the fix for the common loop where an agent treats an + intermittent async/GenServer/LiveView failure as a real regression. + + Retry is skipped (run-1 output reported unchanged) for `--no-retry`, + `config :ex_unit_json, retry: false`, `--failed`, `--summary-only`, + `--first-failure`, `--compact`, `--group-by-error`, `--filter-out`, a `file:line` + target, or umbrella projects. A green suite never triggers a second run. + + # Disable globally in config/test.exs + config :ex_unit_json, retry: false + ## Code Coverage Coverage is disabled by default for faster test runs. Use `--cover` to enable: @@ -185,6 +213,8 @@ defmodule ExUnitJSON do "seed": 12345, "summary": { ... }, "tests": [ ... ], + "flaky": [ ... ], + "retry": { ... }, "error_groups": [ ... ], "module_failures": [ ... ] } @@ -195,10 +225,35 @@ defmodule ExUnitJSON do | `seed` | integer | Random seed used for test ordering | | `summary` | object | Aggregate test statistics | | `tests` | array | Individual test results (omitted with `--summary-only`) | + | `flaky` | array | Tests that failed then passed on retry (only present when a retry healed something) | + | `retry` | object | Retry metadata (only present when a retry ran) | | `error_groups` | array | Failures grouped by message (only with `--group-by-error`) | | `module_failures` | array | setup_all failures (only present when failures occur) | | `coverage` | object | Code coverage data (included with `--cover`) | + ### Retry Object + + Present only when an automatic retry ran (a first run had failures): + + { + "ran": true, + "passes": 1, + "retried": 3, + "confirmed": 2, + "flaky": 1 + } + + | Field | Type | Description | + |-------|------|-------------| + | `ran` | boolean | Always `true` when present | + | `passes` | integer | Number of retry passes (currently always 1) | + | `retried` | integer | Number of failed tests re-run | + | `confirmed` | integer | Failures that recurred (still red) | + | `flaky` | integer | Failures that healed on retry | + + The `flaky` array contains full test objects (run-1 failure detail preserved); + flaky module failures additionally carry `"scope": "module"`. + ### Summary Object { @@ -209,6 +264,7 @@ defmodule ExUnitJSON do "excluded": 0, "invalid": 0, "filtered": 0, + "flaky": 0, "duration_us": 123456, "result": "failed" } @@ -217,13 +273,14 @@ defmodule ExUnitJSON do |-------|------|-------------| | `total` | integer | Total number of tests | | `passed` | integer | Tests that passed | - | `failed` | integer | Tests that failed | + | `failed` | integer | Confirmed failures (after retry, if one ran) | | `skipped` | integer | Tests skipped with `@tag :skip` | | `excluded` | integer | Tests excluded by tag filters | | `invalid` | integer | Tests with invalid state | | `filtered` | integer | Failed tests matching `--filter-out` patterns (only present when non-zero) | + | `flaky` | integer | Failures that healed on retry (only present when a retry ran) | | `duration_us` | integer | Total duration in microseconds | - | `result` | string | `"passed"` or `"failed"` | + | `result` | string | `"passed"` or `"failed"` (`"passed"` when all failures healed) | ### Test Object @@ -413,6 +470,7 @@ defmodule ExUnitJSON do * `ExUnitJSON.JSONEncoder` - Converts ExUnit structs to JSON maps * `ExUnitJSON.Config` - Configuration handling * `ExUnitJSON.Filters` - Test filtering logic + * `ExUnitJSON.Retry` - Merges a run with its retry to classify flaky vs confirmed * `ExUnitJSON.ErrorGroups` - Groups failures by error message * `ExUnitJSON.Coverage` - Code coverage collection * `ExUnitJSON.CompactOutput` - Compact JSONL output format diff --git a/lib/ex_unit_json/config.ex b/lib/ex_unit_json/config.ex index 15d96f9..5e98e83 100644 --- a/lib/ex_unit_json/config.ex +++ b/lib/ex_unit_json/config.ex @@ -17,6 +17,7 @@ defmodule ExUnitJSON.Config do * `:group_by_error` - When true, add error_groups array grouping failures by message * `:quiet` - When true, suppress Logger output for clean JSON * `:hint` - Controls the "use --failed" tip behavior + * `:retry` - When false (via `--no-retry`), disable auto-retry of failed tests """ @@ -31,6 +32,7 @@ defmodule ExUnitJSON.Config do | :group_by_error | :quiet | :hint + | :retry @typedoc "Keyword list of ExUnitJSON options" @type opts :: [ @@ -42,7 +44,8 @@ defmodule ExUnitJSON.Config do compact: boolean(), group_by_error: boolean(), quiet: boolean(), - hint: boolean() + hint: boolean(), + retry: boolean() ] @valid_options [ @@ -54,7 +57,8 @@ defmodule ExUnitJSON.Config do :compact, :group_by_error, :quiet, - :hint + :hint, + :retry ] @doc """ @@ -157,6 +161,20 @@ defmodule ExUnitJSON.Config do get_opt(:group_by_error, false) end + @doc """ + Checks if automatic retry-on-flaky is enabled via project config. + + Reads `config :ex_unit_json, :retry` directly from the application + environment (default `true`), mirroring how `:enforce_failed` is read. This + is a project-level setting evaluated before per-invocation `:opts` are stored, + so it deliberately does not consult `get_opts/0`. The `--no-retry` flag is + honored separately as a per-invocation opt by `Mix.Tasks.Test.Json`. + """ + @spec retry?() :: boolean() + def retry? do + Application.get_env(:ex_unit_json, :retry, true) + end + @doc false # Validates and filters options to only known keys defp validate_opts(opts) when is_list(opts) do diff --git a/lib/ex_unit_json/retry.ex b/lib/ex_unit_json/retry.ex new file mode 100644 index 0000000..565260c --- /dev/null +++ b/lib/ex_unit_json/retry.ex @@ -0,0 +1,151 @@ +defmodule ExUnitJSON.Retry do + @moduledoc """ + Merges a first test run with an automatic retry run to distinguish + *flaky* failures from *confirmed* ones. + + `mix test.json` re-runs the previously-failed tests (ExUnit's native + `--failed`) after a run with failures, then calls `merge/2` to overlay the + two result documents. The goal is to never block an AI agent on a failure + that heals on re-run, while never hiding a failure that doesn't. + + ## Classification + + Tests are matched across runs by their `{module, name}` identity (ExUnit's + canonical manifest key — `file:line` can shift between runs). + + * **confirmed** — failed run 1 and did *not* pass run 2. Stays in `tests` + and keeps run 1's failure detail. Counts toward `summary.failed`. + * **flaky** — failed run 1 but passed run 2. Moved out of `tests` into a + top-level `flaky` array (run 1's failure detail preserved so the agent + sees what flaked). Counted in `summary.flaky`, never in `summary.failed`. + + A test that failed run 1 but was not re-verified as passing (e.g. `--failed` + could not re-run it) stays **confirmed** — conservative by design: nothing is + marked flaky we could not observe passing. + + `setup_all` failures (`module_failures`) are classified the same way by module + name: recurs in run 2 → stays confirmed; cleared → moves to `flaky` tagged + `"scope" => "module"`. + + ## Output shape (additive to schema v1) + + The merged document is run 1's document with: + + * `tests` — flaky entries removed (passing/confirmed entries preserved, so + `--all` runs keep their passing tests) + * `module_failures` — only the recurring (confirmed) ones, omitted if none + * `flaky` — flaky tests and modules, omitted when empty + * `summary.failed` — confirmed test count; `summary.flaky` — flaky count; + `summary.result` — `"passed"` iff nothing is confirmed + * `retry` — `%{"ran" => true, "passes" => 1, "retried" => N, "confirmed" => X, "flaky" => Y}` + + All functions are pure — the orchestration (subprocess spawn, file IO, exit + code) lives in `Mix.Tasks.Test.Json`. + """ + + @typedoc "A decoded (string-keyed) JSON result document." + @type document :: %{optional(String.t()) => term()} + + @doc """ + Merges run 1 and run 2 result documents into a single document distinguishing + confirmed failures from flaky ones. + + Both arguments are string-keyed maps (as produced by `:json.decode/1` on a + buffered run). Run 2 is expected to have been produced with `--all` so every + re-run test carries its `"state"`. + """ + @spec merge(document(), document()) :: document() + def merge(run1, run2) do + run1_tests = Map.get(run1, "tests", []) + run2_passed_ids = passed_ids(run2) + + {flaky_tests, kept_tests} = + Enum.split_with(run1_tests, fn test -> + failed?(test) and id(test) in run2_passed_ids + end) + + {confirmed_mods, flaky_mods} = classify_modules(run1, run2) + + flaky = flaky_tests ++ Enum.map(flaky_mods, &Map.put(&1, "scope", "module")) + confirmed_test_count = count_failed(kept_tests) + confirmed_count = confirmed_test_count + length(confirmed_mods) + retried = count_failed(run1_tests) + length(Map.get(run1, "module_failures", [])) + + run1 + |> Map.put("tests", kept_tests) + |> put_or_drop("module_failures", confirmed_mods) + |> put_or_drop("flaky", flaky) + |> Map.put("summary", merge_summary(run1, confirmed_test_count, length(flaky), confirmed_count)) + |> Map.put("retry", %{ + "ran" => true, + "passes" => 1, + "retried" => retried, + "confirmed" => confirmed_count, + "flaky" => length(flaky) + }) + end + + @doc false + # {module, name} identity used to match tests across runs. + @spec id(map()) :: {term(), term()} + defp id(test), do: {Map.get(test, "module"), Map.get(test, "name")} + + @doc false + @spec failed?(map()) :: boolean() + defp failed?(test), do: Map.get(test, "state") == "failed" + + @doc false + @spec count_failed([map()]) :: non_neg_integer() + defp count_failed(tests), do: Enum.count(tests, &failed?/1) + + @doc false + # Ids of tests that passed in run 2 (the re-run subset). + @spec passed_ids(document()) :: MapSet.t() + defp passed_ids(run2) do + run2 + |> Map.get("tests", []) + |> Enum.filter(&(Map.get(&1, "state") == "passed")) + |> MapSet.new(&id/1) + end + + @doc false + # Splits run 1 module failures into {confirmed (recurred), flaky (cleared)} + # by module name. + @spec classify_modules(document(), document()) :: {[map()], [map()]} + defp classify_modules(run1, run2) do + run1_mods = Map.get(run1, "module_failures", []) + + run2_failed_names = + run2 + |> Map.get("module_failures", []) + |> MapSet.new(&Map.get(&1, "name")) + + Enum.split_with(run1_mods, &(Map.get(&1, "name") in run2_failed_names)) + end + + @doc false + # Builds the merged summary: confirmed failures stay in `failed`, healed ones + # surface in `flaky`, result goes green only when nothing is confirmed. + @spec merge_summary(document(), non_neg_integer(), non_neg_integer(), non_neg_integer()) :: map() + defp merge_summary(run1, confirmed_test_count, flaky_count, confirmed_count) do + result = if confirmed_count == 0, do: "passed", else: "failed" + + summary = + run1 + |> Map.get("summary", %{}) + |> Map.put("failed", confirmed_test_count) + |> Map.put("flaky", flaky_count) + |> Map.put("result", result) + + # When everything healed, any run-1 `invalid` count (from a setup_all that + # has since recovered) is stale — passing means nothing invalid remains. + if result == "passed", do: Map.put(summary, "invalid", 0), else: summary + end + + @doc false + # Sets a key to a non-empty list, or removes it when the list is empty + # (matches the formatter's omit-when-empty convention for module_failures/flaky). + @spec put_or_drop(map(), String.t(), [term()]) :: map() + defp put_or_drop(doc, key, []), do: Map.delete(doc, key) + defp put_or_drop(doc, key, list), do: Map.put(doc, key, list) +end diff --git a/lib/mix/tasks/test_json.ex b/lib/mix/tasks/test_json.ex index ecbc53d..b8b151c 100644 --- a/lib/mix/tasks/test_json.ex +++ b/lib/mix/tasks/test_json.ex @@ -38,6 +38,7 @@ defmodule Mix.Tasks.Test.Json do * `--group-by-error` - Group failures by similar error message * `--quiet` - Suppress Logger output and TIP warnings for clean JSON piping * `--no-warn` - Suppress the "use --failed" warning when previous failures exist + * `--no-retry` - Disable automatic retry of failed tests (see "Automatic Retry") * `--cover` - Enable code coverage (off by default for faster runs) * `--cover-threshold N` - Fail if overall coverage is below N (0-100). Requires `--cover` @@ -97,6 +98,38 @@ defmodule Mix.Tasks.Test.Json do * `--only` or `--exclude` tag filters are used * `--no-warn` flag is passed + ## Automatic Retry (v0.5.0+) + + By **default**, when a run has failures, `mix test.json` automatically re-runs + only the previously-failed tests (ExUnit's native `--failed`) in a subprocess, + then merges the two runs to distinguish: + + * **confirmed** failures — failed both runs. Stay in `tests`, stay red. + * **flaky** failures — failed run 1, passed run 2. Surfaced in a top-level + `flaky` array (never hidden) but no longer block the run. + + When every first-run failure heals on retry, the suite reports + `summary.result == "passed"` and exits 0, so an AI agent isn't blocked by a + flake — while the flaky tests are still named in the output. A test that fails + both runs stays a hard failure (exit non-zero). + + The merged output adds (only when a retry ran): a `flaky` array, a + `summary.flaky` count, and a `retry` metadata block. The schema `version` + stays `1` (additive). + + Auto-retry is **skipped** (run-1 output is reported unchanged) when it would be + meaningless or unsupported: `--no-retry`, `config :ex_unit_json, retry: false`, + `--failed` (already iterating), `--summary-only`, `--first-failure`, + `--compact`, `--group-by-error`, `--filter-out`, a `file:line` target, or an + umbrella project. + + Disable it entirely: + + mix test.json --no-retry + + # or in config/test.exs + config :ex_unit_json, retry: false + ## Strict Enforcement To block full test runs when failures exist (useful for AI-assisted workflows): @@ -139,6 +172,12 @@ defmodule Mix.Tasks.Test.Json do # Coverage is OFF by default, enable with --cover cover_enabled? = Keyword.get(opts, :cover, false) + # Decide auto-retry up front (drives temp buffering). Capture the user's + # passthrough args before coverage injects its own --exclude, so the retry + # subprocess re-runs with the same selection the user asked for. + retry_enabled? = retry_enabled?(opts, test_args) + passthrough_args = test_args + # Ensure project is compiled before coverage instrumentation. # On clean builds, compile_project_modules() would otherwise find no beam files # because Mix.Task.run("test", ...) triggers compilation AFTER coverage starts. @@ -146,15 +185,17 @@ defmodule Mix.Tasks.Test.Json do test_args = maybe_start_coverage(test_args, cover_enabled?) - # When coverage is enabled or --quiet is used, we need to buffer output to a temp file - # so we can merge coverage data into the JSON before final output - {opts, temp_output_path} = maybe_use_temp_output_for_coverage(opts, cover_enabled?) + # Buffer output to a temp file when we must post-process before final output: + # coverage merge, --quiet stdout hygiene, or auto-retry overlay. + {opts, temp_output_path} = maybe_use_temp_output(opts, cover_enabled?, retry_enabled?) - # Check if user should use --failed (warn by default, block if configured) - handle_failed_usage_check(opts, test_args) + # Check if user should use --failed (warn by default, block if configured). + # Auto-retry supersedes the manual hint, so the TIP is suppressed when on. + handle_failed_usage_check(opts, test_args, retry_enabled?) - # Compute hint for JSON output (suggests --failed when appropriate) - opts = maybe_add_hint_opt(opts, test_args) + # Compute hint for JSON output (suggests --failed when appropriate). + # Skipped when auto-retry is handling re-runs to avoid double-signalling. + opts = if retry_enabled?, do: opts, else: maybe_add_hint_opt(opts, test_args) # In umbrella projects, each app runs its own ExUnit suite, each triggering # suite_finished which writes to the output file. Clear the file at the start @@ -171,28 +212,37 @@ defmodule Mix.Tasks.Test.Json do # as it uses mix test's native formatter handling. Mix.Task.run("test", ["--formatter", "ExUnitJSON.Formatter" | test_args]) - # Handle coverage and output based on configuration + post_run(opts, cover_enabled?, retry_enabled?, temp_output_path, passthrough_args) + end + + @doc false + # Dispatches to the retry overlay flow or the plain coverage/buffer flow. + @spec post_run(keyword(), boolean(), boolean(), String.t() | nil, [String.t()]) :: :ok + defp post_run(opts, cover_enabled?, true = _retry_enabled?, temp_output_path, passthrough_args) do + run_retry_flow(opts, cover_enabled?, temp_output_path, passthrough_args) + end + + defp post_run(opts, cover_enabled?, false = _retry_enabled?, temp_output_path, _passthrough_args) do + finalize_without_retry(opts, cover_enabled?, temp_output_path) + end + + @doc false + # The original (no-retry) coverage/buffer output handling. + # Note: we don't call Coverage.stop() here because :cover.stop() can kill + # processes that imported cover-compiled modules. The cover server is cleaned + # up when the process exits. + @spec finalize_without_retry(keyword(), boolean(), String.t() | nil) :: :ok + defp finalize_without_retry(opts, cover_enabled?, temp_output_path) do cond do - # Coverage enabled with temp buffer (stdout output) cover_enabled? and temp_output_path -> merge_coverage_into_output(temp_output_path, opts) - # Note: We don't call Coverage.stop() here because :cover.stop() - # can kill processes that imported cover-compiled modules. - # The cover server will be cleaned up when the process exits. - - # Coverage enabled with explicit --output file cover_enabled? and Keyword.has_key?(opts, :output) -> - output_path = Keyword.get(opts, :output) - merge_coverage_into_file(output_path, opts) + merge_coverage_into_file(Keyword.get(opts, :output), opts) - # Same as above - skip stop() to avoid killing the process - - # Temp buffer without coverage (just output it) temp_output_path -> output_buffered_json(temp_output_path) - # No temp buffer, no coverage - formatter already wrote output true -> :ok end @@ -273,6 +323,10 @@ defmodule Mix.Tasks.Test.Json do extract_json_opts(rest, [{:no_warn, true} | opts], remaining) end + defp extract_json_opts(["--no-retry" | rest], opts, remaining) do + extract_json_opts(rest, [{:retry, false} | opts], remaining) + end + defp extract_json_opts(["--cover" | rest], opts, remaining) do extract_json_opts(rest, [{:cover, true} | opts], remaining) end @@ -323,9 +377,12 @@ defmodule Mix.Tasks.Test.Json do defp maybe_start_coverage(test_args, false = _cover_enabled?), do: test_args @doc false - # Checks failed usage and shows warning/error as appropriate - @spec handle_failed_usage_check(keyword(), [String.t()]) :: :ok - defp handle_failed_usage_check(opts, test_args) do + # Checks failed usage and shows warning/error as appropriate. + # The `{:warn, _}` TIP is suppressed when auto-retry is enabled (retry + # supersedes the manual --failed suggestion); the enforce_failed block always + # fires, since it is a deliberate stricter config independent of retry. + @spec handle_failed_usage_check(keyword(), [String.t()], boolean()) :: :ok + defp handle_failed_usage_check(opts, test_args, retry_enabled?) do quiet? = Keyword.get(opts, :quiet, false) case check_failed_usage(opts, test_args) do @@ -348,7 +405,7 @@ defmodule Mix.Tasks.Test.Json do exit({:shutdown, 1}) - {:warn, count} when not quiet? -> + {:warn, count} when not quiet? and not retry_enabled? -> other_args = Enum.join(test_args, " ") IO.puts(:stderr, """ @@ -499,17 +556,16 @@ defmodule Mix.Tasks.Test.Json do end @doc false - # When coverage is enabled or --quiet is used, buffer output to temp file. - # This allows merging coverage data into JSON before final output. - @spec maybe_use_temp_output_for_coverage(keyword(), boolean()) :: {keyword(), String.t() | nil} - defp maybe_use_temp_output_for_coverage(opts, cover_enabled?) do - quiet? = Keyword.get(opts, :quiet, false) + # Buffer output to a temp file when we must post-process before final output. + # Buffer when (and only when) no explicit --output was given AND any of: + # 1. Coverage is enabled (need to merge coverage data) + # 2. --quiet is used (avoid stdout pollution) + # 3. Auto-retry is enabled (need to read run-1 before deciding to re-run) + @spec maybe_use_temp_output(keyword(), boolean(), boolean()) :: {keyword(), String.t() | nil} + defp maybe_use_temp_output(opts, cover_enabled?, retry_enabled?) do has_output? = Keyword.has_key?(opts, :output) - - # Buffer to temp file when: - # 1. Coverage is enabled (need to merge coverage data) - # 2. --quiet is used without explicit --output (avoid stdout pollution) - needs_temp_buffer? = (cover_enabled? and not has_output?) or (quiet? and not has_output?) + quiet? = Keyword.get(opts, :quiet, false) + needs_temp_buffer? = (cover_enabled? or quiet? or retry_enabled?) and not has_output? if needs_temp_buffer? do temp_path = Path.join(System.tmp_dir!(), "ex_unit_json_#{System.unique_integer([:positive])}.json") @@ -519,6 +575,227 @@ defmodule Mix.Tasks.Test.Json do end end + @doc false + # Decides whether to auto-retry failed tests. Default ON (project config + + # per-invocation opt both default true). Disabled when the manual `--failed` + # workflow is in play (also prevents the retry subprocess recursing), for + # output modes that strip the per-test data the merge needs (summary-only, + # first-failure, compact, group-by-error), when --filter-out provides an + # alternative flaky strategy, for a focused file:line target, or in umbrella + # projects (per-app suites + --failed interaction untested). + @spec retry_enabled?(keyword(), [String.t()]) :: boolean() + defp retry_enabled?(opts, test_args) do + ExUnitJSON.Config.retry?() and + Keyword.get(opts, :retry, true) and + not retry_disqualified_opts?(opts) and + not retry_disqualified_args?(test_args) and + not Mix.Project.umbrella?() + end + + @doc false + @spec retry_disqualified_opts?(keyword()) :: boolean() + defp retry_disqualified_opts?(opts) do + Keyword.get(opts, :summary_only, false) or + Keyword.get(opts, :first_failure, false) or + Keyword.get(opts, :compact, false) or + Keyword.get(opts, :group_by_error, false) or + Keyword.has_key?(opts, :filter_out) + end + + @doc false + @spec retry_disqualified_args?([String.t()]) :: boolean() + defp retry_disqualified_args?(test_args) do + "--failed" in test_args or Enum.any?(test_args, &String.contains?(&1, ".exs:")) + end + + @doc false + # Orchestrates the retry overlay: read run 1, and if it has failures, re-run + # the failed subset in a subprocess and merge. Coverage (if enabled) is + # collected from run 1 only and re-attached to the final document. + @spec run_retry_flow(keyword(), boolean(), String.t() | nil, [String.t()]) :: :ok + defp run_retry_flow(opts, cover_enabled?, temp_output_path, passthrough_args) do + run1_path = temp_output_path || Keyword.get(opts, :output) + coverage = if cover_enabled?, do: collect_coverage_with_threshold(opts) + + case read_document(run1_path) do + {:ok, run1_doc} -> + if document_has_failures?(run1_doc) do + retry_and_finalize(run1_doc, opts, coverage, temp_output_path, passthrough_args) + else + # Green run: no second run, emit run 1 (with coverage) unchanged. + finalize_document(run1_doc, opts, coverage, temp_output_path) + end + + {:error, _} -> + # Run 1 produced no parseable output (e.g. tests crashed early). + # Fall back to the plain coverage/buffer path; never mask the failure. + finalize_without_retry(opts, cover_enabled?, temp_output_path) + end + end + + @doc false + @spec retry_and_finalize(map(), keyword(), term(), String.t() | nil, [String.t()]) :: :ok + defp retry_and_finalize(run1_doc, opts, coverage, temp_output_path, passthrough_args) do + case run_retry_subprocess(passthrough_args) do + {:ok, run2_doc} -> + merged = ExUnitJSON.Retry.merge(run1_doc, run2_doc) + finalize_retry(merged, opts, coverage, temp_output_path) + + :error -> + IO.puts( + :stderr, + "Warning: ex_unit_json retry pass produced no parseable output; reporting first-run results." + ) + + finalize_document(run1_doc, opts, coverage, temp_output_path) + end + end + + @doc false + # Re-runs only the previously-failed tests in a fresh `mix test.json --failed` + # subprocess. ExUnit cannot run twice in one VM, so a subprocess is required. + # `--failed` keeps the retry from recursing (retry_enabled?/2 is false when + # --failed is present). `--all` makes run 2 report every re-run test's state. + @spec run_retry_subprocess([String.t()]) :: {:ok, map()} | :error + defp run_retry_subprocess(passthrough_args) do + tmp2 = Path.join(System.tmp_dir!(), "ex_unit_json_retry_#{System.unique_integer([:positive])}.json") + args = ["test.json", "--failed", "--all", "--output", tmp2 | passthrough_args] + + {_output, _exit_code} = + System.cmd("mix", args, cd: File.cwd!(), stderr_to_stdout: true, env: [{"MIX_ENV", "test"}]) + + result = read_document(tmp2) + File.rm(tmp2) + + case result do + {:ok, doc} -> {:ok, doc} + {:error, _} -> :error + end + end + + @doc false + # Writes a single (non-merged) document — used for the green and fallback + # paths. Attaches coverage when present and applies the cover threshold. + @spec finalize_document(map(), keyword(), term(), String.t() | nil) :: :ok + defp finalize_document(doc, opts, coverage, temp_output_path) do + doc + |> maybe_attach_coverage(coverage) + |> write_final(opts, temp_output_path) + + cleanup_temp(temp_output_path) + maybe_exit_for_coverage(coverage) + :ok + end + + @doc false + # Writes the merged document, then decides the exit code. ExUnit's at_exit + # already yields a non-zero status because run 1 failed; the only override is + # heal-to-green (all failures flaky) with coverage passing, where we force + # exit 0 via System.halt/1 (bypasses at_exit, flushes stdout). + @spec finalize_retry(map(), keyword(), term(), String.t() | nil) :: :ok + defp finalize_retry(merged, opts, coverage, temp_output_path) do + merged + |> maybe_attach_coverage(coverage) + |> write_final(opts, temp_output_path) + + cleanup_temp(temp_output_path) + maybe_halt_for_retry_result(merged, coverage) + :ok + end + + @doc false + @spec maybe_attach_coverage(map(), term()) :: map() + defp maybe_attach_coverage(doc, nil), do: doc + defp maybe_attach_coverage(doc, {coverage, _threshold_met?}), do: Map.put(doc, "coverage", coverage) + + @doc false + # Writes the final JSON to stdout (when buffered to temp) or to the user's + # explicit --output file (when no temp buffer was needed). + @spec write_final(map(), keyword(), String.t() | nil) :: :ok + defp write_final(doc, opts, nil) do + # sobelow_skip ["Traversal.FileModule"] + File.write!(Keyword.get(opts, :output), :json.encode(doc)) + end + + defp write_final(doc, _opts, _temp_output_path) do + IO.write(:json.encode(doc)) + end + + @doc false + @spec cleanup_temp(String.t() | nil) :: :ok + defp cleanup_temp(nil), do: :ok + + defp cleanup_temp(path) do + File.rm(path) + :ok + end + + @doc false + # For the green/fallback path: enforce the cover threshold if one was set. + @spec maybe_exit_for_coverage(term()) :: :ok + defp maybe_exit_for_coverage(nil), do: :ok + defp maybe_exit_for_coverage({coverage, threshold_met?}), do: maybe_exit_on_cover_threshold(threshold_met?, coverage) + + @doc false + # Heal-to-green override. Coverage-below-threshold wins (report it; ExUnit's + # at_exit yields the non-zero status since run 1 failed). Otherwise, when the + # merged result is green, halt(0) to clear ExUnit's pending failure status. + @spec maybe_halt_for_retry_result(map(), term()) :: :ok + defp maybe_halt_for_retry_result(merged, coverage) do + if coverage_threshold_ok?(coverage) and get_in(merged, ["summary", "result"]) == "passed" do + System.halt(0) + end + + :ok + end + + @doc false + # Returns true when there is no threshold or it was met; emits the standard + # coverage error to stderr and returns false when the threshold was missed. + @spec coverage_threshold_ok?(term()) :: boolean() + defp coverage_threshold_ok?(nil), do: true + defp coverage_threshold_ok?({_coverage, nil}), do: true + defp coverage_threshold_ok?({_coverage, true}), do: true + + defp coverage_threshold_ok?({coverage, false}) do + total = coverage["total_percentage"] + threshold = coverage["threshold"] + IO.puts(:stderr, "ERROR: Coverage #{total}% is below threshold #{threshold}%") + false + end + + @doc false + # Reads and decodes a buffered JSON document. Returns {:error, _} when the + # file is missing, empty, or not valid JSON (so callers can fall back). + @spec read_document(String.t() | nil) :: {:ok, map()} | {:error, atom()} + defp read_document(nil), do: {:error, :no_path} + + defp read_document(path) do + case File.read(path) do + {:ok, content} when byte_size(content) > 0 -> + try do + {:ok, :json.decode(content)} + rescue + _ -> {:error, :invalid_json} + end + + _ -> + {:error, :no_output} + end + end + + @doc false + # Detects failures from the summary (robust across --all / failures-only) plus + # any setup_all module failures. + @spec document_has_failures?(map()) :: boolean() + defp document_has_failures?(doc) do + summary = Map.get(doc, "summary", %{}) + + Map.get(summary, "failed", 0) > 0 or + Map.get(summary, "invalid", 0) > 0 or + Map.get(doc, "module_failures", []) != [] + end + @doc false # Outputs buffered JSON from temp file and cleans up. @spec output_buffered_json(String.t()) :: :ok diff --git a/roadmap/data.json b/roadmap/data.json index 5ddeeab..df3e941 100644 --- a/roadmap/data.json +++ b/roadmap/data.json @@ -698,6 +698,29 @@ "body": "Optional Jason fallback for Elixir versions without the built-in :json module. Score estimated during migration (not scored in source).", "scored_at": "2026-05-31", "cross_repo": [] + }, + { + "id": 30, + "phase": 2, + "bundle": "failed_iteration", + "status": "done", + "title": "Automatic retry-on-flaky (default ON)", + "scores": { + "d": 7, + "b": 9, + "u": 9 + }, + "eff": 1.29, + "dep_layer": 0, + "markers": [], + "depends_on": [], + "body": "Agents run bare `mix test.json` and can't be forced to pass `--failed`, so a 1-2 red flake out of hundreds gets treated as a real regression. Make retry default behavior inside the task: on failures, re-run only the previously-failed tests (ExUnit `--failed`) in a subprocess, merge the two runs, classify confirmed (failed both) vs flaky (failed then passed). All-flaky heals to result=passed/exit 0 so the agent is unblocked; flaky tests are surfaced in a `flaky[]` array (never hidden, honors NEVER HIDE TEST FAILURES). Opt out via `--no-retry` / `config :ex_unit_json, retry: false`.", + "created_at": "2026-06-01", + "done_at": "2026-06-01", + "scored_at": "2026-06-01", + "implemented": "New ExUnitJSON.Retry.merge/2 overlays run-1 and the --failed --all retry subprocess, matching tests by {module,name}; adds top-level flaky[], summary.flaky, and a retry{} block (schema v1, additive). Heal-to-green forces exit 0 via System.halt(0); confirmed failures ride ExUnit's at_exit exit 2. Default ON; skipped for --no-retry/config, --failed, --summary-only, --first-failure, --compact, --group-by-error, --filter-out, file:line targets, and umbrella projects. Config.retry?/0 reads project config (default true). 12 unit + 4 integration tests, all green.", + "delivered_by": "claude", + "cross_repo": [] } ] } \ No newline at end of file diff --git a/roadmap/tasks.toml b/roadmap/tasks.toml index b8e0829..9f9bde4 100644 --- a/roadmap/tasks.toml +++ b/roadmap/tasks.toml @@ -453,3 +453,17 @@ title = "Optional Jason fallback" body = "Optional Jason fallback for Elixir versions without the built-in :json module. Score estimated during migration (not scored in source)." scores = { d = 4, b = 5, u = 5 } scored_at = "2026-05-31" + +[[task]] +id = 30 +phase = 2 +bundle = "failed_iteration" +status = "done" +title = "Automatic retry-on-flaky (default ON)" +scores = { d = 7, b = 9, u = 9 } +body = "Agents run bare `mix test.json` and can't be forced to pass `--failed`, so a 1-2 red flake out of hundreds gets treated as a real regression. Make retry default behavior inside the task: on failures, re-run only the previously-failed tests (ExUnit `--failed`) in a subprocess, merge the two runs, classify confirmed (failed both) vs flaky (failed then passed). All-flaky heals to result=passed/exit 0 so the agent is unblocked; flaky tests are surfaced in a `flaky[]` array (never hidden, honors NEVER HIDE TEST FAILURES). Opt out via `--no-retry` / `config :ex_unit_json, retry: false`." +implemented = "New ExUnitJSON.Retry.merge/2 overlays run-1 and the --failed --all retry subprocess, matching tests by {module,name}; adds top-level flaky[], summary.flaky, and a retry{} block (schema v1, additive). Heal-to-green forces exit 0 via System.halt(0); confirmed failures ride ExUnit's at_exit exit 2. Default ON; skipped for --no-retry/config, --failed, --summary-only, --first-failure, --compact, --group-by-error, --filter-out, file:line targets, and umbrella projects. Config.retry?/0 reads project config (default true). 12 unit + 4 integration tests, all green." +delivered_by = "claude" +created_at = "2026-06-01" +scored_at = "2026-06-01" +done_at = "2026-06-01" diff --git a/test/ex_unit_json/config_test.exs b/test/ex_unit_json/config_test.exs index df33500..d2d2c59 100644 --- a/test/ex_unit_json/config_test.exs +++ b/test/ex_unit_json/config_test.exs @@ -194,4 +194,38 @@ defmodule ExUnitJSON.ConfigTest do assert Keyword.get(opts, :quiet) == nil end end + + describe "retry?/0" do + setup do + original = Application.get_env(:ex_unit_json, :retry) + on_exit(fn -> restore_env(:retry, original) end) + :ok + end + + test "defaults to true when :retry is not configured" do + Application.delete_env(:ex_unit_json, :retry) + assert Config.retry?() == true + end + + test "returns false when config :ex_unit_json, retry: false" do + Application.put_env(:ex_unit_json, :retry, false) + assert Config.retry?() == false + end + + test "returns true when config :ex_unit_json, retry: true" do + Application.put_env(:ex_unit_json, :retry, true) + assert Config.retry?() == true + end + end + + describe "retry option" do + test ":retry is preserved through get_opts/0 (so --no-retry survives validation)" do + Application.put_env(:ex_unit_json, :opts, retry: false) + opts = Config.get_opts() + assert Keyword.get(opts, :retry) == false + end + end + + defp restore_env(key, nil), do: Application.delete_env(:ex_unit_json, key) + defp restore_env(key, value), do: Application.put_env(:ex_unit_json, key, value) end diff --git a/test/ex_unit_json/retry_test.exs b/test/ex_unit_json/retry_test.exs new file mode 100644 index 0000000..ca3c76b --- /dev/null +++ b/test/ex_unit_json/retry_test.exs @@ -0,0 +1,234 @@ +defmodule ExUnitJSON.RetryTest do + use ExUnit.Case, async: true + + alias ExUnitJSON.Retry + + # Builds a string-keyed test map matching the formatter's decoded output. + defp test_map(module, name, state, extra \\ %{}) do + Map.merge( + %{ + "module" => module, + "name" => name, + "state" => state, + "file" => "test/example_test.exs", + "line" => 1, + "failures" => failures_for(state) + }, + extra + ) + end + + defp failures_for("failed"), do: [%{"kind" => "assertion", "message" => "boom"}] + defp failures_for(_), do: [] + + # Builds a string-keyed run document. + defp doc(tests, opts \\ []) do + failed = Enum.count(tests, &(&1["state"] == "failed")) + passed = Enum.count(tests, &(&1["state"] == "passed")) + + summary = + %{ + "total" => length(tests), + "passed" => passed, + "failed" => failed, + "skipped" => 0, + "excluded" => 0, + "invalid" => Keyword.get(opts, :invalid, 0), + "duration_us" => 1234, + "result" => if(failed > 0 or Keyword.get(opts, :invalid, 0) > 0, do: "failed", else: "passed") + } + + maybe_put( + %{"version" => 1, "seed" => 7, "summary" => summary, "tests" => tests}, + "module_failures", + Keyword.get(opts, :module_failures) + ) + end + + defp maybe_put(map, _key, nil), do: map + defp maybe_put(map, key, value), do: Map.put(map, key, value) + + defp module_failure(name) do + %{ + "name" => name, + "file" => "test/foo_test.exs", + "state" => "failed", + "failures" => [%{"message" => "setup_all boom"}] + } + end + + describe "merge/2 — all flaky (heal to green)" do + test "every run-1 failure passing in run 2 goes green, surfaced as flaky" do + run1 = doc([test_map("FooTest", "test a", "failed"), test_map("FooTest", "test b", "failed")]) + run2 = doc([test_map("FooTest", "test a", "passed"), test_map("FooTest", "test b", "passed")]) + + merged = Retry.merge(run1, run2) + + assert merged["summary"]["result"] == "passed" + assert merged["summary"]["failed"] == 0 + assert merged["summary"]["flaky"] == 2 + assert merged["tests"] == [] + assert length(merged["flaky"]) == 2 + assert merged["retry"] == %{"ran" => true, "passes" => 1, "retried" => 2, "confirmed" => 0, "flaky" => 2} + end + + test "flaky entries preserve run-1 failure detail" do + run1 = doc([test_map("FooTest", "test a", "failed")]) + run2 = doc([test_map("FooTest", "test a", "passed")]) + + merged = Retry.merge(run1, run2) + + [flaky] = merged["flaky"] + assert flaky["name"] == "test a" + assert flaky["failures"] == [%{"kind" => "assertion", "message" => "boom"}] + end + end + + describe "merge/2 — all confirmed (stays red)" do + test "every run-1 failure failing again stays in tests, no flaky key" do + run1 = doc([test_map("FooTest", "test a", "failed"), test_map("FooTest", "test b", "failed")]) + run2 = doc([test_map("FooTest", "test a", "failed"), test_map("FooTest", "test b", "failed")]) + + merged = Retry.merge(run1, run2) + + assert merged["summary"]["result"] == "failed" + assert merged["summary"]["failed"] == 2 + assert merged["summary"]["flaky"] == 0 + assert length(merged["tests"]) == 2 + refute Map.has_key?(merged, "flaky") + assert merged["retry"]["confirmed"] == 2 + end + + test "confirmed failures retain run-1 detail (not run-2)" do + run1 = doc([test_map("FooTest", "test a", "failed", %{"failures" => [%{"message" => "run1 detail"}]})]) + run2 = doc([test_map("FooTest", "test a", "failed", %{"failures" => [%{"message" => "run2 detail"}]})]) + + merged = Retry.merge(run1, run2) + + [confirmed] = merged["tests"] + assert confirmed["failures"] == [%{"message" => "run1 detail"}] + end + end + + describe "merge/2 — mixed" do + test "one flaky, one confirmed, one not re-verified stays confirmed" do + run1 = + doc([ + test_map("FooTest", "heals", "failed"), + test_map("FooTest", "stays", "failed"), + test_map("FooTest", "not_rerun", "failed") + ]) + + # run 2 only re-ran two of them: one healed, one still failing. + run2 = doc([test_map("FooTest", "heals", "passed"), test_map("FooTest", "stays", "failed")]) + + merged = Retry.merge(run1, run2) + + assert merged["summary"]["result"] == "failed" + assert merged["summary"]["failed"] == 2 + assert merged["summary"]["flaky"] == 1 + + flaky_names = Enum.map(merged["flaky"], & &1["name"]) + assert flaky_names == ["heals"] + + confirmed_names = merged["tests"] |> Enum.map(& &1["name"]) |> Enum.sort() + assert confirmed_names == ["not_rerun", "stays"] + end + end + + describe "merge/2 — {module, name} matching" do + test "same test name in different modules is matched independently" do + run1 = doc([test_map("ATest", "same name", "failed"), test_map("BTest", "same name", "failed")]) + # Only A's heals; B still fails. + run2 = doc([test_map("ATest", "same name", "passed"), test_map("BTest", "same name", "failed")]) + + merged = Retry.merge(run1, run2) + + assert Enum.map(merged["flaky"], & &1["module"]) == ["ATest"] + assert Enum.map(merged["tests"], & &1["module"]) == ["BTest"] + end + end + + describe "merge/2 — --all preserves passing tests" do + test "passing tests survive the merge; only flaky failures are removed" do + run1 = + doc([ + test_map("FooTest", "passer", "passed"), + test_map("FooTest", "flaker", "failed"), + test_map("FooTest", "real", "failed") + ]) + + run2 = doc([test_map("FooTest", "flaker", "passed"), test_map("FooTest", "real", "failed")]) + + merged = Retry.merge(run1, run2) + + names = merged["tests"] |> Enum.map(& &1["name"]) |> Enum.sort() + assert names == ["passer", "real"] + assert Enum.map(merged["flaky"], & &1["name"]) == ["flaker"] + end + end + + describe "merge/2 — module failures (setup_all)" do + test "a cleared setup_all failure becomes a flaky module" do + run1 = doc([], module_failures: [module_failure("FlakyModuleTest")]) + run2 = doc([]) + + merged = Retry.merge(run1, run2) + + assert merged["summary"]["result"] == "passed" + refute Map.has_key?(merged, "module_failures") + [flaky] = merged["flaky"] + assert flaky["name"] == "FlakyModuleTest" + assert flaky["scope"] == "module" + assert merged["retry"]["confirmed"] == 0 + assert merged["retry"]["flaky"] == 1 + end + + test "a recurring setup_all failure stays confirmed" do + run1 = doc([], module_failures: [module_failure("RealModuleTest")], invalid: 2) + run2 = doc([], module_failures: [module_failure("RealModuleTest")]) + + merged = Retry.merge(run1, run2) + + assert merged["summary"]["result"] == "failed" + assert merged["module_failures"] == [module_failure("RealModuleTest")] + refute Map.has_key?(merged, "flaky") + assert merged["retry"]["confirmed"] == 1 + end + + test "healed module clears the stale invalid count" do + run1 = doc([], module_failures: [module_failure("FlakyModuleTest")], invalid: 3) + run2 = doc([]) + + merged = Retry.merge(run1, run2) + + assert merged["summary"]["invalid"] == 0 + end + end + + describe "merge/2 — preserves baseline document fields" do + test "version, seed, and untouched summary counts carry through" do + run1 = doc([test_map("FooTest", "a", "failed")]) + run2 = doc([test_map("FooTest", "a", "passed")]) + + merged = Retry.merge(run1, run2) + + assert merged["version"] == 1 + assert merged["seed"] == 7 + assert merged["summary"]["total"] == 1 + assert merged["summary"]["duration_us"] == 1234 + end + + test "empty run (no failures) yields a green, retry-tagged document" do + run1 = doc([]) + run2 = doc([]) + + merged = Retry.merge(run1, run2) + + assert merged["summary"]["result"] == "passed" + assert merged["tests"] == [] + refute Map.has_key?(merged, "flaky") + assert merged["retry"] == %{"ran" => true, "passes" => 1, "retried" => 0, "confirmed" => 0, "flaky" => 0} + end + end +end diff --git a/test/mix/tasks/test_json_test.exs b/test/mix/tasks/test_json_test.exs index 7432359..da3a923 100644 --- a/test/mix/tasks/test_json_test.exs +++ b/test/mix/tasks/test_json_test.exs @@ -84,6 +84,13 @@ defmodule Mix.Tasks.Test.JsonTest do assert rest == [] end + test "parses --no-retry flag" do + {opts, rest} = parse_args(["--no-retry"]) + + assert opts[:retry] == false + assert rest == [] + end + test "parses --cover flag" do {opts, rest} = parse_args(["--cover"]) @@ -933,6 +940,107 @@ defmodule Mix.Tasks.Test.JsonTest do end end + describe "integration: auto-retry-on-flaky" do + @tag :integration + test "flaky failure heals to green and is surfaced, not hidden" do + {test_file, cleanup} = flaky_fixture() + marker = Path.join(System.tmp_dir!(), "flaky_marker_#{System.unique_integer([:positive])}") + File.rm(marker) + + try do + {output, exit_code} = run_mix_test_json_with_env([test_file], [{"FLAKY_MARKER", marker}]) + + assert exit_code == 0, "Expected heal-to-green exit 0. Output: #{output}" + assert {:ok, json} = decode_json(output) + assert json["summary"]["result"] == "passed" + assert json["summary"]["failed"] == 0 + assert json["summary"]["flaky"] == 1 + # The flaky test is named, never swept away. + assert [flaky] = json["flaky"] + assert flaky["name"] =~ "heals on retry" + assert json["retry"] == %{"ran" => true, "passes" => 1, "retried" => 1, "confirmed" => 0, "flaky" => 1} + after + cleanup.() + File.rm(marker) + end + end + + @tag :integration + test "a genuine hard failure stays red after retry" do + {test_file, cleanup} = + create_temp_test_file(""" + defmodule AutoRetryHardFailTest do + use ExUnit.Case + test "always fails" do + assert 1 == 2 + end + end + """) + + try do + {output, exit_code} = run_mix_test_json([test_file]) + + assert exit_code != 0, "Expected confirmed failure to stay red. Output: #{output}" + assert {:ok, json} = decode_json(output) + assert json["summary"]["result"] == "failed" + assert json["summary"]["failed"] == 1 + assert json["summary"]["flaky"] == 0 + assert length(json["tests"]) == 1 + refute Map.has_key?(json, "flaky") + assert json["retry"]["confirmed"] == 1 + after + cleanup.() + end + end + + @tag :integration + test "a green suite runs once and adds no retry metadata" do + {test_file, cleanup} = + create_temp_test_file(""" + defmodule AutoRetryGreenTest do + use ExUnit.Case + test "passes" do + assert true + end + end + """) + + try do + {output, exit_code} = run_mix_test_json([test_file]) + + assert exit_code == 0 + assert {:ok, json} = decode_json(output) + assert json["summary"]["result"] == "passed" + # No second run happened: no retry block, no flaky key. + refute Map.has_key?(json, "retry") + refute Map.has_key?(json, "flaky") + after + cleanup.() + end + end + + @tag :integration + test "--no-retry reports the raw first run with no retry metadata" do + {test_file, cleanup} = flaky_fixture() + marker = Path.join(System.tmp_dir!(), "flaky_marker_#{System.unique_integer([:positive])}") + File.rm(marker) + + try do + {output, exit_code} = run_mix_test_json_with_env([test_file, "--no-retry"], [{"FLAKY_MARKER", marker}]) + + # Opt-out: the flaky test is reported as a plain failure, no healing. + assert exit_code != 0, "Expected --no-retry to leave the failure red. Output: #{output}" + assert {:ok, json} = decode_json(output) + assert json["summary"]["result"] == "failed" + refute Map.has_key?(json, "retry") + refute Map.has_key?(json, "flaky") + after + cleanup.() + File.rm(marker) + end + end + end + describe "focused_run?/1 helper" do test "detects .exs file targeting" do assert focused_run?(["test/foo_test.exs"]) @@ -1283,6 +1391,10 @@ defmodule Mix.Tasks.Test.JsonTest do extract_json_opts(rest, [{:no_warn, true} | opts], remaining) end + defp extract_json_opts(["--no-retry" | rest], opts, remaining) do + extract_json_opts(rest, [{:retry, false} | opts], remaining) + end + defp extract_json_opts(["--cover" | rest], opts, remaining) do extract_json_opts(rest, [{:cover, true} | opts], remaining) end @@ -1315,6 +1427,29 @@ defmodule Mix.Tasks.Test.JsonTest do {path, cleanup} end + # A deterministically-flaky fixture: fails the first run (no marker file yet), + # passes the retry (marker now exists). The marker persists across the + # in-process run 1 and the `--failed` retry subprocess via a shared tmp path + # passed through the FLAKY_MARKER env var. + defp flaky_fixture do + create_temp_test_file(""" + defmodule AutoRetryFlakyHealsTest do + use ExUnit.Case + + test "heals on retry" do + marker = System.get_env("FLAKY_MARKER") + + if marker && File.exists?(marker) do + assert true + else + if marker, do: File.write!(marker, "x") + flunk("first run fails on purpose") + end + end + end + """) + end + # Helper to run mix test.json as a shell command defp run_mix_test_json(args) do run_mix_test_json_with_env(args, [])