Add retry/backoff and circuit breaker to health checks

diagnostic/build-94e0fb08.json

@@ -0,0 +1,87 @@
+{
+  "generated_at": "2026-06-20T16:39:09.565245+00:00",
+  "commit": "94e0fb08",
+  "diagnostic_logd": "diagnostic/build-94e0fb08.logd",
+  "diagnostic_logd_error": null,
+  "message_blocker": null,
+  "chunked": false,
+  "chunk_size_bytes": null,
+  "password": "efcd3c4a127e4b6e294d",
+  "decrypt_command": "encryptly unpack diagnostic/build-94e0fb08.logd <outdir> --password efcd3c4a127e4b6e294d",
+  "total_modules": 10,
+  "passed": 4,
+  "failed": 6,
+  "modules": [
+    {
+      "name": "backend",
+      "status": "FAIL",
+      "elapsed_seconds": 0,
+      "artifact": null,
+      "output": "Command not found: [Errno 2] No such file or directory: 'cargo'"
+    },
+    {
+      "name": "frontend",
+      "status": "PASS",
+      "elapsed_seconds": 6.051,
+      "artifact": "/Users/zhangsiqiang/Documents/Codex/2026-06-19/new-chat/outputs/thanhle-kickama-15-health-retry/frontend/dist",
+      "output": "> tent-frontend@0.0.0 build\n> tsc -b && vite build\n\nvite v6.4.3 building for production...\ntransforming...\n\u2713 100 modules transformed.\nrendering chunks...\ncomputing gzip size...\ndist/index.html                   0.62 kB \u2502 gzip:  0.34 kB\ndist/assets/state-BkjSKDbY.js     8.91 kB \u2502 gzip:  3.55 kB \u2502 map:    57.15 kB\ndist/assets/vendor-CREcWLHI.js   48.93 kB \u2502 gzip: 17.22 kB \u2502 map:   481.27 kB\ndist/assets/index-CyxcoTyU.js   231.32 kB \u2502 gzip: 72.02 kB \u2502 map: 1,044.42 kB\n\u2713 built in 645ms"
+    },
+    {
+      "name": "market",
+      "status": "FAIL",
+      "elapsed_seconds": 0,
+      "artifact": null,
+      "output": "Command not found: [Errno 2] No such file or directory: 'go'"
+    },
+    {
+      "name": "frailbox",
+      "status": "FAIL",
+      "elapsed_seconds": 0.471,
+      "artifact": null,
+      "output": "gcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c src/arena.c -o build/src/arena.o\nsrc/arena.c:17:23: error: use of undeclared identifier 'MAP_HUGETLB'\n   17 |         mmap_flags |= MAP_HUGETLB;\n      |                       ^~~~~~~~~~~\nsrc/arena.c:179:17: warning: comparison of distinct pointer types ('const void *' and 'char *') [-Wcompare-distinct-pointer-types]\n  179 |             ptr < (char *)region->start + region->size) {\n      |             ~~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n1 warning and 1 error generated.\nmake: *** [build/src/arena.o] Error 1"
+    },
+    {
+      "name": "engine",
+      "status": "PASS",
+      "elapsed_seconds": 12.815,
+      "artifact": "/Users/zhangsiqiang/Documents/Codex/2026-06-19/new-chat/outputs/thanhle-kickama-15-health-retry/frailbox/engine/build/trial-engine",
+      "output": "[ 11%] Building CXX object CMakeFiles/trial-engine.dir/main.cpp.o\n[ 22%] Building CXX object CMakeFiles/trial-engine.dir/core/math.cpp.o\n[ 33%] Building CXX object CMakeFiles/trial-engine.dir/core/ecs.cpp.o\n[ 44%] Building CXX object CMakeFiles/trial-engine.dir/dynamics/rigidbody.cpp.o\n[ 55%] Building CXX object CMakeFiles/trial-engine.dir/dynamics/constraint.cpp.o\n[ 66%] Building CXX object CMakeFiles/trial-engine.dir/collision/collision.cpp.o\n[ 77%] Building CXX object CMakeFiles/trial-engine.dir/Users/zhangsiqiang/Documents/Codex/2026-06-19/new-chat/outputs/thanhle-kickama-15-health-retry/frailbox/wat.cpp.o\n[ 88%] Building CXX object CMakeFiles/trial-engine.dir/Users/zhangsiqiang/Documents/Codex/2026-06-19/new-chat/outputs/thanhle-kickama-15-health-retry/frailbox/engine.cpp.o\n[100%] Linking CXX executable trial-engine\n[100%] Built target trial-engine"
+    },
+    {
+      "name": "compliance",
+      "status": "PASS",
+      "elapsed_seconds": 2.613,
+      "artifact": "/Users/zhangsiqiang/Documents/Codex/2026-06-19/new-chat/outputs/thanhle-kickama-15-health-retry/compliance/build",
+      "output": "\u6ce8: ComplianceAuditor.java\u4f7f\u7528\u6216\u8986\u76d6\u4e86\u5df2\u8fc7\u65f6\u7684 API\u3002\n\u6ce8: \u6709\u5173\u8be6\u7ec6\u4fe1\u606f, \u8bf7\u4f7f\u7528 -Xlint:deprecation \u91cd\u65b0\u7f16\u8bd1\u3002"
+    },
+    {
+      "name": "v2-market-stream",
+      "status": "PASS",
+      "elapsed_seconds": 0.16,
+      "artifact": null,
+      "output": "Syntax OK"
+    },
+    {
+      "name": "nfc-scanner",
+      "status": "FAIL",
+      "elapsed_seconds": 0,
+      "artifact": null,
+      "output": "Command not found: [Errno 2] No such file or directory: 'luac'"
+    },
+    {
+      "name": "openapi-haskell",
+      "status": "FAIL",
+      "elapsed_seconds": 0,
+      "artifact": null,
+      "output": "Command not found: [Errno 2] No such file or directory: 'ghc'"
+    },
+    {
+      "name": "openapi-tools",
+      "status": "FAIL",
+      "elapsed_seconds": 0,
+      "artifact": null,
+      "output": "Command not found: [Errno 2] No such file or directory: 'luac'"
+    }
+  ],
+  "pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic/build-94e0fb08.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging."
+}

docs/OPERATIONS.md

@@ -37,6 +37,30 @@ The health check returns a 200 OK response with a JSON body:
 }
 ```
 
+### HTTP Probe Retry and Circuit Breaker
+
+`tools/health_check.py` can retry transient HTTP probe failures and avoid
+hammering endpoints that keep failing.
+
+```sh
+python3 tools/health_check.py --service backend \
+  --max-retries 3 \
+  --backoff-factor 2 \
+  --circuit-threshold 4 \
+  --circuit-cooldown 60
+```
+
+Retry delays use exponential backoff:
+
+```text
+delay = base_delay * (backoff_factor ^ attempt)
+```
+
+The default base delay is 1 second. A circuit opens after the configured number
+of consecutive HTTP probe failures and stays open until the cooldown expires.
+JSON and text output include summary counts for OK, WARNING, CRITICAL, and open
+circuits.
+
 ### Prometheus Metrics
 
 Each service exposes Prometheus metrics at `/metrics` on the same port as the

tests/test_health_check_retry_circuit.py

@@ -0,0 +1,124 @@
+import sys
+import unittest
+from pathlib import Path
+from typing import List
+
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from tools import health_check  # noqa: E402
+
+
+class HealthCheckRetryCircuitTest(unittest.TestCase):
+    def test_http_probe_retries_until_success(self) -> None:
+        attempts = 0
+
+        def request_once(host: str, port: int, path: str, timeout: int) -> tuple[int, str]:
+            nonlocal attempts
+            attempts += 1
+            if attempts == 1:
+                raise ConnectionRefusedError("not ready")
+            return 200, "ok"
+
+        status, detail, code = health_check.check_http_service(
+            "localhost",
+            8080,
+            "/health",
+            5,
+            max_retries=2,
+            backoff_factor=0,
+            request_once=request_once,
+            sleep=lambda seconds: None,
+        )
+
+        self.assertEqual(status, "OK")
+        self.assertEqual(code, 200)
+        self.assertEqual(attempts, 2)
+        self.assertIn("after 2 attempts", detail)
+
+    def test_retry_backoff_uses_exponential_schedule(self) -> None:
+        sleeps: List[float] = []
+
+        def request_once(host: str, port: int, path: str, timeout: int) -> tuple[int, str]:
+            raise TimeoutError("timeout")
+
+        status, detail, code = health_check.check_http_service(
+            "localhost",
+            8080,
+            "/health",
+            5,
+            max_retries=2,
+            backoff_factor=2,
+            backoff_base_delay=0.5,
+            request_once=request_once,
+            sleep=sleeps.append,
+        )
+
+        self.assertEqual(status, "CRITICAL")
+        self.assertEqual(code, 0)
+        self.assertIn("timeout", detail)
+        self.assertEqual(sleeps, [0.5, 1.0])
+
+    def test_circuit_breaker_opens_after_threshold(self) -> None:
+        breaker = health_check.CircuitBreaker(threshold=2, cooldown_seconds=30, clock=lambda: 100.0)
+
+        breaker.record_failure()
+        self.assertTrue(breaker.allow_request())
+
+        breaker.record_failure()
+        self.assertFalse(breaker.allow_request())
+
+    def test_circuit_breaker_resets_after_cooldown(self) -> None:
+        now = 100.0
+
+        def clock() -> float:
+            return now
+
+        breaker = health_check.CircuitBreaker(threshold=1, cooldown_seconds=30, clock=clock)
+        breaker.record_failure()
+        self.assertFalse(breaker.allow_request())
+
+        now = 131.0
+        self.assertTrue(breaker.allow_request())
+
+    def test_health_summary_counts_statuses_and_circuit_breakers(self) -> None:
+        results = {
+            "services": {
+                "backend": {"status": "OK"},
+                "market": {"status": "WARNING", "circuit": "closed"},
+                "frailbox": {"status": "CRITICAL", "circuit": "open"},
+            },
+            "infrastructure": {"redis": {"status": "CRITICAL"}},
+            "system": {"disk": {"status": "OK"}, "memory": {"status": "WARNING"}},
+        }
+
+        self.assertEqual(
+            health_check.summarize_results(results),
+            {
+                "ok": 2,
+                "warning": 2,
+                "critical": 2,
+                "circuit_open": 1,
+            },
+        )
+
+    def test_retry_flags_are_parsed(self) -> None:
+        args = health_check.parse_args(
+            [
+                "--max-retries",
+                "3",
+                "--backoff-factor",
+                "1.5",
+                "--circuit-threshold",
+                "4",
+            ]
+        )
+
+        self.assertEqual(args.max_retries, 3)
+        self.assertEqual(args.backoff_factor, 1.5)
+        self.assertEqual(args.circuit_threshold, 4)
+
+
+if __name__ == "__main__":
+    unittest.main()