diff --git a/lib/env.sh b/lib/env.sh index 80e4ee0..15befa5 100755 --- a/lib/env.sh +++ b/lib/env.sh @@ -27,27 +27,12 @@ ingo_load_env() { : "${INGO_ROLE:=all}" : "${INGO_HTTP_CONNECT_TIMEOUT:=5}" : "${INGO_HTTP_READ_TIMEOUT:=30}" - - if [ -n "${INGO_HTTP_RETRY_ATTEMPTS:-}" ]; then - : - elif [ -n "${INGO_HTTP_RETRY_MAX:-}" ]; then - INGO_HTTP_RETRY_ATTEMPTS="$INGO_HTTP_RETRY_MAX" - else - INGO_HTTP_RETRY_ATTEMPTS="2" - fi - - if [ -n "${INGO_HTTP_RETRY_BACKOFF_MIN:-}" ]; then - : - elif [ -n "${INGO_HTTP_RETRY_BACKOFF:-}" ]; then - INGO_HTTP_RETRY_BACKOFF_MIN="$INGO_HTTP_RETRY_BACKOFF" - else - INGO_HTTP_RETRY_BACKOFF_MIN="1" - fi - - : "${INGO_HTTP_RETRY_BACKOFF_MAX:=8}" + : "${INGO_HTTP_RETRY_MAX:=2}" + : "${INGO_HTTP_RETRY_BACKOFF:=1}" + : "${INGO_HTTP_RETRY_BACKOFF_MIN:=$INGO_HTTP_RETRY_BACKOFF}" + : "${INGO_HTTP_RETRY_BACKOFF_MAX:=30}" : "${INGO_HTTP_RETRY_BACKOFF_FACTOR:=2}" - INGO_HTTP_RETRY_MAX="$INGO_HTTP_RETRY_ATTEMPTS" - INGO_HTTP_RETRY_BACKOFF="$INGO_HTTP_RETRY_BACKOFF_MIN" + : "${INGO_HTTP_RETRY_AFTER_MAX:=$INGO_HTTP_RETRY_BACKOFF_MAX}" } ingo_require_bin() { diff --git a/lib/http.sh b/lib/http.sh index a43f59f..6a0dc3a 100644 --- a/lib/http.sh +++ b/lib/http.sh @@ -2,12 +2,160 @@ set -euo pipefail +# Retriable HTTP status matrix: +# 408 Request Timeout – transient server-side timeout +# 429 Too Many Requests – rate limited; honor Retry-After when present +# 500 Internal Server Error – transient upstream fault +# 502 Bad Gateway – transient upstream fault +# 503 Service Unavailable – transient upstream fault; honor Retry-After +# 504 Gateway Timeout – transient upstream timeout +ingo_http_is_retriable_status() { + case "$1" in + 408|429|500|502|503|504) return 0 ;; + *) return 1 ;; + esac +} + +# Retriable curl exit code matrix: +# 7 CURLE_COULDNT_CONNECT – connection refused / host unreachable +# 18 CURLE_PARTIAL_FILE – transfer interrupted mid-stream +# 28 CURLE_OPERATION_TIMEDOUT – request exceeded --max-time +# 52 CURLE_GOT_NOTHING – server closed connection without response +# 56 CURLE_RECV_ERROR – network receive error +ingo_http_is_retriable_exit_code() { + case "$1" in + 7|18|28|52|56) return 0 ;; + *) return 1 ;; + esac +} + +ingo_http_extract_status_code() { + local headers_file="$1" + grep -E '^HTTP/[0-9.]+ [0-9]{3}' "$headers_file" | tail -n 1 | awk '{print $2}' +} + +ingo_http_extract_retry_after_raw() { + local headers_file="$1" + awk -F': *' ' + /^HTTP\// {retry_after=""} + /^[Rr]etry-[Aa]fter:/ { + value=$2 + sub(/\r$/, "", value) + retry_after=value + } + END {print retry_after} + ' "$headers_file" +} + +ingo_http_to_epoch() { + local value="$1" + if date -u -d "$value" +%s >/dev/null 2>&1; then + date -u -d "$value" +%s + return 0 + fi + if date -u -j -f "%a, %d %b %Y %H:%M:%S GMT" "$value" +%s >/dev/null 2>&1; then + date -u -j -f "%a, %d %b %Y %H:%M:%S GMT" "$value" +%s + return 0 + fi + return 1 +} + +ingo_http_retry_after_seconds() { + local headers_file="$1" + local raw now epoch + + raw="$(ingo_http_extract_retry_after_raw "$headers_file")" + if [ -z "$raw" ]; then + return 1 + fi + + if [[ "$raw" =~ ^[0-9]+$ ]]; then + printf "%s\n" "$raw" + return 0 + fi + + if ! epoch="$(ingo_http_to_epoch "$raw")"; then + return 1 + fi + now="$(date -u +%s)" + if [ "$epoch" -le "$now" ]; then + printf "0\n" + else + printf "%s\n" $((epoch - now)) + fi +} + +ingo_http_computed_backoff() { + local retry_index="$1" + local delay="$INGO_HTTP_RETRY_BACKOFF_MIN" + local i=0 + + while [ "$i" -lt "$retry_index" ]; do + delay=$((delay * INGO_HTTP_RETRY_BACKOFF_FACTOR)) + i=$((i + 1)) + done + + if [ "$delay" -gt "$INGO_HTTP_RETRY_BACKOFF_MAX" ]; then + delay="$INGO_HTTP_RETRY_BACKOFF_MAX" + fi + printf "%s\n" "$delay" +} + +ingo_http_retry_delay_seconds() { + local retry_index="$1" + local headers_file="$2" + local delay + + if delay="$(ingo_http_retry_after_seconds "$headers_file")"; then + if [ "$delay" -gt "$INGO_HTTP_RETRY_AFTER_MAX" ]; then + delay="$INGO_HTTP_RETRY_AFTER_MAX" + fi + if [ "$delay" -gt "$INGO_HTTP_RETRY_BACKOFF_MAX" ]; then + delay="$INGO_HTTP_RETRY_BACKOFF_MAX" + fi + printf "%s\n" "$delay" + return 0 + fi + + ingo_http_computed_backoff "$retry_index" +} + ingo_http_curl() { - curl \ - --connect-timeout "$INGO_HTTP_CONNECT_TIMEOUT" \ - --max-time "$INGO_HTTP_READ_TIMEOUT" \ - --retry "$INGO_HTTP_RETRY_ATTEMPTS" \ - --retry-delay "$INGO_HTTP_RETRY_BACKOFF_MIN" \ - --retry-connrefused \ - "$@" + local attempt=0 + local status curl_rc should_retry delay headers_file body_file err_file + + while :; do + headers_file="$(mktemp)" + body_file="$(mktemp)" + err_file="$(mktemp)" + set +e + curl \ + --connect-timeout "$INGO_HTTP_CONNECT_TIMEOUT" \ + --max-time "$INGO_HTTP_READ_TIMEOUT" \ + -D "$headers_file" \ + "$@" >"$body_file" 2>"$err_file" + curl_rc=$? + set -e + + status="$(ingo_http_extract_status_code "$headers_file" || true)" + should_retry=1 + if [ -n "$status" ] && ingo_http_is_retriable_status "$status"; then + should_retry=0 + elif [ "$curl_rc" -ne 0 ] && ingo_http_is_retriable_exit_code "$curl_rc"; then + should_retry=0 + fi + + if [ "$should_retry" -eq 0 ] && [ "$attempt" -lt "$INGO_HTTP_RETRY_MAX" ]; then + delay="$(ingo_http_retry_delay_seconds "$attempt" "$headers_file")" + rm -f "$headers_file" "$body_file" "$err_file" + sleep "$delay" + attempt=$((attempt + 1)) + continue + fi + + cat "$body_file" + cat "$err_file" >&2 + rm -f "$headers_file" "$body_file" "$err_file" + return "$curl_rc" + done } diff --git a/tests/http_wrapper_test.sh b/tests/http_wrapper_test.sh index b304593..2ced93a 100755 --- a/tests/http_wrapper_test.sh +++ b/tests/http_wrapper_test.sh @@ -38,61 +38,116 @@ assert_contains() { fi } +assert_not_contains() { + local haystack="$1" + local needle="$2" + local msg="$3" + if printf "%s" "$haystack" | grep -F -- "$needle" >/dev/null; then + fail "$msg (unexpected '$needle')" + fi +} + +MOCK_CURL_CALLS=0 +MOCK_CURL_CODES=() +MOCK_CURL_RC=() +MOCK_CURL_BODIES=() +MOCK_CURL_RETRY_AFTER=() +MOCK_CURL_LOG_FILE="" + +reset_http_mocks() { + MOCK_CURL_CALLS=0 + MOCK_CURL_CODES=() + MOCK_CURL_RC=() + MOCK_CURL_BODIES=() + MOCK_CURL_RETRY_AFTER=() + MOCK_CURL_LOG_FILE="" +} + +curl() { + local idx="$MOCK_CURL_CALLS" + local status="${MOCK_CURL_CODES[$idx]:-200}" + local rc="${MOCK_CURL_RC[$idx]:-0}" + local body="${MOCK_CURL_BODIES[$idx]:-body-$idx}" + local retry_after="${MOCK_CURL_RETRY_AFTER[$idx]:-}" + local headers_file="" + local arg + + while [ "$#" -gt 0 ]; do + arg="$1" + if [ "$arg" = "-D" ]; then + shift + headers_file="$1" + fi + if [ -n "$MOCK_CURL_LOG_FILE" ]; then + printf "%s\n" "$arg" >> "$MOCK_CURL_LOG_FILE" + fi + shift + done + + if [ -n "$headers_file" ]; then + { + printf "HTTP/1.1 %s Mock\r\n" "$status" + if [ -n "$retry_after" ]; then + printf "Retry-After: %s\r\n" "$retry_after" + fi + printf "\r\n" + } > "$headers_file" + fi + + printf "%s" "$body" + MOCK_CURL_CALLS=$((MOCK_CURL_CALLS + 1)) + return "$rc" +} + test_http_defaults() { - unset INGO_HTTP_CONNECT_TIMEOUT INGO_HTTP_READ_TIMEOUT INGO_HTTP_RETRY_ATTEMPTS INGO_HTTP_RETRY_BACKOFF_MIN INGO_HTTP_RETRY_BACKOFF_MAX INGO_HTTP_RETRY_BACKOFF_FACTOR INGO_HTTP_RETRY_MAX INGO_HTTP_RETRY_BACKOFF + unset INGO_HTTP_CONNECT_TIMEOUT INGO_HTTP_READ_TIMEOUT INGO_HTTP_RETRY_MAX INGO_HTTP_RETRY_BACKOFF + unset INGO_HTTP_RETRY_BACKOFF_MIN INGO_HTTP_RETRY_BACKOFF_MAX INGO_HTTP_RETRY_BACKOFF_FACTOR INGO_HTTP_RETRY_AFTER_MAX ingo_load_env assert_eq "$INGO_HTTP_CONNECT_TIMEOUT" "5" "default connect timeout" assert_eq "$INGO_HTTP_READ_TIMEOUT" "30" "default read timeout" - assert_eq "$INGO_HTTP_RETRY_ATTEMPTS" "2" "default retry attempts" - assert_eq "$INGO_HTTP_RETRY_BACKOFF_MIN" "1" "default retry backoff min" - assert_eq "$INGO_HTTP_RETRY_BACKOFF_MAX" "8" "default retry backoff max" - assert_eq "$INGO_HTTP_RETRY_BACKOFF_FACTOR" "2" "default retry backoff factor" assert_eq "$INGO_HTTP_RETRY_MAX" "2" "default retry max" - assert_eq "$INGO_HTTP_RETRY_BACKOFF" "1" "default retry backoff" + assert_eq "$INGO_HTTP_RETRY_BACKOFF" "1" "default legacy retry backoff" + assert_eq "$INGO_HTTP_RETRY_BACKOFF_MIN" "1" "default retry min" + assert_eq "$INGO_HTTP_RETRY_BACKOFF_MAX" "30" "default retry max sleep" + assert_eq "$INGO_HTTP_RETRY_BACKOFF_FACTOR" "2" "default retry factor" + assert_eq "$INGO_HTTP_RETRY_AFTER_MAX" "30" "default retry-after cap" } test_http_overrides() { INGO_HTTP_CONNECT_TIMEOUT="9" INGO_HTTP_READ_TIMEOUT="44" - INGO_HTTP_RETRY_ATTEMPTS="7" - INGO_HTTP_RETRY_BACKOFF_MIN="3" + INGO_HTTP_RETRY_MAX="7" + INGO_HTTP_RETRY_BACKOFF="3" + INGO_HTTP_RETRY_BACKOFF_MIN="4" INGO_HTTP_RETRY_BACKOFF_MAX="21" - INGO_HTTP_RETRY_BACKOFF_FACTOR="4" + INGO_HTTP_RETRY_BACKOFF_FACTOR="5" + INGO_HTTP_RETRY_AFTER_MAX="13" ingo_load_env assert_eq "$INGO_HTTP_CONNECT_TIMEOUT" "9" "override connect timeout" assert_eq "$INGO_HTTP_READ_TIMEOUT" "44" "override read timeout" - assert_eq "$INGO_HTTP_RETRY_ATTEMPTS" "7" "override retry attempts" - assert_eq "$INGO_HTTP_RETRY_BACKOFF_MIN" "3" "override retry backoff min" - assert_eq "$INGO_HTTP_RETRY_BACKOFF_MAX" "21" "override retry backoff max" - assert_eq "$INGO_HTTP_RETRY_BACKOFF_FACTOR" "4" "override retry backoff factor" assert_eq "$INGO_HTTP_RETRY_MAX" "7" "override retry max" - assert_eq "$INGO_HTTP_RETRY_BACKOFF" "3" "override retry backoff" -} - -test_http_legacy_alias_overrides() { - unset INGO_HTTP_RETRY_ATTEMPTS INGO_HTTP_RETRY_BACKOFF_MIN INGO_HTTP_RETRY_BACKOFF_MAX INGO_HTTP_RETRY_BACKOFF_FACTOR - INGO_HTTP_RETRY_MAX="6" - INGO_HTTP_RETRY_BACKOFF="4" - ingo_load_env - assert_eq "$INGO_HTTP_RETRY_ATTEMPTS" "6" "legacy retry max maps to attempts" - assert_eq "$INGO_HTTP_RETRY_BACKOFF_MIN" "4" "legacy retry backoff maps to min" - assert_eq "$INGO_HTTP_RETRY_BACKOFF_MAX" "8" "legacy override keeps default max" - assert_eq "$INGO_HTTP_RETRY_BACKOFF_FACTOR" "2" "legacy override keeps default factor" + assert_eq "$INGO_HTTP_RETRY_BACKOFF" "3" "override retry legacy backoff" + assert_eq "$INGO_HTTP_RETRY_BACKOFF_MIN" "4" "override retry min" + assert_eq "$INGO_HTTP_RETRY_BACKOFF_MAX" "21" "override retry max sleep" + assert_eq "$INGO_HTTP_RETRY_BACKOFF_FACTOR" "5" "override retry factor" + assert_eq "$INGO_HTTP_RETRY_AFTER_MAX" "13" "override retry-after cap" } test_http_wrapper_curl_flags() { + reset_http_mocks + local args_file args_file="$(mktemp)" - - curl() { - printf "%s\n" "$@" > "$args_file" - } + MOCK_CURL_LOG_FILE="$args_file" INGO_HTTP_CONNECT_TIMEOUT="11" INGO_HTTP_READ_TIMEOUT="22" - INGO_HTTP_RETRY_ATTEMPTS="4" - INGO_HTTP_RETRY_BACKOFF_MIN="2" - ingo_http_curl -fsSL "https://example.com/a.pdf" -o "/tmp/out.pdf" + INGO_HTTP_RETRY_MAX="0" + INGO_HTTP_RETRY_BACKOFF_MIN="1" + INGO_HTTP_RETRY_BACKOFF_MAX="2" + INGO_HTTP_RETRY_BACKOFF_FACTOR="2" + INGO_HTTP_RETRY_AFTER_MAX="2" + ingo_http_curl -fsSL "https://example.com/a.pdf" -o "/tmp/out.pdf" >/dev/null local args args="$(cat "$args_file")" @@ -100,14 +155,102 @@ test_http_wrapper_curl_flags() { assert_contains "$args" "11" "wrapper uses connect-timeout value" assert_contains "$args" "--max-time" "wrapper adds max-time flag" assert_contains "$args" "22" "wrapper uses max-time value" - assert_contains "$args" "--retry" "wrapper adds retry flag" - assert_contains "$args" "4" "wrapper uses retry value" - assert_contains "$args" "--retry-delay" "wrapper adds retry-delay flag" - assert_contains "$args" "2" "wrapper uses retry-delay value" - assert_contains "$args" "--retry-connrefused" "wrapper adds retry-connrefused flag" + assert_contains "$args" "-D" "wrapper collects response headers" + assert_not_contains "$args" "--retry" "wrapper does not rely on curl retry flags" + assert_not_contains "$args" "--retry-delay" "wrapper does not rely on curl retry-delay flag" assert_contains "$args" "https://example.com/a.pdf" "wrapper keeps original curl args" } +test_http_retry_after_precedence_and_limits() { + reset_http_mocks + + local sleep_file + sleep_file="$(mktemp)" + sleep() { + printf "%s\n" "$1" >> "$sleep_file" + } + + INGO_HTTP_RETRY_MAX="2" + INGO_HTTP_RETRY_BACKOFF_MIN="1" + INGO_HTTP_RETRY_BACKOFF_MAX="30" + INGO_HTTP_RETRY_BACKOFF_FACTOR="2" + INGO_HTTP_RETRY_AFTER_MAX="10" + + MOCK_CURL_CODES=(429 200) + MOCK_CURL_RC=(0 0) + MOCK_CURL_BODIES=(first second) + MOCK_CURL_RETRY_AFTER=(7 "") + + local out_file out + out_file="$(mktemp)" + ingo_http_curl -sS "https://example.com/rate" > "$out_file" + out="$(cat "$out_file")" + + assert_eq "$MOCK_CURL_CALLS" "2" "429 retried once" + assert_eq "$(cat "$sleep_file")" "7" "retry-after takes precedence over computed backoff" + assert_eq "$out" "second" "final attempt body is returned" +} + +test_http_retry_after_fallback_to_computed_backoff() { + reset_http_mocks + + local sleep_file + sleep_file="$(mktemp)" + sleep() { + printf "%s\n" "$1" >> "$sleep_file" + } + + INGO_HTTP_RETRY_MAX="2" + INGO_HTTP_RETRY_BACKOFF_MIN="4" + INGO_HTTP_RETRY_BACKOFF_MAX="5" + INGO_HTTP_RETRY_BACKOFF_FACTOR="3" + INGO_HTTP_RETRY_AFTER_MAX="20" + + MOCK_CURL_CODES=(429 503 200) + MOCK_CURL_RC=(0 0 0) + MOCK_CURL_BODIES=(first second third) + MOCK_CURL_RETRY_AFTER=(invalid "" "") + + local out_file out + out_file="$(mktemp)" + ingo_http_curl -sS "https://example.com/flaky" > "$out_file" + out="$(cat "$out_file")" + + assert_eq "$MOCK_CURL_CALLS" "3" "retriable statuses retried up to limit" + assert_eq "$(tr '\n' ',' < "$sleep_file" | sed 's/,$//')" "4,5" "computed backoff used and clamped" + assert_eq "$out" "third" "final success body returned" +} + +test_http_non_retriable_4xx_fails_fast() { + reset_http_mocks + + local sleep_file + sleep_file="$(mktemp)" + sleep() { + printf "%s\n" "$1" >> "$sleep_file" + } + + INGO_HTTP_RETRY_MAX="3" + INGO_HTTP_RETRY_BACKOFF_MIN="1" + INGO_HTTP_RETRY_BACKOFF_MAX="9" + INGO_HTTP_RETRY_BACKOFF_FACTOR="2" + INGO_HTTP_RETRY_AFTER_MAX="9" + + MOCK_CURL_CODES=(404) + MOCK_CURL_RC=(22) + MOCK_CURL_BODIES=(missing) + MOCK_CURL_RETRY_AFTER=("") + + if ingo_http_curl -fsS "https://example.com/missing" >/dev/null 2>&1; then + fail "404 should fail" + fi + + assert_eq "$MOCK_CURL_CALLS" "1" "non-retriable 4xx should not retry" + if [ -s "$sleep_file" ]; then + fail "non-retriable 4xx should not sleep" + fi +} + test_fetch_uses_http_wrapper() { local inbox args_file out inbox="$(mktemp -d)" @@ -164,8 +307,10 @@ test_embed_and_query_use_http_wrapper() { main() { test_http_defaults test_http_overrides - test_http_legacy_alias_overrides test_http_wrapper_curl_flags + test_http_retry_after_precedence_and_limits + test_http_retry_after_fallback_to_computed_backoff + test_http_non_retriable_4xx_fails_fast test_fetch_uses_http_wrapper test_embed_and_query_use_http_wrapper echo "ok"