From 1460d3a1539cc3a310564ef6ac035f63ca48af10 Mon Sep 17 00:00:00 2001 From: fullsend-code <278716306+fullsend-ai-coder[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 09:06:21 +0000 Subject: [PATCH 1/2] KONFLUX-14156(#945): add curl retry to OTP server calls Under high concurrency (200 concurrent RPM builds), the OTP server DNS name intermittently fails to resolve, causing curl to exit with code 6 and the build step to fail. This was observed as a 1% failure rate (2/200) on stone-stg-rh01. Add --retry 3 --retry-all-errors --retry-delay 1 to the curl calls that POST SSH keys to the OTP server /store-key endpoint in all three provisioning scripts: - deploy/operator/provision-shared-host.sh - deploy/operator/provision-host-windows.sh - deploy/operator/provision-host-macos.sh This matches the existing retry pattern already used for the otelcol-contrib download in provision-shared-host.sh. Closes #945 Authored-By: Claude Code --- deploy/operator/provision-host-macos.sh | 2 +- deploy/operator/provision-host-windows.sh | 2 +- deploy/operator/provision-shared-host.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/operator/provision-host-macos.sh b/deploy/operator/provision-host-macos.sh index ff1ffc689..3b6ae0c0d 100644 --- a/deploy/operator/provision-host-macos.sh +++ b/deploy/operator/provision-host-macos.sh @@ -46,7 +46,7 @@ DIR=$(echo "/Users/${USERNAME}" | base64 -w 0) if [ -e "/tls/tls.crt" ]; then echo "{message: \"Creating secret file using TLS certificate...\", level: \"INFO\"}" KEY=$(cat id_rsa) - if ! otp_raw=$(curl --fail --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then + if ! otp_raw=$(curl --fail --retry 3 --retry-all-errors --retry-delay 1 --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then echo "{message: \"Failed to store SSH key in OTP server. Please, retry build in a few minutes, and if problem persists, please report it as an MPC bug.\", level: \"ERROR\"}" >&2 exit 1 fi diff --git a/deploy/operator/provision-host-windows.sh b/deploy/operator/provision-host-windows.sh index 73a4df964..5c27a5884 100644 --- a/deploy/operator/provision-host-windows.sh +++ b/deploy/operator/provision-host-windows.sh @@ -47,7 +47,7 @@ DIR=$(echo 'C:\\Users\\'"${USERNAME}" | base64 -w 0) if [ -e "/tls/tls.crt" ]; then echo "{message: \"Creating secret file using TLS certificate...\", level: \"INFO\"}" KEY=$(cat id_rsa) - if ! otp_raw=$(curl --fail --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then + if ! otp_raw=$(curl --fail --retry 3 --retry-all-errors --retry-delay 1 --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then echo "{message: \"Failed to store SSH key in OTP server. Please, retry build in a few minutes, and if problem persists, please report it as an MPC bug.\", level: \"ERROR\"}" >&2 exit 1 fi diff --git a/deploy/operator/provision-shared-host.sh b/deploy/operator/provision-shared-host.sh index 7b69e799d..44f8f2d6c 100755 --- a/deploy/operator/provision-shared-host.sh +++ b/deploy/operator/provision-shared-host.sh @@ -209,7 +209,7 @@ DIR=$(echo /home/"$USERNAME" | base64 -w 0) if [ -e "/tls/tls.crt" ]; then echo "{message: \"Creating secret file using TLS certificate...\", level: \"INFO\"}" KEY=$(cat id_rsa) - if ! otp_raw=$(curl --fail --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then + if ! otp_raw=$(curl --fail --retry 3 --retry-all-errors --retry-delay 1 --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then echo "{message: \"Failed to store SSH key in OTP server. Please, retry build in a few minutes, and if problem persists, please report it as an MPC bug.\", level: \"ERROR\"}" >&2 exit 1 fi From b6556111d46da5b987a57eda48532489d4b198ce Mon Sep 17 00:00:00 2001 From: fullsend-fix <278716306+fullsend-ai-coder[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:58:07 +0000 Subject: [PATCH 2/2] fix: add connect-timeout and max-time to OTP curl calls Add --connect-timeout 5 --max-time 30 to the curl /store-key POST calls in all three provisioning scripts, addressing the pattern-inconsistency review comment on PR #946. Addresses review feedback on #946 Authored-By: Claude Code --- deploy/operator/provision-host-macos.sh | 2 +- deploy/operator/provision-host-windows.sh | 2 +- deploy/operator/provision-shared-host.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/operator/provision-host-macos.sh b/deploy/operator/provision-host-macos.sh index 3b6ae0c0d..c72a258b3 100644 --- a/deploy/operator/provision-host-macos.sh +++ b/deploy/operator/provision-host-macos.sh @@ -46,7 +46,7 @@ DIR=$(echo "/Users/${USERNAME}" | base64 -w 0) if [ -e "/tls/tls.crt" ]; then echo "{message: \"Creating secret file using TLS certificate...\", level: \"INFO\"}" KEY=$(cat id_rsa) - if ! otp_raw=$(curl --fail --retry 3 --retry-all-errors --retry-delay 1 --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then + if ! otp_raw=$(curl --fail --connect-timeout 5 --max-time 30 --retry 3 --retry-all-errors --retry-delay 1 --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then echo "{message: \"Failed to store SSH key in OTP server. Please, retry build in a few minutes, and if problem persists, please report it as an MPC bug.\", level: \"ERROR\"}" >&2 exit 1 fi diff --git a/deploy/operator/provision-host-windows.sh b/deploy/operator/provision-host-windows.sh index 5c27a5884..c39d70ae0 100644 --- a/deploy/operator/provision-host-windows.sh +++ b/deploy/operator/provision-host-windows.sh @@ -47,7 +47,7 @@ DIR=$(echo 'C:\\Users\\'"${USERNAME}" | base64 -w 0) if [ -e "/tls/tls.crt" ]; then echo "{message: \"Creating secret file using TLS certificate...\", level: \"INFO\"}" KEY=$(cat id_rsa) - if ! otp_raw=$(curl --fail --retry 3 --retry-all-errors --retry-delay 1 --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then + if ! otp_raw=$(curl --fail --connect-timeout 5 --max-time 30 --retry 3 --retry-all-errors --retry-delay 1 --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then echo "{message: \"Failed to store SSH key in OTP server. Please, retry build in a few minutes, and if problem persists, please report it as an MPC bug.\", level: \"ERROR\"}" >&2 exit 1 fi diff --git a/deploy/operator/provision-shared-host.sh b/deploy/operator/provision-shared-host.sh index 44f8f2d6c..372f6b6f9 100755 --- a/deploy/operator/provision-shared-host.sh +++ b/deploy/operator/provision-shared-host.sh @@ -209,7 +209,7 @@ DIR=$(echo /home/"$USERNAME" | base64 -w 0) if [ -e "/tls/tls.crt" ]; then echo "{message: \"Creating secret file using TLS certificate...\", level: \"INFO\"}" KEY=$(cat id_rsa) - if ! otp_raw=$(curl --fail --retry 3 --retry-all-errors --retry-delay 1 --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then + if ! otp_raw=$(curl --fail --connect-timeout 5 --max-time 30 --retry 3 --retry-all-errors --retry-delay 1 --cacert /tls/tls.crt -XPOST -d "$KEY" https://multi-platform-otp-server.multi-platform-controller.svc.cluster.local/store-key); then echo "{message: \"Failed to store SSH key in OTP server. Please, retry build in a few minutes, and if problem persists, please report it as an MPC bug.\", level: \"ERROR\"}" >&2 exit 1 fi