-
Notifications
You must be signed in to change notification settings - Fork 31
Expand file tree
/
Copy pathlibcheckout
More file actions
executable file
·473 lines (390 loc) · 13.1 KB
/
libcheckout
File metadata and controls
executable file
·473 lines (390 loc) · 13.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
function checkout() {
local use_cache=${1:-""}
if [ -z "${SEMAPHORE_GIT_BRANCH}" ] || [ -z "${SEMAPHORE_GIT_URL}" ] || [ -z "${SEMAPHORE_GIT_DIR}" ] || [ -z "${SEMAPHORE_GIT_SHA}" ]; then
checkout::validation_message
return 1
fi
rm -rf "$SEMAPHORE_GIT_DIR"
if [ "$use_cache" = "--use-cache" ]; then
echo "[Experimental stability] Using cached Git repository."
checkout::use_cache
else
if [ -n "${SEMAPHORE_GIT_REF_TYPE:-}" ]; then
checkout::refbased
else
checkout::shallow
fi
exit_code=$?
if [ "$exit_code" -ne "0" ]; then
checkout::metric 0
fi
return $exit_code
fi
}
function checkout::validation_message() {
echo "[CHECKOUT ERROR] Some of these variables are unset:"
echo "SEMAPHORE_GIT_URL=$SEMAPHORE_GIT_URL"
echo "SEMAPHORE_GIT_DIR=$SEMAPHORE_GIT_DIR"
echo "SEMAPHORE_GIT_BRANCH=$SEMAPHORE_GIT_BRANCH"
echo "SEMAPHORE_GIT_SHA=$SEMAPHORE_GIT_SHA"
}
function checkout::use_cache() {
export CACHE_FAIL_ON_ERROR="true"
if ! cache restore git-cache-; then
echo "Failed to restore from the cache"
rm -rf "${SEMAPHORE_GIT_DIR}"
fi
if [ -d "$HOME/$SEMAPHORE_GIT_DIR" ]; then
checkout::fetch
else
checkout::clone
fi
# SC2181: checking the last command's exit code
# with $? simplifies the control flow of this function.
# shellcheck disable=SC2181
if [ "$?" -eq "0" ]; then
checkout::cache_store
else
return 1
fi
}
function checkout::fetch() {
echo "Restored cache"
cd "${HOME}/${SEMAPHORE_GIT_DIR}" || exit
git remote prune origin
retry git fetch --tags origin
checkout::switch_revision
}
function checkout::clone() {
checkout::git_clone "${SEMAPHORE_GIT_URL}" "${SEMAPHORE_GIT_DIR}"
cd "${SEMAPHORE_GIT_DIR}" || exit
checkout::switch_revision
}
function checkout::reset_to_sha {
if checkout::checkrevision; then
git reset --hard "${SEMAPHORE_GIT_SHA}" 2>/dev/null
checkout::metric "$(du -s . | awk '{ print $1 }')"
return 0
else
return 1
fi
}
function checkout::switch_revision() {
if [ "${SEMAPHORE_GIT_REF_TYPE:-""}" = "pull-request" ]; then
if ! retry git fetch origin +"${SEMAPHORE_GIT_REF}": 2>/dev/null; then
echo "Reference: ${SEMAPHORE_GIT_REF} not found .... Exiting"
return 1
else
checkout::reset_to_sha
fi
elif [ "${SEMAPHORE_GIT_REF_TYPE:-''}" = "tag" ]; then
if ! git checkout -qf "${SEMAPHORE_GIT_TAG_NAME}"; then
echo "Release $SEMAPHORE_GIT_TAG_NAME not found .... Exiting"
return 1
else
checkout::metric "$(du -s . | awk '{ print $1 }')"
echo "HEAD is now at ${SEMAPHORE_GIT_SHA} Release ${SEMAPHORE_GIT_TAG_NAME}"
return 0
fi
else
local branch_origin="origin/$SEMAPHORE_GIT_BRANCH"
if [[ -n $(git show-ref "${SEMAPHORE_GIT_REF}") ]]; then
git checkout "${SEMAPHORE_GIT_BRANCH}"
else
git checkout -b "${SEMAPHORE_GIT_BRANCH}" -t "${branch_origin}";
fi
checkout::reset_to_sha
fi
}
function checkout::cache_store() {
# update cache if older then 72h -> 25920s
SEMAPHORE_GIT_CACHE_AGE="${SEMAPHORE_GIT_CACHE_AGE:-259200}"
local cache_key
local cache_age
cache_key="$(cache list 2>&1 | grep git-cache- || true )"
if [[ -n "$cache_key" ]];then
cache_key=$(echo "${cache_key}" | awk '{ print $1 }'| head -1)
fi
cache_age=$(echo "${cache_key}" | cut -d'-' -f8)
if [[ -n "$cache_age" ]] && [[ $cache_age =~ ^[0-9]+$ ]]; then
local diff=$(($(date +%s) - cache_age))
echo "diff: $diff"
if (( diff > SEMAPHORE_GIT_CACHE_AGE )); then
echo "Git cache outdated, refreshing..."
cd ..
checkout::cleanupcache
cache store "git-cache-$SEMAPHORE_JOB_ID-$(date +%s)" "${SEMAPHORE_GIT_DIR}"
cd "${SEMAPHORE_GIT_DIR}" || exit
else
echo "Git cache up-to-data."
fi
else
echo "No git cache... caching"
cd ..
cache store "git-cache-${SEMAPHORE_JOB_ID}-$(date +%s)" "${SEMAPHORE_GIT_DIR}"
cd "${SEMAPHORE_GIT_DIR}" || exit
fi
}
function checkout::cleanupcache {
if [ -z "${SEMAPHORE_GIT_CACHE_KEEP:-""}" ]; then
SEMAPHORE_GIT_CACHE_KEEP=0
fi
local k=$SEMAPHORE_GIT_CACHE_KEEP
local list=()
while IFS='' read -r line; do list+=("$line"); done < <(cache list 2>&1 | grep git-cache- | awk '{ print $1 }')
for i in "${list[@]:$k}"; do
cache delete "${i}"
done
}
function checkout::checkrevision {
if ! git rev-list HEAD.."${SEMAPHORE_GIT_SHA}" 2>/dev/null; then
echo "Revision: ${SEMAPHORE_GIT_SHA} not found .... Exiting"
return 1
fi
}
function checkout::branch_checkout {
if ! git checkout -f "$SEMAPHORE_GIT_BRANCH"; then
echo "Branch: ${SEMAPHORE_GIT_BRANCH} not found .... Exiting"
return 1
fi
}
function checkout::shallow() {
if [ -z "${SEMAPHORE_GIT_DEPTH:-""}" ]; then
SEMAPHORE_GIT_DEPTH=50
fi
echo "Performing shallow clone with depth: $SEMAPHORE_GIT_DEPTH"
if ! checkout::git_clone --depth "${SEMAPHORE_GIT_DEPTH}" -b "${SEMAPHORE_GIT_BRANCH}" "${SEMAPHORE_GIT_URL}" "${SEMAPHORE_GIT_DIR}"; then
echo "Branch not found performing full clone"
checkout::git_clone "${SEMAPHORE_GIT_URL}" "${SEMAPHORE_GIT_DIR}"
cd "$SEMAPHORE_GIT_DIR" || exit
if ! checkout::branch_checkout; then
return 1
fi
if checkout::checkrevision; then
git reset --hard "${SEMAPHORE_GIT_SHA}" 2>/dev/null
else
return 1
fi
else
cd "${SEMAPHORE_GIT_DIR}" || exit
if ! git reset --hard "${SEMAPHORE_GIT_SHA}" 2>/dev/null; then
echo "SHA: $SEMAPHORE_GIT_SHA not found performing full clone"
retry git fetch --unshallow
if checkout::checkrevision; then
git reset --hard "${SEMAPHORE_GIT_SHA}" 2>/dev/null
else
return 1
fi
fi
fi
checkout::metric "$(du -s . | awk '{ print $1 }')"
}
function checkout::refbased() {
if [ -z "${SEMAPHORE_GIT_DEPTH:-""}" ]; then
SEMAPHORE_GIT_DEPTH=50
fi
if [ "${SEMAPHORE_GIT_REF_TYPE:-""}" = "pull-request" ]; then
checkout::git_clone --depth "${SEMAPHORE_GIT_DEPTH}" "${SEMAPHORE_GIT_URL}" "${SEMAPHORE_GIT_DIR}"
cd "${SEMAPHORE_GIT_DIR}" || exit
# Try fetching refs/objects with a depth.
if retry git fetch --depth "${SEMAPHORE_GIT_DEPTH}" origin +"${SEMAPHORE_GIT_REF}": 2>/dev/null; then
git checkout -qf FETCH_HEAD
echo "HEAD is now at ${SEMAPHORE_GIT_SHA}"
return 0
fi
# Fetch with depth failed, fallback to not using a depth.
if retry git fetch origin +"${SEMAPHORE_GIT_REF}": 2>/dev/null; then
git checkout -qf FETCH_HEAD
echo "HEAD is now at ${SEMAPHORE_GIT_SHA}"
return 0
fi
echo "Revision: ${SEMAPHORE_GIT_SHA} not found .... Exiting"
return 1
fi
if [ "${SEMAPHORE_GIT_REF_TYPE:-""}" = "tag" ]; then
if ! checkout::git_clone --depth "${SEMAPHORE_GIT_DEPTH}" -b "${SEMAPHORE_GIT_TAG_NAME}" "${SEMAPHORE_GIT_URL}" "${SEMAPHORE_GIT_DIR}"; then
echo "Release $SEMAPHORE_GIT_TAG_NAME not found .... Exiting"
return 1
else
cd "${SEMAPHORE_GIT_DIR}" || exit
git checkout -qf "${SEMAPHORE_GIT_TAG_NAME}"
echo "HEAD is now at ${SEMAPHORE_GIT_SHA} Release ${SEMAPHORE_GIT_TAG_NAME}"
return 0
fi
fi
checkout::shallow
}
function checkout::metric() {
if [[ "${SEMAPHORE_TOOLBOX_METRICS_ENABLED:-''}" == "true" ]]; then
ref_type=${SEMAPHORE_GIT_REF_TYPE:-""}
size=${1:-"0"}
[[ $size -gt 0 ]] && status='success' || status='fail'
echo "libcheckout,provider=$SEMAPHORE_GIT_PROVIDER,reftype=$ref_type,status=$status size=$1" >> /tmp/toolbox_metrics
fi
}
# === Slow clone detection and resilient retry ===
# Enabled via SEMAPHORE_GIT_CLONE_SLOW_RETRY=true
# When disabled (default), checkout::git_clone delegates to `retry git clone`
function checkout::git_clone() {
if [ "${SEMAPHORE_GIT_CLONE_SLOW_RETRY:-false}" = "true" ]; then
checkout::resilient_clone "$@"
else
retry git clone "$@"
fi
}
function checkout::resilient_clone() {
local retry_count="${SEMAPHORE_GIT_CLONE_RETRY_COUNT:-2}"
local alt_ip_retries="${SEMAPHORE_GIT_CLONE_ALT_IP_RETRIES:-3}"
local target_dir="${SEMAPHORE_GIT_DIR}"
# Tier 1: normal clone with speed monitoring
local attempt
local last_result=0
for attempt in $(seq 1 "${retry_count}"); do
checkout::clone_with_speed_check git clone "$@"
last_result=$?
if [ "$last_result" -eq 0 ]; then
return 0
fi
if [ "$attempt" -lt "${retry_count}" ]; then
if [ "$last_result" -eq 1 ]; then
echo "[checkout] Slow clone detected, retrying... (attempt ${attempt}/${retry_count})"
else
echo "[checkout] Clone failed, retrying... (attempt ${attempt}/${retry_count})"
fi
rm -rf "${target_dir}"
sleep 2
fi
done
echo "[checkout] Clone failed after ${retry_count} attempts, trying alternative endpoints..."
# Tier 2: parse host + port from SEMAPHORE_GIT_URL, then resolve alt IPs via DoH
local git_host git_port
if [[ "${SEMAPHORE_GIT_URL}" == https://* ]]; then
git_host="${SEMAPHORE_GIT_URL#https://}"
git_host="${git_host%%/*}"
git_port=443
elif [[ "${SEMAPHORE_GIT_URL}" == ssh://* ]]; then
local rest="${SEMAPHORE_GIT_URL#ssh://}"
rest="${rest#*@}"
if [[ "$rest" =~ ^[^/]+:[0-9]+ ]]; then
git_host="${rest%%:*}"
local tail="${rest#*:}"
git_port="${tail%%/*}"
else
git_host="${rest%%/*}"
git_port=22
fi
else
local rest="${SEMAPHORE_GIT_URL#*@}"
git_host="${rest%%:*}"
git_port=22
fi
# GeoDNS-based alt-IP fallback is GitHub-specific; skip for other providers
if [ "$git_host" != "github.com" ]; then
echo "[checkout] Alternative endpoint fallback only supported for github.com (got: ${git_host})"
return 1
fi
local current_ip
current_ip=$(dig +short "$git_host" | head -1)
local alt_ips
alt_ips=$(checkout::resolve_alt_ips "$git_host" "$current_ip")
if [ -z "$alt_ips" ]; then
echo "[checkout] No alternative endpoints found"
return 1
fi
local alt_count=0
local alt_ip
while IFS= read -r alt_ip; do
[ -z "$alt_ip" ] && continue
alt_count=$((alt_count + 1))
[ "$alt_count" -gt "${alt_ip_retries}" ] && break
echo "[checkout] Switching to alternative GitHub endpoint (${alt_ip})"
rm -rf "${target_dir}"
if checkout::clone_with_alt_ip "$alt_ip" "$git_host" "$git_port" git clone "$@"; then
return 0
fi
echo "[checkout] Alternative endpoint (${alt_ip}) also slow, trying next..."
done <<< "$alt_ips"
echo "[checkout] All clone attempts failed"
return 1
}
function checkout::clone_with_speed_check() {
# Runs a git clone with speed monitoring.
# Returns: 0 = success, 1 = killed (slow), 2 = git error
local threshold="${SEMAPHORE_GIT_CLONE_SLOW_THRESHOLD:-20000}"
local timeout="${SEMAPHORE_GIT_CLONE_SLOW_TIMEOUT:-15}"
local check_interval=5
local target_dir="${SEMAPHORE_GIT_DIR}"
"$@" &
local pid=$!
local prev_size=0
local slow_seconds=0
while kill -0 "$pid" 2>/dev/null; do
sleep "$check_interval"
kill -0 "$pid" 2>/dev/null || break
local cur_size
cur_size=$(du -sk "${target_dir}" 2>/dev/null | awk '{print $1}')
cur_size=${cur_size:-0}
if [ "$cur_size" -gt 0 ]; then
local speed=$(( (cur_size - prev_size) * 1024 / check_interval ))
prev_size=$cur_size
if [ "$speed" -lt "$threshold" ]; then
slow_seconds=$((slow_seconds + check_interval))
if [ "$slow_seconds" -ge "$timeout" ]; then
echo "[checkout] Slow clone detected (${speed} bytes/sec for ${slow_seconds}s)"
kill "$pid" 2>/dev/null
wait "$pid" 2>/dev/null
return 1
fi
else
slow_seconds=0
fi
fi
done
wait "$pid"
local exit_code=$?
[ "$exit_code" -eq 0 ] && return 0
return 2
}
function checkout::resolve_alt_ips() {
local git_host="$1"
local current_ip="$2"
local regions
IFS=',' read -ra regions <<< "${SEMAPHORE_GIT_CLONE_ALT_REGIONS:-74.0.0.0/8,177.0.0.0/8,110.0.0.0/8}"
for region in "${regions[@]}"; do
local ip
ip=$(curl -sf "https://dns.google/resolve?name=${git_host}&type=A&edns_client_subnet=${region}" | \
grep -o '"data":"[^"]*"' | sed 's/"data":"//;s/"//' | head -1)
if [ -n "$ip" ] && [ "$ip" != "$current_ip" ]; then
echo "$ip"
fi
done | sort -u
}
function checkout::clone_with_alt_ip() {
local alt_ip="$1"
local git_host="$2"
local git_port="$3"
shift 3
# $@ is now: git clone [args...]
if [[ "${SEMAPHORE_GIT_URL}" == https://* ]]; then
# HTTPS: inject curloptResolve between 'git' and 'clone'
checkout::clone_with_speed_check git -c "http.curloptResolve=${git_host}:${git_port}:${alt_ip}" "${@:2}"
else
# SSH: route through alternative IP via ProxyCommand
local orig_ssh_command="${GIT_SSH_COMMAND:-}"
export GIT_SSH_COMMAND="${orig_ssh_command:-ssh} -o ProxyCommand='nc ${alt_ip} ${git_port}'"
checkout::clone_with_speed_check "$@"
local result=$?
if [ -n "$orig_ssh_command" ]; then
export GIT_SSH_COMMAND="$orig_ssh_command"
else
unset GIT_SSH_COMMAND
fi
return $result
fi
}
export -f checkout
export -f checkout::git_clone
export -f checkout::resilient_clone
export -f checkout::clone_with_speed_check
export -f checkout::resolve_alt_ips
export -f checkout::clone_with_alt_ip