monitor.sh

#!/usr/bin/env bash
# Live monitoring dashboard for 8 vLLM instances.
# Usage: bash monitor.sh [refresh_interval_seconds]
# Press Ctrl+C to exit.

INTERVAL=${1:-5}
NUM_GPUS=8

# Helper: extract a numeric Prometheus metric value, returning 0 for
# missing / NaN / Inf.  Uses awk for all numeric conversion so bash
# never sees non-integer strings.
# Usage: prom_val "$METRICS" "metric_name{" [scale]
#   scale is an optional multiplier (e.g., 100 to convert 0-1 → percentage)
prom_val() {
    local metrics="$1" pattern="$2" scale="${3:-1}"
    echo "$metrics" \
        | grep "^${pattern}" \
        | head -1 \
        | awk -v s="$scale" '{
            v = $NF + 0          # $NF = last field (value); +0 forces numeric
            if (v != v) v = 0    # NaN check (NaN != NaN in awk)
            printf "%.0f", v * s
        }'
}

# Like prom_val but keeps one decimal place (for KV cache %).
prom_val_f1() {
    local metrics="$1" pattern="$2" scale="${3:-1}"
    echo "$metrics" \
        | grep "^${pattern}" \
        | head -1 \
        | awk -v s="$scale" '{
            v = $NF + 0
            if (v != v) v = 0
            printf "%.1f", v * s
        }'
}

# Like prom_val but sums ALL matching lines (for metrics with multiple label sets,
# e.g., vllm:request_success_total has one line per finished_reason).
prom_sum() {
    local metrics="$1" pattern="$2" scale="${3:-1}"
    echo "$metrics" \
        | grep "^${pattern}" \
        | awk -v s="$scale" '{
            v = $NF + 0
            if (v == v) total += v   # skip NaN
        } END { printf "%.0f", total * s }'
}

# ── State arrays for per-second throughput calculation ──────────────
declare -a prev_prompt prev_gen prev_success
for i in $(seq 0 $((NUM_GPUS - 1))); do
    prev_prompt[$i]=0; prev_gen[$i]=0; prev_success[$i]=0
done
first_iteration=1

while true; do
    clear
    echo "═══════════════════════════════════════════════════════════════════════════════════════════"
    echo "  vLLM Monitor — $(date '+%Y-%m-%d %H:%M:%S') — refresh ${INTERVAL}s"
    echo "═══════════════════════════════════════════════════════════════════════════════════════════"
    printf "%-5s %-6s %6s %6s %7s %11s %11s %7s %8s %9s\n" \
        "GPU" "STATE" "RUN" "WAIT" "KV%" "PROMPT_TK" "GEN_TK" "TK/s" "DONE" "PREEMPT"
    echo "─────────────────────────────────────────────────────────────────────────────────────────"

    total_running=0
    total_waiting=0
    total_prompt=0
    total_gen=0
    total_tps=0
    total_done=0
    total_preempt=0

    for i in $(seq 0 $((NUM_GPUS - 1))); do
        PORT=$((8000 + i))
        # -f: fail on HTTP errors (4xx/5xx) so we don't parse error pages
        METRICS=$(curl -sf --max-time 2 "http://localhost:${PORT}/metrics" 2>/dev/null)

        if [ -z "$METRICS" ]; then
            printf "%-5s %-6s\n" "GPU$i" "DOWN"
            continue
        fi

        # ── Extract metrics ────────────────────────────────────────
        # Gauges (no _total suffix)
        RUNNING=$(prom_val  "$METRICS" 'vllm:num_requests_running{');  RUNNING=${RUNNING:-0}
        WAITING=$(prom_val  "$METRICS" 'vllm:num_requests_waiting{');  WAITING=${WAITING:-0}
        KV_USAGE=$(prom_val_f1 "$METRICS" 'vllm:kv_cache_usage_perc{' 100); KV_USAGE=${KV_USAGE:-0.0}

        # Counters (_total suffix auto-appended by prometheus_client)
        PROMPT_TK=$(prom_val "$METRICS" 'vllm:prompt_tokens_total{');  PROMPT_TK=${PROMPT_TK:-0}
        GEN_TK=$(prom_val    "$METRICS" 'vllm:generation_tokens_total{'); GEN_TK=${GEN_TK:-0}
        DONE=$(prom_sum      "$METRICS" 'vllm:request_success_total{'); DONE=${DONE:-0}
        PREEMPT=$(prom_val   "$METRICS" 'vllm:num_preemptions_total{'); PREEMPT=${PREEMPT:-0}

        # ── Tokens/sec (delta from previous iteration) ─────────────
        TPS="-"
        if [ "$first_iteration" -eq 0 ]; then
            delta_tk=$(( (PROMPT_TK + GEN_TK) - (prev_prompt[$i] + prev_gen[$i]) ))
            if [ "$INTERVAL" -gt 0 ]; then
                TPS=$(( delta_tk / INTERVAL ))
            fi
        fi
        prev_prompt[$i]=$PROMPT_TK
        prev_gen[$i]=$GEN_TK
        prev_success[$i]=$DONE

        # ── Accumulate totals ──────────────────────────────────────
        total_running=$((total_running + RUNNING))
        total_waiting=$((total_waiting + WAITING))
        total_prompt=$((total_prompt + PROMPT_TK))
        total_gen=$((total_gen + GEN_TK))
        total_done=$((total_done + DONE))
        total_preempt=$((total_preempt + PREEMPT))
        if [ "$TPS" != "-" ]; then
            total_tps=$((total_tps + TPS))
        fi

        printf "%-5s %-6s %6s %6s %6s%% %11s %11s %7s %8s %9s\n" \
            "GPU$i" "OK" "$RUNNING" "$WAITING" "$KV_USAGE" \
            "$PROMPT_TK" "$GEN_TK" "$TPS" "$DONE" "$PREEMPT"
    done

    echo "─────────────────────────────────────────────────────────────────────────────────────────"
    TOTAL_TPS_STR="-"
    [ "$first_iteration" -eq 0 ] && TOTAL_TPS_STR="$total_tps"
    printf "%-5s %-6s %6s %6s %7s %11s %11s %7s %8s %9s\n" \
        "TOTAL" "" "$total_running" "$total_waiting" "" \
        "$total_prompt" "$total_gen" "$TOTAL_TPS_STR" "$total_done" "$total_preempt"
    echo "═══════════════════════════════════════════════════════════════════════════════════════════"

    first_iteration=0
    sleep "$INTERVAL"
done