run_eval.sh

#!/usr/bin/env bash
set -euo pipefail

# ==============================================================================
# BFCL Evaluation Runner
#
# Deploys a vLLM server, runs generation + evaluation N times, and averages scores.
#
# Usage:
#   # HuggingFace model
#   ./run_eval.sh --model allenai/Olmo-3-7B-Instruct-SFT --runs 3
#
#   # Local checkpoint with custom serve name
#   ./run_eval.sh --model /mnt/nfs/ytahtah/bfcl/dolci-fc-sft-hf \
#                 --serve-name allenai/Olmo-3-7B-Instruct-SFT \
#                 --runs 5 --gpus 4,5,6,7
#
#   # Custom output directory and test categories
#   ./run_eval.sh --model allenai/Olmo-3-7B-Instruct-SFT \
#                 --runs 3 --output-dir results_experiment1 \
#                 --test-category single_turn,multi_turn
#
# Note: If a .env file exists in the BFCL directory with LOCAL_SERVER_PORT set,
# it will override the --port value due to python-dotenv's override=True.
# Make sure no .env file conflicts with your port setting.
# ==============================================================================

# ── Defaults ──────────────────────────────────────────────────────────────────
RUNS=1
START_RUN=1   # First run number in the loop (1 = normal full run; >1 = resume)
GPUS="4,5,6,7"
PORT=8000
CONTAINER_NAME="ytahtah-vllm-eval"
VLLM_IMAGE="vllm/vllm-openai:v0.19.0"
BFCL_MODEL_KEY="allenai/Olmo-3-7B-Instruct-SFT-FC"
TEST_CATEGORY="all"
OUTPUT_DIR=""
SERVE_NAME=""
MODEL=""
SKIP_SERVER=false
BATCH_INVARIANT=true
SHM_SIZE="16g"
TP_SIZE=""

# Paths (adjust if your setup differs)
BFCL_DIR="$(cd "$(dirname "$0")" && pwd)"
HF_HOME="/mnt/nfs/ytahtah/hf_home"
VLLM_CACHE="/mnt/nfs/ytahtah/.cache/vllm_compile"
TMP_DIR="/mnt/nfs/ytahtah/tmp_compile"
TRITON_CACHE="/mnt/nfs/ytahtah/.triton_cache"
# BFCL writes intermediate result/, score/, .file_locks/ under PROJECT_ROOT.
# Default to BFCL_DIR (single-run behavior unchanged); override per-lane to
# isolate parallel runs.
PROJECT_ROOT=""

# ── Parse arguments ───────────────────────────────────────────────────────────
usage() {
    cat <<'USAGE'
Usage: ./run_eval.sh --model <model> [options]

Required:
  --model <path|hf_id>       HuggingFace model ID or local checkpoint path

Options:
  --serve-name <name>        Model name for vLLM API (required for local checkpoints,
                             defaults to --model value for HF models)
  --bfcl-model-key <key>     BFCL model key (default: allenai/Olmo-3-7B-Instruct-SFT-FC)
  --runs <N>                 Number of evaluation runs (default: 1)
  --start-run <K>            First run number to execute (default: 1).
                             With --runs N, executes runs K..N. Useful for
                             resuming a partially-completed eval: anything
                             below K is left untouched on disk; iteration K
                             wipes run_K/ before starting, so partial state
                             from a prior crash is overwritten cleanly.
                             The end-of-run averaging still reads ALL
                             run_1..run_N from disk.
  --gpus <ids>               Comma-separated GPU IDs (default: 4,5,6,7)
  --port <port>              vLLM server port (default: 8000)
  --test-category <cats>     BFCL test categories (default: all)
  --output-dir <dir>         Output directory under BFCL_DIR (default: eval_results/eval_<model>_<timestamp>)
  --skip-server              Don't deploy vLLM server (assume it's already running)
  --no-batch-invariant       Disable VLLM_BATCH_INVARIANT
  --container-name <name>    Docker container name (default: ytahtah-vllm-eval)
  --vllm-image <image>       vLLM Docker image (default: vllm/vllm-openai:v0.19.0)
  --tp-size <N>              Tensor parallel size (default: number of GPUs)
  --shm-size <size>          Shared memory size (default: 16g)
  --vllm-cache-dir <dir>     Host dir mounted as /root/.cache/vllm
                             (default: /mnt/nfs/ytahtah/.cache/vllm_compile)
  --tmp-dir <dir>            Host dir mounted as /tmp
                             (default: /mnt/nfs/ytahtah/tmp_compile)
  --triton-cache-dir <dir>   Host dir mounted as /root/.triton
                             (default: /mnt/nfs/ytahtah/.triton_cache)
  --project-root <dir>       BFCL_PROJECT_ROOT — where bfcl writes result/,
                             score/, .file_locks/ (default: BFCL_DIR).
                             Use a per-lane dir for parallel runs.
  -h, --help                 Show this help
USAGE
    exit 0
}

while [[ $# -gt 0 ]]; do
    case $1 in
        --model)            MODEL="$2"; shift 2 ;;
        --serve-name)       SERVE_NAME="$2"; shift 2 ;;
        --bfcl-model-key)   BFCL_MODEL_KEY="$2"; shift 2 ;;
        --runs)             RUNS="$2"; shift 2 ;;
        --start-run)        START_RUN="$2"; shift 2 ;;
        --gpus)             GPUS="$2"; shift 2 ;;
        --port)             PORT="$2"; shift 2 ;;
        --test-category)    TEST_CATEGORY="$2"; shift 2 ;;
        --output-dir)       OUTPUT_DIR="$2"; shift 2 ;;
        --skip-server)      SKIP_SERVER=true; shift ;;
        --no-batch-invariant) BATCH_INVARIANT=false; shift ;;
        --container-name)   CONTAINER_NAME="$2"; shift 2 ;;
        --vllm-image)       VLLM_IMAGE="$2"; shift 2 ;;
        --tp-size)          TP_SIZE="$2"; shift 2 ;;
        --shm-size)         SHM_SIZE="$2"; shift 2 ;;
        --vllm-cache-dir)   VLLM_CACHE="$2"; shift 2 ;;
        --tmp-dir)          TMP_DIR="$2"; shift 2 ;;
        --triton-cache-dir) TRITON_CACHE="$2"; shift 2 ;;
        --project-root)     PROJECT_ROOT="$2"; shift 2 ;;
        -h|--help)          usage ;;
        *)                  echo "Unknown option: $1" >&2; exit 1 ;;
    esac
done

if [[ -z "$MODEL" ]]; then
    echo "Error: --model is required" >&2
    usage
fi

# Validate START_RUN: must be in [1, RUNS]. Anything outside that range is
# almost certainly a script bug in the caller; failing fast prevents silent
# misuse (e.g. --start-run 0 would re-run nothing; --start-run > RUNS would
# do nothing then run averaging across phantom runs).
if ! [[ "$START_RUN" =~ ^[0-9]+$ ]] || (( START_RUN < 1 )); then
    echo "Error: --start-run must be a positive integer (got '${START_RUN}')" >&2
    exit 1
fi
if (( START_RUN > RUNS )); then
    echo "Error: --start-run (${START_RUN}) cannot exceed --runs (${RUNS})." >&2
    echo "       Nothing would be executed. Caller probably has a bug." >&2
    exit 1
fi

# ── Derived values ────────────────────────────────────────────────────────────
NUM_GPUS=$(echo "$GPUS" | tr ',' '\n' | wc -l)
TP_SIZE="${TP_SIZE:-$NUM_GPUS}"

# Determine if model is local path or HF ID
IS_LOCAL=false
if [[ -d "$MODEL" ]]; then
    IS_LOCAL=true
    MODEL="$(cd "$MODEL" && pwd)"  # resolve to absolute path
    if [[ -z "$SERVE_NAME" ]]; then
        echo "Error: --serve-name is required for local checkpoints" >&2
        exit 1
    fi
fi

SERVE_NAME="${SERVE_NAME:-$MODEL}"

# Output directory (always under eval_results/ by default)
if [[ -z "$OUTPUT_DIR" ]]; then
    SAFE_NAME=$(echo "$SERVE_NAME" | tr '/' '_')
    TIMESTAMP=$(date +%Y%m%d_%H%M%S)
    OUTPUT_DIR="eval_results/eval_${SAFE_NAME}_${TIMESTAMP}"
fi
OUTPUT_PATH="${BFCL_DIR}/${OUTPUT_DIR}"
PROJECT_ROOT="${PROJECT_ROOT:-$BFCL_DIR}"
mkdir -p "$OUTPUT_PATH" "$VLLM_CACHE" "$TMP_DIR" "$TRITON_CACHE" \
         "${PROJECT_ROOT}/result" "${PROJECT_ROOT}/score" "${PROJECT_ROOT}/.file_locks"

# BFCL result directory name (model key with / -> _)
BFCL_RESULT_NAME=$(echo "$BFCL_MODEL_KEY" | tr '/' '_')

echo "============================================================"
echo "BFCL Evaluation Runner"
echo "============================================================"
echo "Model:            $MODEL"
echo "Serve name:       $SERVE_NAME"
echo "BFCL model key:   $BFCL_MODEL_KEY"
echo "Local checkpoint: $IS_LOCAL"
echo "GPUs:             $GPUS ($NUM_GPUS GPUs, TP=$TP_SIZE)"
echo "Port:             $PORT"
if (( START_RUN == 1 )); then
    echo "Runs:             $RUNS"
else
    echo "Runs:             ${START_RUN}..${RUNS}  (resuming; runs below ${START_RUN} kept on disk)"
fi
echo "Test categories:  $TEST_CATEGORY"
echo "Batch invariant:  $BATCH_INVARIANT"
echo "Output:           $OUTPUT_PATH"
echo "vLLM cache:       $VLLM_CACHE"
echo "tmp dir:          $TMP_DIR"
echo "Triton cache:     $TRITON_CACHE"
echo "Project root:     $PROJECT_ROOT"
echo "============================================================"

# ── Tear down our own container on exit/interrupt ────────────────────────────
# Without this, a Ctrl-C / SIGTERM / error mid-run orphans the vLLM container
# holding the GPUs. Only touches the container WE deploy — never a server we
# were told to --skip-server (that one belongs to the caller).
cleanup_run_eval() {
    local code=$?
    if [[ "$SKIP_SERVER" == "false" ]]; then
        docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
    fi
    exit "$code"
}
trap cleanup_run_eval EXIT
trap 'exit 130' INT TERM

# ── Activate venv ─────────────────────────────────────────────────────────────
source "${BFCL_DIR}/.venv/bin/activate"

# ── Deploy vLLM server ────────────────────────────────────────────────────────
deploy_server() {
    echo ""
    echo "[server] Stopping existing container '$CONTAINER_NAME' if any..."
    docker rm -f "$CONTAINER_NAME" 2>/dev/null || true

    echo "[server] Starting vLLM server..."

    # Build docker run command as a string.
    # We use a string (not an array) because --gpus requires literal inner quotes
    # that bash arrays cannot represent: --gpus '"device=4,5,6,7"'
    local DOCKER_CMD="docker run -d"
    DOCKER_CMD+=" --name ${CONTAINER_NAME}"
    DOCKER_CMD+=" --gpus '\"device=${GPUS}\"'"
    DOCKER_CMD+=" --shm-size ${SHM_SIZE}"
    DOCKER_CMD+=" -p ${PORT}:${PORT}"
    DOCKER_CMD+=" -v ${HF_HOME}:/hf_home"
    DOCKER_CMD+=" -v ${VLLM_CACHE}:/root/.cache/vllm"
    DOCKER_CMD+=" -v ${TMP_DIR}:/tmp"
    DOCKER_CMD+=" -v ${TRITON_CACHE}:/root/.triton"
    DOCKER_CMD+=" -e HF_HOME=/hf_home"

    # Add HF_TOKEN if set
    if [[ -n "${HF_TOKEN:-}" ]]; then
        DOCKER_CMD+=" -e HF_TOKEN=${HF_TOKEN}"
    fi

    # Batch invariant
    if [[ "$BATCH_INVARIANT" == "true" ]]; then
        DOCKER_CMD+=" -e VLLM_BATCH_INVARIANT=1"
    fi

    if [[ "$IS_LOCAL" == "true" ]]; then
        DOCKER_CMD+=" -v ${MODEL}:/model"
    fi

    DOCKER_CMD+=" ${VLLM_IMAGE}"

    # vLLM args
    if [[ "$IS_LOCAL" == "true" ]]; then
        DOCKER_CMD+=" --model /model --served-model-name ${SERVE_NAME}"
    else
        DOCKER_CMD+=" --model ${MODEL} --served-model-name ${SERVE_NAME}"
    fi

    DOCKER_CMD+=" --tensor-parallel-size ${TP_SIZE}"
    DOCKER_CMD+=" --port ${PORT}"
    DOCKER_CMD+=" --attention-backend FLASH_ATTN"
    DOCKER_CMD+=" --trust-remote-code"

    # Run docker command via eval (needed for --gpus quoting)
    echo "[server] Running: ${DOCKER_CMD}"
    eval "${DOCKER_CMD}"

    echo "[server] Waiting for server to be ready..."
    local MAX_WAIT=300
    local WAITED=0
    while ! curl -sf "http://localhost:${PORT}/v1/models" >/dev/null 2>&1; do
        sleep 5
        WAITED=$((WAITED + 5))
        if [[ $WAITED -ge $MAX_WAIT ]]; then
            echo "[server] ERROR: Server did not start within ${MAX_WAIT}s"
            echo "[server] Last logs:"
            docker logs --tail 50 "$CONTAINER_NAME"
            exit 1
        fi
        echo "[server] Waiting... (${WAITED}s)"
    done
    echo "[server] Server is ready!"
}

if [[ "$SKIP_SERVER" == "false" ]]; then
    deploy_server
else
    echo "[server] Skipping server deployment (--skip-server)"
fi

# ── Run evaluation rounds [START_RUN..RUNS] ───────────────────────────────────
for RUN_NUM in $(seq "$START_RUN" "$RUNS"); do
    echo ""
    echo "============================================================"
    echo "Run ${RUN_NUM}/${RUNS}"
    echo "============================================================"

    # Wipe any stale or partial state for THIS run number. This is the key
    # invariant: at the start of iteration K, ${OUTPUT_PATH}/run_K is empty,
    # so the later `cp -r SRC ${RUN_DIR}/result` creates result/ flat instead
    # of nesting as result/<model_name>/. Runs below START_RUN are NOT touched.
    rm -rf "${OUTPUT_PATH}/run_${RUN_NUM}"

    # Clean previous BFCL results for this model only (under PROJECT_ROOT)
    rm -rf "${PROJECT_ROOT}/result/${BFCL_RESULT_NAME}"
    # score/ is transient — BFCL writes all models' scores here, but we only
    # care about the current model. Safest to wipe and regenerate.
    rm -rf "${PROJECT_ROOT}/score"

    # Generate
    echo "[run ${RUN_NUM}] Generating responses..."
    BFCL_PROJECT_ROOT="$PROJECT_ROOT" LOCAL_SERVER_PORT="$PORT" bfcl generate \
        --model "$BFCL_MODEL_KEY" \
        --test-category "$TEST_CATEGORY" \
        --backend vllm \
        --skip-server-setup \
        --num-gpus "$NUM_GPUS"

    # Evaluate
    echo "[run ${RUN_NUM}] Evaluating..."
    BFCL_PROJECT_ROOT="$PROJECT_ROOT" bfcl evaluate \
        --model "$BFCL_MODEL_KEY" \
        --test-category "$TEST_CATEGORY"

    # Save this run's results and scores
    RUN_DIR="${OUTPUT_PATH}/run_${RUN_NUM}"
    mkdir -p "$RUN_DIR"

    # Defense-in-depth against the cp -r nesting trap: ensure the destinations
    # don't exist so `cp -r SRC dst` creates dst flat instead of nesting into
    # dst/<name>/. The run_K wipe at the loop top already guarantees this, but
    # this makes the invariant local to the copy so a future refactor can't
    # silently reintroduce nesting / mixed-provenance results.
    rm -rf "${RUN_DIR}/result" "${RUN_DIR}/score"

    if [[ -d "${PROJECT_ROOT}/result/${BFCL_RESULT_NAME}" ]]; then
        cp -r "${PROJECT_ROOT}/result/${BFCL_RESULT_NAME}" "${RUN_DIR}/result"
    fi
    if [[ -d "${PROJECT_ROOT}/score" ]]; then
        cp -r "${PROJECT_ROOT}/score" "${RUN_DIR}/score"
    fi

    echo "[run ${RUN_NUM}] Saved to ${RUN_DIR}"
done

# ── Average scores across runs ────────────────────────────────────────────────
echo ""
echo "============================================================"
echo "Averaging scores across ${RUNS} runs"
echo "============================================================"

python3 - "$OUTPUT_PATH" "$RUNS" << 'PYEOF'
import csv
import sys
import os

output_path = sys.argv[1]
num_runs = int(sys.argv[2])

# Collect all overall CSVs
all_rows = []
headers = None

skipped_mismatch = 0
for run_num in range(1, num_runs + 1):
    csv_path = os.path.join(output_path, f"run_{run_num}", "score", "data_overall.csv")
    if not os.path.exists(csv_path):
        print(f"  WARNING: {csv_path} not found, skipping run {run_num}")
        continue
    if os.path.getsize(csv_path) == 0:
        print(f"  WARNING: {csv_path} is empty (0 bytes), skipping run {run_num}")
        continue
    with open(csv_path) as f:
        reader = csv.reader(f)
        try:
            h = next(reader)
            row = next(reader)
        except StopIteration:
            # Present but no header+data row (truncated mid-write / crash).
            print(f"  WARNING: {csv_path} has no data row, skipping run {run_num}")
            continue
    if headers is None:
        headers = h
    elif h != headers:
        # Guard against averaging mismatched columns by fixed index when a
        # resume combines runs produced with different category sets / schema.
        print(f"  WARNING: {csv_path} header differs from the first run — "
              f"skipping run {run_num} (refusing to average mismatched columns)")
        skipped_mismatch += 1
        continue
    all_rows.append(row)

if skipped_mismatch:
    print(f"  WARNING: {skipped_mismatch} run(s) skipped due to header mismatch — "
          f"averaged over {len(all_rows)} consistent run(s) only")

if not all_rows:
    print("  ERROR: No usable score files found!")
    sys.exit(1)

print(f"  Found {len(all_rows)} valid runs out of {num_runs}")

# Identify numeric (percentage) columns — only average columns where ALL runs have values
avg_row = list(all_rows[0])  # start with first row as template
numeric_cols = []

# Skip non-metric columns that shouldn't be averaged
SKIP_COLUMNS = {'Rank', 'Total Cost ($)', 'Model', 'Model Link', 'Organization', 'License'}

for col_idx, header in enumerate(headers):
    if header in SKIP_COLUMNS:
        continue
    values = []
    for row in all_rows:
        val = row[col_idx].strip()
        if val == 'N/A':
            break  # skip column entirely if any run has N/A
        if val.endswith('%'):
            try:
                values.append(float(val[:-1]))
            except ValueError:
                break
        elif val.replace('.', '', 1).replace('-', '', 1).isdigit():
            try:
                values.append(float(val))
            except ValueError:
                break

    if len(values) == len(all_rows):
        numeric_cols.append(col_idx)
        mean = sum(values) / len(values)
        std = (sum((v - mean) ** 2 for v in values) / len(values)) ** 0.5

        # Format back as percentage or number
        orig = all_rows[0][col_idx].strip()
        if orig.endswith('%'):
            avg_row[col_idx] = f"{mean:.2f}%"
        else:
            avg_row[col_idx] = f"{mean:.2f}"

# Write averaged CSV
avg_csv_path = os.path.join(output_path, "data_overall_averaged.csv")
with open(avg_csv_path, 'w', newline='') as f:
    writer = csv.writer(f)
    writer.writerow(headers)
    writer.writerow(avg_row)

# Write detailed summary with per-run breakdown
summary_path = os.path.join(output_path, "summary.txt")
with open(summary_path, 'w') as f:
    f.write(f"BFCL Evaluation Summary ({len(all_rows)} runs)\n")
    f.write("=" * 80 + "\n\n")

    for col_idx in numeric_cols:
        header = headers[col_idx]
        values = []
        for row in all_rows:
            val = row[col_idx].strip()
            if val.endswith('%'):
                values.append(float(val[:-1]))
            else:
                values.append(float(val))

        mean = sum(values) / len(values)
        std = (sum((v - mean) ** 2 for v in values) / len(values)) ** 0.5
        is_pct = all_rows[0][col_idx].strip().endswith('%')
        unit = "%" if is_pct else ""

        f.write(f"{header}:\n")
        f.write(f"  Mean: {mean:.2f}{unit}  Std: {std:.2f}{unit}\n")
        for i, v in enumerate(values):
            f.write(f"  Run {i+1}: {v:.2f}{unit}\n")
        f.write("\n")

# Also print key results to stdout
print(f"\n  Averaged scores saved to: {avg_csv_path}")
print(f"  Detailed summary saved to: {summary_path}")
print()

# Print key metrics
key_cols = ['Overall Acc', 'Non-Live AST Acc', 'Live Acc', 'Multi Turn Acc',
            'Memory Acc', 'Web Search Acc', 'Relevance Detection', 'Irrelevance Detection']

for col_name in key_cols:
    if col_name not in headers:
        continue
    col_idx = headers.index(col_name)
    values = []
    all_valid = True
    for row in all_rows:
        val = row[col_idx].strip()
        if val == 'N/A':
            all_valid = False
            break
        if val.endswith('%'):
            values.append(float(val[:-1]))
        else:
            values.append(float(val))

    if not all_valid or not values:
        print(f"  {col_name:<25}    N/A")
        continue

    mean = sum(values) / len(values)
    std = (sum((v - mean) ** 2 for v in values) / len(values)) ** 0.5
    print(f"  {col_name:<25} {mean:6.2f}% +/- {std:.2f}%")

PYEOF

# ── Cleanup ───────────────────────────────────────────────────────────────────
if [[ "$SKIP_SERVER" == "false" ]]; then
    echo ""
    echo "[server] Stopping container '$CONTAINER_NAME'..."
    docker stop "$CONTAINER_NAME" >/dev/null 2>&1 || true
    docker rm "$CONTAINER_NAME" >/dev/null 2>&1 || true
fi

echo ""
echo "============================================================"
echo "Done! All results saved to: ${OUTPUT_PATH}"
echo "============================================================"