Skip to content
41 changes: 28 additions & 13 deletions .github/workflows/api-compliance-runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,38 +49,53 @@ jobs:

- name: Determine test parameters
id: params
env:
PATTERNS_INPUT: ${{ github.event.inputs.patterns }}
MODELS_INPUT: ${{ github.event.inputs.models }}
run: |
# Use input values or defaults
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
PATTERNS="${{ github.event.inputs.patterns }}"
MODELS="${{ github.event.inputs.models }}"
PATTERNS="$PATTERNS_INPUT"
MODELS="$MODELS_INPUT"
else
PATTERNS=""
MODELS=""
fi

# Build command args
ARGS=""
if [ -n "$PATTERNS" ]; then
ARGS="$ARGS --patterns $PATTERNS"
fi
if [ -n "$MODELS" ]; then
ARGS="$ARGS --models $MODELS"
else
ARGS="$ARGS --models $DEFAULT_MODELS"
if [ -z "$MODELS" ]; then
MODELS="$DEFAULT_MODELS"
fi

echo "args=$ARGS" >> $GITHUB_OUTPUT
write_output() {
local name=$1
local value=$2
local delimiter="${name}_$(openssl rand -hex 16)"
{
echo "$name<<$delimiter"
printf '%s\n' "$value"
echo "$delimiter"
} >> "$GITHUB_OUTPUT"
}

write_output patterns "$PATTERNS"
write_output models "$MODELS"

- name: Run API compliance tests
id: compliance
env:
LLM_API_KEY: ${{ secrets.LLM_API_KEY_EVAL }}
LLM_BASE_URL: https://llm-proxy.eval.all-hands.dev
GITHUB_RUN_ID: ${{ github.run_id }}
PATTERNS: ${{ steps.params.outputs.patterns }}
MODELS: ${{ steps.params.outputs.models }}
run: |
args=(--models "$MODELS")
if [ -n "$PATTERNS" ]; then
args=(--patterns "$PATTERNS" "${args[@]}")
fi

uv run python tests/integration/api_compliance/run_compliance.py \
${{ steps.params.outputs.args }} \
"${args[@]}" \
--output-dir compliance-results/
continue-on-error: true # Tests may "fail" but that's expected

Expand Down
7 changes: 4 additions & 3 deletions .github/workflows/condenser-runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,9 @@ jobs:

- name: Create consolidated PR comment
if: github.event_name == 'pull_request_target'
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
# Add header to clarify these are non-blocking tests
echo "## Condenser Test Results (Non-Blocking)" > final_report.md
Expand All @@ -239,6 +242,4 @@ jobs:
# Sanitize @OpenHands mentions to prevent self-mention loops
COMMENT_BODY=$(uv run python -c "from openhands.sdk.utils.github import sanitize_openhands_mentions; import sys; print(sanitize_openhands_mentions(sys.stdin.read()), end='')" < final_report.md)
# Use GitHub CLI to create comment with explicit PR number
echo "$COMMENT_BODY" | gh pr comment ${{ github.event.pull_request.number }} --body-file -
env:
GH_TOKEN: ${{ github.token }}
printf '%s' "$COMMENT_BODY" | gh pr comment "$PR_NUMBER" --body-file -
31 changes: 20 additions & 11 deletions .github/workflows/integration-runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,18 @@ jobs:
echo "Using specified model_ids: $MODEL_IDS"
fi

export MODEL_IDS

# Resolve model configs using resolve_model_config.py
# Transform output to matrix format for integration tests
MATRIX=$(python3 << EOF
MATRIX=$(python3 << 'EOF'
import json
import os
import sys
sys.path.insert(0, '.github/run-eval')
from resolve_model_config import MODELS

model_ids = "$MODEL_IDS".split(",")
model_ids = os.environ["MODEL_IDS"].split(",")
Comment thread
enyst marked this conversation as resolved.
model_ids = [m.strip() for m in model_ids if m.strip()]

matrix = []
Expand Down Expand Up @@ -129,10 +132,14 @@ jobs:
id: resolve-issue
env:
ISSUE_NUMBER_INPUT: ${{ github.event.inputs.issue_number || '' }}
PR_NUMBER: ${{ github.event.pull_request.number }}
PR_NUMBER: ${{ github.event.pull_request.number || '' }}
run: |
# Priority: explicit input > PR number from label trigger
if [ -n "$ISSUE_NUMBER_INPUT" ]; then
if ! [[ "$ISSUE_NUMBER_INPUT" =~ ^[0-9]+$ ]]; then
echo "Invalid issue_number input. Expected a numeric issue or PR number." >&2
exit 1
fi
echo "issue_number=$ISSUE_NUMBER_INPUT" >> "$GITHUB_OUTPUT"
elif [ -n "$PR_NUMBER" ]; then
echo "issue_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
Expand Down Expand Up @@ -181,22 +188,23 @@ jobs:
MODEL_IDS: ${{ github.event.inputs.model_ids || 'all models' }}
TEST_TYPE: ${{ github.event.inputs.test_type || 'all' }}
REASON: ${{ github.event.inputs.reason }}
WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
# Sanitize @OpenHands mentions to prevent self-mention loops
SANITIZED_REASON=$(echo "$REASON" | sed 's/@OpenHands/@\u200BOpenHands/g; s/@openhands/@\u200Bopenhands/g')
SANITIZED_MODEL_IDS=$(echo "$MODEL_IDS" | sed 's/@OpenHands/@\u200BOpenHands/g; s/@openhands/@\u200Bopenhands/g')
SANITIZED_REASON=$(printf '%s' "$REASON" | sed 's/@OpenHands/@\u200BOpenHands/g; s/@openhands/@\u200Bopenhands/g')
SANITIZED_MODEL_IDS=$(printf '%s' "$MODEL_IDS" | sed 's/@OpenHands/@\u200BOpenHands/g; s/@openhands/@\u200Bopenhands/g')
COMMENT_BODY=$(cat <<EOF
**Integration Tests Triggered**

- **Reason:** $SANITIZED_REASON
- **Test type:** $TEST_TYPE
- **Models:** $SANITIZED_MODEL_IDS
- **Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- **Workflow run:** $WORKFLOW_RUN_URL

Results will be posted here when complete.
EOF
)
gh issue comment "$ISSUE_NUMBER" --body "$COMMENT_BODY"
printf '%s' "$COMMENT_BODY" | gh issue comment "$ISSUE_NUMBER" --body-file -

run-integration-tests:
# Security: Only run when integration-related labels are present, via workflow_dispatch, or on schedule
Expand Down Expand Up @@ -260,6 +268,8 @@ jobs:

# Run integration test evaluation
- name: Determine test selection
env:
TEST_TYPE_INPUT: ${{ github.event.inputs.test_type }}
run: |
TEST_TYPE_ARGS=""
if [ "${{ github.event_name }}" = "pull_request_target" ] && [ "${{ github.event.label.name }}" = "behavior-test" ]; then
Expand All @@ -269,7 +279,7 @@ jobs:
TEST_TYPE_ARGS="--test-type integration"
echo "integration-test label detected; running integration tests only."
elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
test_type="${{ github.event.inputs.test_type }}"
test_type="$TEST_TYPE_INPUT"
case "$test_type" in
behavior)
TEST_TYPE_ARGS="--test-type behavior"
Expand Down Expand Up @@ -415,8 +425,6 @@ jobs:

- name: Consolidate test results
env:
EVENT_NAME: ${{ github.event_name }}
PR_NUMBER: ${{ github.event.pull_request.number }}
MANUAL_REASON: ${{ github.event.inputs.reason }}
COMMIT_SHA: ${{ github.sha }}
PYTHONPATH: ${{ github.workspace }}
Expand Down Expand Up @@ -447,9 +455,10 @@ jobs:
# Sanitize @OpenHands mentions to prevent self-mention loops
COMMENT_BODY=$(uv run python -c "from openhands.sdk.utils.github import sanitize_openhands_mentions; import sys; print(sanitize_openhands_mentions(sys.stdin.read()), end='')" < consolidated_report.md)
# Use GitHub CLI to create comment with explicit PR number
echo "$COMMENT_BODY" | gh pr comment ${{ github.event.pull_request.number }} --body-file -
printf '%s' "$COMMENT_BODY" | gh pr comment "$PR_NUMBER" --body-file -
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}

- name: Comment on specified issue/PR (workflow_dispatch)
if: github.event_name == 'workflow_dispatch' && needs.setup-matrix.outputs.issue_number != ''
Expand Down
5 changes: 4 additions & 1 deletion .github/workflows/pr-artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@ jobs:
steps:
- name: Check if fork PR
id: check-fork
env:
PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
PR_BASE_REPO: ${{ github.event.pull_request.base.repo.full_name }}
run: |
if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.event.pull_request.base.repo.full_name }}" ]; then
if [ "$PR_HEAD_REPO" != "$PR_BASE_REPO" ]; then
echo "is_fork=true" >> $GITHUB_OUTPUT
echo "::notice::Fork PR detected - skipping auto-cleanup (manual removal required)"
else
Expand Down
58 changes: 39 additions & 19 deletions .github/workflows/prepare-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Validate version format
env:
INPUTS_VERSION: ${{ inputs.version }}
run: |
if ! [[ "${{ inputs.version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
if ! [[ "$INPUTS_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "❌ Invalid version format. Expected: X.Y.Z (e.g., 1.2.3)"
exit 1
fi
echo "✅ Version format is valid: ${{ inputs.version }}"
echo "✅ Version format is valid: $INPUTS_VERSION"

- name: Checkout repository
uses: actions/checkout@v6
Expand All @@ -38,46 +40,60 @@ jobs:
git config user.email "github-actions[bot]@users.noreply.github.com"

- name: Create release branch
env:
INPUTS_VERSION: ${{ inputs.version }}
run: |
BRANCH_NAME="rel-${{ inputs.version }}"
BRANCH_NAME="rel-$INPUTS_VERSION"
echo "Creating branch: $BRANCH_NAME"
git checkout -b "$BRANCH_NAME"
echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV

- name: Set package version
env:
INPUTS_VERSION: ${{ inputs.version }}
run: |
echo "🔧 Setting version to ${{ inputs.version }}"
make set-package-version version=${{ inputs.version }}
echo "🔧 Setting version to $INPUTS_VERSION"
make set-package-version version="$INPUTS_VERSION"

- name: Update sdk_ref default in run-eval workflow
run: python3 .github/scripts/update_sdk_ref_default.py "${{ inputs.version }}"
env:
INPUTS_VERSION: ${{ inputs.version }}
run: python3 .github/scripts/update_sdk_ref_default.py "$INPUTS_VERSION"

- name: Commit version changes
env:
INPUTS_VERSION: ${{ inputs.version }}
run: |
git add .
if git diff --staged --quiet; then
echo "No changes to commit"
else
git commit -m "Release v${{ inputs.version }}" -m "Co-authored-by: openhands <openhands@all-hands.dev>"
git commit -m "Release v$INPUTS_VERSION" -m "Co-authored-by: openhands <openhands@all-hands.dev>"
echo "✅ Changes committed"
fi

- name: Push release branch
run: |
git push -u origin "${{ env.BRANCH_NAME }}"
echo "✅ Branch pushed: ${{ env.BRANCH_NAME }}"
git push -u origin "$BRANCH_NAME"
echo "✅ Branch pushed: $BRANCH_NAME"

- name: Create Pull Request
env:
GH_TOKEN: ${{ secrets.OPENHANDS_BOT_GITHUB_PAT_PUBLIC }}
INPUTS_VERSION: ${{ inputs.version }}
run: |
cat > pr_body.txt << 'EOF'
## Release v${{ inputs.version }}
python3 << 'PY'
import os
from pathlib import Path

version = os.environ["INPUTS_VERSION"]
Path("pr_body.txt").write_text(
f"""## Release v{version}

This PR prepares the release for version **${{ inputs.version }}**.
This PR prepares the release for version **{version}**.

### Release Checklist
- [x] Version set to ${{ inputs.version }}
- [x] Version set to {version}
- [ ] Fix any deprecation deadlines if they exist
- [ ] Integration tests pass (tagged with `integration-test`)
- [ ] Behavior tests pass (tagged with `behavior-test`)
Expand All @@ -87,16 +103,18 @@ jobs:

### What happens on merge
When this PR is merged, the `create-release.yml` workflow will automatically:
1. Create a GitHub release with tag `v${{ inputs.version }}` and auto-generated notes, plus an explicit preamble for merged `release-note-required` PRs
1. Create a GitHub release with tag `v{version}` and auto-generated notes, plus an explicit preamble for merged `release-note-required` PRs
2. Trigger `pypi-release.yml` to publish all packages to PyPI
3. Trigger `version-bump-prs.yml` to create downstream version bump PRs
EOF
"""
)
PY

gh pr create \
--title "Release v${{ inputs.version }}" \
--title "Release v$INPUTS_VERSION" \
--body-file pr_body.txt \
--base main \
--head "${{ env.BRANCH_NAME }}" \
--head "$BRANCH_NAME" \
--label "integration-test" \
--label "behavior-test" \
--label "test-examples"
Expand All @@ -105,15 +123,17 @@ jobs:
echo "✅ Pull request created successfully!"

# Get PR URL and display it
PR_URL=$(gh pr view "${{ env.BRANCH_NAME }}" --json url --jq '.url')
PR_URL=$(gh pr view "$BRANCH_NAME" --json url --jq '.url')
echo "🔗 PR URL: $PR_URL"
echo "PR_URL=$PR_URL" >> $GITHUB_ENV

- name: Summary
env:
INPUTS_VERSION: ${{ inputs.version }}
run: |
echo "## ✅ Release Preparation Complete!" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "- **Version**: ${{ inputs.version }}" >> $GITHUB_STEP_SUMMARY
echo "- **Version**: $INPUTS_VERSION" >> $GITHUB_STEP_SUMMARY
echo "- **Branch**: ${{ env.BRANCH_NAME }}" >> $GITHUB_STEP_SUMMARY
echo "- **PR URL**: ${{ env.PR_URL }}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/pypi-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@ jobs:

- name: Extract version from release tag
id: extract_version
env:
GITHUB_EVENT_RELEASE_TAG_NAME: ${{ github.event.release.tag_name }}
run: |
# Get version from release tag (e.g., v1.2.3 -> 1.2.3)
if [[ "${{ github.event_name }}" == "release" ]]; then
VERSION="${{ github.event.release.tag_name }}"
VERSION="$GITHUB_EVENT_RELEASE_TAG_NAME"
VERSION="${VERSION#v}" # Remove 'v' prefix if present
else
# For manual dispatch, extract from pyproject.toml
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/release-binaries.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,16 @@ jobs:
steps:
- id: resolve
shell: bash
env:
GITHUB_EVENT_RELEASE_TAG_NAME: ${{ github.event.release.tag_name }}
INPUTS_RELEASE_TAG: ${{ inputs.release_tag }}
run: |
set -euo pipefail
if [[ "${{ github.event_name }}" == "release" ]]; then
TAG="${{ github.event.release.tag_name }}"
TAG="$GITHUB_EVENT_RELEASE_TAG_NAME"
VERSION="${TAG#v}"
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
TAG="${{ inputs.release_tag }}"
TAG="$INPUTS_RELEASE_TAG"
VERSION="${TAG#v}"
elif [[ "${{ github.event_name }}" == "push" ]]; then
TAG=""
Expand Down
Loading
Loading