Scheduled External Validation #620
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scheduled External Validation | |
| on: | |
| schedule: | |
| # Run every 6 hours to validate against external services | |
| - cron: "0 */6 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| test_servers: | |
| description: "Comma-separated list of MCP servers to test" | |
| required: false | |
| default: "" | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_LOG: info | |
| permissions: | |
| contents: write | |
| issues: write | |
| jobs: | |
| validate-external-servers: | |
| name: Validate External MCP Servers | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Build validation tools | |
| run: | | |
| cargo build --release --package pulseengine-mcp-external-validation | |
| - name: Validate known MCP servers | |
| run: | | |
| # Default test servers (can be overridden) | |
| if [ -n "${{ github.event.inputs.test_servers }}" ]; then | |
| IFS=',' read -ra SERVERS <<< "${{ github.event.inputs.test_servers }}" | |
| else | |
| # Use localhost examples only until proper test servers are available | |
| SERVERS=( | |
| "http://localhost:3000" | |
| ) | |
| fi | |
| # Create results directory | |
| mkdir -p validation-results | |
| # Test each server (skip if unreachable) | |
| for server in "${SERVERS[@]}"; do | |
| server=$(echo "$server" | xargs) # Trim whitespace | |
| echo "Testing $server..." | |
| # Generate safe filename | |
| filename=$(echo "$server" | sed 's/[^a-zA-Z0-9]/_/g') | |
| # Check if server is reachable first | |
| if [[ "$server" == http://localhost:* ]]; then | |
| echo "Skipping localhost validation (no server running in CI)" | |
| echo '{"server_url":"'$server'","status":"skipped","message":"localhost not available in CI"}' > "validation-results/${filename}.json" | |
| else | |
| # Run validation (output JSON format) | |
| ./target/release/mcp-validate --server-url "$server" \ | |
| --output json > "validation-results/${filename}.json" || echo '{"server_url":"'$server'","status":"failed","message":"validation failed"}' > "validation-results/${filename}.json" | |
| fi | |
| done | |
| - name: Generate summary report | |
| run: | | |
| # Create summary of all validations | |
| echo "# MCP Server Validation Summary" > validation-summary.md | |
| echo "" >> validation-summary.md | |
| echo "Validation run: $(date -u)" >> validation-summary.md | |
| echo "" >> validation-summary.md | |
| for result in validation-results/*.json; do | |
| if [ -f "$result" ]; then | |
| server_url=$(jq -r '.server_url' "$result" 2>/dev/null || echo "Unknown") | |
| status=$(jq -r '.status' "$result" 2>/dev/null || echo "Error") | |
| score=$(jq -r '.compliance_score // 0' "$result" 2>/dev/null || echo "0") | |
| echo "## $server_url" >> validation-summary.md | |
| echo "- Status: $status" >> validation-summary.md | |
| echo "- Compliance Score: $score%" >> validation-summary.md | |
| echo "" >> validation-summary.md | |
| fi | |
| done | |
| - name: Upload validation results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: validation-results-${{ github.run_id }} | |
| path: validation-results/ | |
| if-no-files-found: ignore | |
| - name: Create issue if failures detected | |
| if: failure() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const title = 'External MCP Server Validation Failures Detected'; | |
| const body = ` | |
| The scheduled validation workflow detected failures when testing external MCP servers. | |
| **Workflow Run:** [#${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| **Date:** ${new Date().toISOString()} | |
| Please check the validation results for details. | |
| `; | |
| // Check if issue already exists | |
| const issues = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| labels: 'validation-failure' | |
| }); | |
| const existingIssue = issues.data.find(issue => issue.title === title); | |
| if (!existingIssue) { | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: title, | |
| body: body, | |
| labels: ['validation-failure', 'automated'] | |
| }); | |
| } | |
| update-compatibility-matrix: | |
| name: Update Compatibility Matrix | |
| runs-on: ubuntu-latest | |
| needs: validate-external-servers | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download validation results | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| name: validation-results-${{ github.run_id }} | |
| path: validation-results/ | |
| - name: Generate compatibility matrix | |
| run: | | |
| # Create compatibility matrix markdown | |
| echo "# MCP Framework Compatibility Matrix" > COMPATIBILITY.md | |
| echo "" >> COMPATIBILITY.md | |
| echo "Last updated: $(date -u)" >> COMPATIBILITY.md | |
| echo "" >> COMPATIBILITY.md | |
| echo "| Server | Status | Compliance | Protocol | Transport | Tools | Resources |" >> COMPATIBILITY.md | |
| echo "|--------|--------|------------|----------|-----------|-------|-----------|" >> COMPATIBILITY.md | |
| if [ -d "validation-results" ] && [ "$(ls -A validation-results)" ]; then | |
| for result in validation-results/*.json; do | |
| if [ -f "$result" ]; then | |
| jq -r ' | |
| "| \(.server_url) " + | |
| "| \(.status) " + | |
| "| \(.compliance_score // 0)% " + | |
| "| \(.protocol_version // "N/A") " + | |
| "| \(.transport_compatible // false) " + | |
| "| \(.tools_compatible // false) " + | |
| "| \(.resources_compatible // false) |" | |
| ' "$result" >> COMPATIBILITY.md || true | |
| fi | |
| done | |
| else | |
| echo "| No validation results available | - | - | - | - | - | - |" >> COMPATIBILITY.md | |
| fi | |
| - name: Commit compatibility matrix | |
| uses: EndBug/add-and-commit@v9 | |
| with: | |
| add: "COMPATIBILITY.md" | |
| message: "Update compatibility matrix [skip ci]" | |
| default_author: github_actions |