diff --git a/.github/workflows/scheduled_runs.yml b/.github/workflows/scheduled_runs.yml index 5084e73fd09c..24d403f2293b 100644 --- a/.github/workflows/scheduled_runs.yml +++ b/.github/workflows/scheduled_runs.yml @@ -2,7 +2,18 @@ name: Scheduled Altinity Builds on: schedule: - - cron: '0 0 * * 6' #Weekly run for stable versions + - cron: '0 0 * * *' + workflow_run: + workflows: ["MasterCI", "ReleaseBranchCI"] + types: + - completed + branches: + - antalya-26.6 + - antalya-26.3 + - antalya-25.8 + - stable-26.3 + - stable-25.8 + - releases/24.8.14 # Make sure that any changes to this file is actually tested with PRs pull_request: types: @@ -11,46 +22,163 @@ on: - opened paths: - '**/scheduled_runs.yml' + - 'ci/jobs/scheduled_run_summary.py' + +permissions: + actions: write + contents: read jobs: - DailyRuns: + ScheduledRuns: + if: github.event_name != 'workflow_run' strategy: fail-fast: false matrix: - branch: - - antalya-26.6 - - antalya-26.3 - - antalya-25.8 + include: + - branch: antalya-26.6 + workflow: master.yml + - branch: antalya-26.3 + workflow: master.yml + - branch: antalya-25.8 + workflow: master.yml + - branch: stable-26.3 + workflow: master.yml + - branch: stable-25.8 + workflow: master.yml + - branch: releases/24.8.14 + workflow: release_branches.yml name: ${{ matrix.branch }} - runs-on: ubuntu-latest + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] steps: - - name: Run ${{ matrix.branch }} workflow - run: | - curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.TOKEN }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/Altinity/ClickHouse/actions/workflows/release_branches.yml/dispatches \ - -d '{"ref":"${{ matrix.branch }}"}' - - WeeklyRuns: - strategy: - fail-fast: false - matrix: - branch: - - stable-26.3 - - stable-25.8 - - releases/24.8.14 - name: ${{ matrix.branch }} - runs-on: ubuntu-latest + - name: Compare branch tip to the latest workflow run + id: check + env: + GH_TOKEN: ${{ github.token }} + BRANCH: ${{ matrix.branch }} + WORKFLOW: ${{ matrix.workflow }} + run: | + set -euo pipefail + + tip=$(gh api -X GET \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f sha="$BRANCH" \ + -F per_page=1 \ + /repos/Altinity/ClickHouse/commits \ + --jq '.[0].sha') + echo "Tip of ${BRANCH}: ${tip}" + + runs=$(gh api -X GET \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f branch="$BRANCH" \ + -F per_page=10 \ + "/repos/Altinity/ClickHouse/actions/workflows/${WORKFLOW}/runs") + + active=$(jq ' + [.workflow_runs[] + | select(.status == "queued" or .status == "in_progress" or .status == "waiting" or .status == "requested" or .status == "pending")] + | length' <<<"$runs") + latest_sha=$(jq -r '.workflow_runs[0].head_sha // empty' <<<"$runs") + latest_id=$(jq -r '.workflow_runs[0].id // empty' <<<"$runs") + latest_status=$(jq -r '.workflow_runs[0].status // empty' <<<"$runs") + latest_conclusion=$(jq -r '.workflow_runs[0].conclusion // empty' <<<"$runs") + + dispatch=false + previous_run_id="" + if [ "$active" -gt 0 ]; then + echo "${WORKFLOW} already has an unfinished run on ${BRANCH}" + elif [ -z "$latest_sha" ]; then + echo "No previous ${WORKFLOW} run on ${BRANCH}" + dispatch=true + elif [ "$latest_sha" = "$tip" ]; then + echo "No recent changes on ${BRANCH} (${latest_status}/${latest_conclusion})" + else + echo "Tip ${tip} differs from latest run ${latest_sha} (${latest_status}/${latest_conclusion})" + dispatch=true + previous_run_id="${latest_id}" + fi + { + echo "dispatch=${dispatch}" + echo "sha=${tip}" + echo "previous_run_id=${previous_run_id}" + } >> "$GITHUB_OUTPUT" + - name: Run ${{ matrix.workflow }} on ${{ matrix.branch }} + id: dispatch + if: steps.check.outputs.dispatch == 'true' + env: + GH_TOKEN: ${{ github.token }} + BRANCH: ${{ matrix.branch }} + WORKFLOW: ${{ matrix.workflow }} + SHA: ${{ steps.check.outputs.sha }} + PREVIOUS_RUN_ID: ${{ steps.check.outputs.previous_run_id }} + run: | + set -euo pipefail + before=$(date -u -d '30 seconds ago' +%Y-%m-%dT%H:%M:%SZ) + http=$(curl -sS -o /tmp/dispatch_body -w '%{http_code}' -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW}/dispatches" \ + -d "{\"ref\":\"${BRANCH}\"}") + if [ "$http" != "204" ]; then + echo "Dispatch failed: HTTP ${http}" + cat /tmp/dispatch_body + exit 1 + fi + + run_id="" + for _ in $(seq 1 24); do + sleep 5 + listed=$(gh run list \ + --repo "$GITHUB_REPOSITORY" \ + --workflow "$WORKFLOW" \ + --branch "$BRANCH" \ + --event workflow_dispatch \ + --limit 5 \ + --json databaseId,headSha,createdAt) + run_id=$(jq -r --arg sha "$SHA" --arg before "$before" ' + [ .[] | select(.headSha == $sha and .createdAt >= $before) ] + | sort_by(.createdAt) | last | .databaseId // empty + ' <<<"$listed") + if [ -n "$run_id" ]; then + echo "Found run ${run_id}" + break + fi + done + if [ -z "$run_id" ]; then + echo "Dispatched ${WORKFLOW} on ${BRANCH} but no run id appeared within 2 minutes" + exit 1 + fi + echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" + jq -n \ + --arg previous_run_id "$PREVIOUS_RUN_ID" \ + '{previous_run_id: (if $previous_run_id == "" then null else ($previous_run_id | tonumber) end)}' \ + > entry.json + - name: Upload batch entry + if: steps.dispatch.outputs.run_id != '' + uses: actions/upload-artifact@v7 + with: + name: batch-entry-${{ steps.dispatch.outputs.run_id }} + path: entry.json + retention-days: 5 + + ReportScheduledRuns: + if: github.event_name == 'workflow_run' + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] steps: - - name: Run ${{ matrix.branch }} workflow - run: | - curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.TOKEN }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/Altinity/ClickHouse/actions/workflows/release_branches.yml/dispatches \ - -d '{"ref":"${{ matrix.branch }}"}' + - name: Checkout summary script + uses: actions/checkout@v6 + with: + sparse-checkout: ci/jobs/scheduled_run_summary.py + sparse-checkout-cone-mode: false + - name: Post Slack summary + env: + GH_TOKEN: ${{ github.token }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id || '' }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + run: python3 ci/jobs/scheduled_run_summary.py diff --git a/ci/jobs/scheduled_run_summary.py b/ci/jobs/scheduled_run_summary.py new file mode 100644 index 000000000000..2d7a010f133c --- /dev/null +++ b/ci/jobs/scheduled_run_summary.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""Post one Slack message for a nightly CI run that has failures. + +Called from the workflow_run job. Exits without posting when the finished run +was not dispatched by the midnight schedule, or when none of the tracked job +groups failed. +""" + +import json +import os +import subprocess +import sys +import tempfile +import urllib.parse +import urllib.request +from pathlib import Path + +GROUPS = ( + ("Build", "Build ("), + ("Stateless", "Stateless tests"), + ("Stateful", "Stateful tests"), + ("Integration", "Integration tests"), + ("Regression", "Regression"), + ("Grype", "GrypeScan"), +) + + +def gh(path, params=None): + if params: + path += "?" + urllib.parse.urlencode(params) + raw = subprocess.check_output(["gh", "api", path], text=True) + return json.loads(raw) + + +def jobs(repo, run_id): + found = [] + page = 1 + while page <= 20: + payload = gh( + f"/repos/{repo}/actions/runs/{run_id}/jobs", + {"per_page": 100, "page": page, "filter": "latest"}, + ) + chunk = payload.get("jobs") or [] + found.extend(chunk) + if not chunk or len(found) >= (payload.get("total_count") or 0): + break + page += 1 + return found + + +def failed_by_group(job_list): + grouped = {label: [] for label, _prefix in GROUPS} + for job in job_list: + if job.get("conclusion") != "failure": + continue + name = job.get("name") or "" + for label, prefix in GROUPS: + if name.startswith(prefix): + grouped[label].append(name) + break + return grouped + + +def fail_suffix(names, previous_names): + if previous_names is None: + return "" + new = len(set(names) - set(previous_names)) + if new == 0: + return ", same as previous" + return f", {new} new" + + +def format_message(branch, sha, run_id, grouped, previous): + """previous is None when this branch has no earlier completed run.""" + lines = [] + for label, _prefix in GROUPS: + names = grouped[label] + if not names: + continue + suffix = fail_suffix(names, None if previous is None else previous.get(label, [])) + lines.append(f"{label}: {len(names)} failed{suffix}") + if not lines: + return None + short = (sha or "")[:7] + header = f"{branch} · {short}" if short else branch + report = ( + "https://s3.amazonaws.com/altinity-build-artifacts/" + f"REFs/{branch}/{sha}/{run_id}/ci_run_report.html" + ) + return "\n".join([header, *lines, f"Report: {report}"]) + + +def previous_groups(repo, previous_run_id): + """Failures from the run id saved when this nightly run was dispatched.""" + if not previous_run_id: + return None + return failed_by_group(jobs(repo, previous_run_id)) + + +def find_entry(repo, run_id): + """Load the dispatch record uploaded as batch-entry-.""" + name = f"batch-entry-{run_id}" + payload = gh(f"/repos/{repo}/actions/artifacts", {"name": name, "per_page": 5}) + artifact = next( + ( + item + for item in payload.get("artifacts") or [] + if item.get("name") == name and not item.get("expired") + ), + None, + ) + if artifact is None: + return None + with tempfile.TemporaryDirectory() as tmp: + subprocess.run( + [ + "gh", "run", "download", str(artifact["workflow_run"]["id"]), + "-n", name, + "-D", tmp, + ], + check=True, + capture_output=True, + text=True, + ) + path = next(Path(tmp).rglob("*.json"), None) + if path is None: + raise RuntimeError(f"{name} downloaded but contained no json") + return json.loads(path.read_text(encoding="utf-8")) + + +def write_summary(text): + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + with open(path, "a", encoding="utf-8") as handle: + handle.write(text) + if not text.endswith("\n"): + handle.write("\n") + + +def post_slack(text): + body = json.dumps({ + "channel": os.environ["SLACK_CHANNEL_ID"], + "text": text, + }).encode("utf-8") + request = urllib.request.Request( + "https://slack.com/api/chat.postMessage", + data=body, + headers={ + "Authorization": f"Bearer {os.environ['SLACK_BOT_TOKEN']}", + "Content-Type": "application/json; charset=utf-8", + }, + ) + with urllib.request.urlopen(request, timeout=30) as response: + payload = json.load(response) + if not payload.get("ok"): + raise RuntimeError(f"Slack chat.postMessage failed: {payload.get('error')}") + + +def report(): + run_id = int(os.environ["TRIGGERING_RUN_ID"]) + repo = os.environ["GITHUB_REPOSITORY"] + entry = find_entry(repo, run_id) + if entry is None: + note = f"run {run_id} was not dispatched by the nightly schedule" + print(note) + write_summary(note) + return + grouped = failed_by_group(jobs(repo, run_id)) + if not any(grouped.values()): + note = f"run {run_id}: no failures" + print(note) + write_summary(note) + return + previous = None + if entry.get("previous_run_id"): + try: + previous = previous_groups(repo, entry["previous_run_id"]) + except (subprocess.CalledProcessError, RuntimeError) as exc: + print(f"previous run lookup failed: {exc}") + text = format_message(os.environ["HEAD_BRANCH"], os.environ["HEAD_SHA"], run_id, grouped, previous) + print(text) + write_summary(text) + if not os.environ.get("SLACK_BOT_TOKEN") or not os.environ.get("SLACK_CHANNEL_ID"): + print("Slack secrets are not set, skipped posting") + return + post_slack(text) + print(f"posted run {run_id}") + + +def self_check(): + grouped = failed_by_group([ + {"name": "Integration tests (amd_asan, 1/4)", "conclusion": "failure"}, + {"name": "Integration tests (amd_asan, 2/4)", "conclusion": "failure"}, + {"name": "Stateless tests (amd_debug, parallel)", "conclusion": "success"}, + {"name": "Build (amd_release)", "conclusion": "skipped"}, + {"name": "Stress test (amd_tsan)", "conclusion": "failure"}, + ]) + text = format_message( + "antalya-26.6", + "abc1234ffff", + 7, + grouped, + {"Integration": {"Integration tests (amd_asan, 1/4)"}}, + ) + assert text.startswith("antalya-26.6 · abc1234") + assert "Integration: 2 failed, 1 new" in text + assert "Stateless" not in text + assert "Build" not in text + assert "Stress" not in text + assert "/7/ci_run_report.html" in text + + same = format_message( + "antalya-26.6", + "abc1234ffff", + 8, + failed_by_group([{"name": "GrypeScanServer", "conclusion": "failure"}]), + {"Grype": {"GrypeScanServer"}}, + ) + assert "Grype: 1 failed, same as previous" in same + + first = format_message( + "antalya-26.6", + "abc1234ffff", + 8, + failed_by_group([{"name": "RegressionTestsRelease", "conclusion": "failure"}]), + None, + ) + assert "Regression: 1 failed\n" in first + assert "new" not in first + + stateful = format_message( + "antalya-26.6", + "abc1234ffff", + 9, + failed_by_group([{"name": "Stateful tests (amd_debug)", "conclusion": "failure"}]), + None, + ) + assert "Stateful: 1 failed\n" in stateful + + assert format_message("antalya-26.3", "abcdef0", 5, failed_by_group([]), None) is None + print("self-check ok") + + +if __name__ == "__main__": + try: + if len(sys.argv) > 1 and sys.argv[1] == "self-check": + self_check() + else: + report() + except Exception: + import traceback + traceback.print_exc() + sys.exit(1)