From d23dfed97f7589aaeb564bcd5ab9291ae7cc2492 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 28 Jun 2026 19:08:18 +0000 Subject: [PATCH 01/10] Add Xcode Cloud screenshot fetch scripts and tests Introduce a Python CLI that downloads Xcode Cloud test result bundles via the App Store Connect API, extracts UI test screenshots with xcresulttool when available, and can post a summary comment to GitHub PRs. Add ci_post_xcodebuild.sh so extraction can run directly on the Xcode Cloud Mac using CI_RESULT_BUNDLE_PATH, avoiding a separate macOS runner. Include pytest coverage with mocked API responses for the fetch path. Co-authored-by: Greg --- Bedtime/ci_scripts/ci_post_xcodebuild.sh | 43 +++++ scripts/.gitignore | 4 + scripts/README.md | 100 ++++++++++ scripts/fetch_xcode_cloud_screenshots.py | 173 ++++++++++++++++++ scripts/pytest.ini | 3 + scripts/requirements-dev.txt | 4 + scripts/tests/conftest.py | 15 ++ scripts/tests/fixtures/artifact_detail.json | 11 ++ scripts/tests/fixtures/artifacts_list.json | 20 ++ scripts/tests/fixtures/build_actions.json | 20 ++ .../tests/fixtures/build_run_complete.json | 10 + scripts/tests/test_asc_auth.py | 26 +++ scripts/tests/test_client.py | 37 ++++ scripts/tests/test_extract.py | 38 ++++ scripts/tests/test_github_pr.py | 17 ++ scripts/tests/test_screenshots.py | 67 +++++++ scripts/xcode_cloud/__init__.py | 1 + scripts/xcode_cloud/asc_auth.py | 59 ++++++ scripts/xcode_cloud/client.py | 94 ++++++++++ scripts/xcode_cloud/extract.py | 49 +++++ scripts/xcode_cloud/github_pr.py | 60 ++++++ scripts/xcode_cloud/screenshots.py | 157 ++++++++++++++++ 22 files changed, 1008 insertions(+) create mode 100755 Bedtime/ci_scripts/ci_post_xcodebuild.sh create mode 100644 scripts/.gitignore create mode 100644 scripts/README.md create mode 100644 scripts/fetch_xcode_cloud_screenshots.py create mode 100644 scripts/pytest.ini create mode 100644 scripts/requirements-dev.txt create mode 100644 scripts/tests/conftest.py create mode 100644 scripts/tests/fixtures/artifact_detail.json create mode 100644 scripts/tests/fixtures/artifacts_list.json create mode 100644 scripts/tests/fixtures/build_actions.json create mode 100644 scripts/tests/fixtures/build_run_complete.json create mode 100644 scripts/tests/test_asc_auth.py create mode 100644 scripts/tests/test_client.py create mode 100644 scripts/tests/test_extract.py create mode 100644 scripts/tests/test_github_pr.py create mode 100644 scripts/tests/test_screenshots.py create mode 100644 scripts/xcode_cloud/__init__.py create mode 100644 scripts/xcode_cloud/asc_auth.py create mode 100644 scripts/xcode_cloud/client.py create mode 100644 scripts/xcode_cloud/extract.py create mode 100644 scripts/xcode_cloud/github_pr.py create mode 100644 scripts/xcode_cloud/screenshots.py diff --git a/Bedtime/ci_scripts/ci_post_xcodebuild.sh b/Bedtime/ci_scripts/ci_post_xcodebuild.sh new file mode 100755 index 0000000..9bbc244 --- /dev/null +++ b/Bedtime/ci_scripts/ci_post_xcodebuild.sh @@ -0,0 +1,43 @@ +#!/bin/sh +set -eu + +# Runs on the Xcode Cloud Mac after xcodebuild finishes. +# Uses the local result bundle directly — no App Store Connect API download needed. + +OUTPUT_DIR="${CI_DERIVED_DATA_PATH:-/tmp}/xcode-cloud-screenshots" +ONLY_FAILURES="${XCODE_CLOUD_SCREENSHOT_ONLY_FAILURES:-false}" + +if [ -z "${CI_RESULT_BUNDLE_PATH:-}" ]; then + echo "ci_post_xcodebuild: CI_RESULT_BUNDLE_PATH is not set; skipping screenshot export." + exit 0 +fi + +if [ ! -e "$CI_RESULT_BUNDLE_PATH" ]; then + echo "ci_post_xcodebuild: result bundle not found at $CI_RESULT_BUNDLE_PATH" + exit 0 +fi + +mkdir -p "$OUTPUT_DIR" + +ARGS="extract-local --bundle-path \"$CI_RESULT_BUNDLE_PATH\" --output-dir \"$OUTPUT_DIR\"" +if [ "$ONLY_FAILURES" = "true" ]; then + ARGS="$ARGS --only-failures" +fi + +REPO_ROOT="${CI_PRIMARY_REPOSITORY_PATH:-$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)}" +PYTHON_BIN="${PYTHON_BIN:-python3}" + +echo "ci_post_xcodebuild: exporting screenshots from $CI_RESULT_BUNDLE_PATH" +# shellcheck disable=SC2086 +"$PYTHON_BIN" "$REPO_ROOT/scripts/fetch_xcode_cloud_screenshots.py" $ARGS + +if [ -n "${GITHUB_REPOSITORY:-}" ] && [ -n "${GITHUB_PULL_REQUEST:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then + echo "ci_post_xcodebuild: posting screenshot summary to PR #${GITHUB_PULL_REQUEST}" + "$PYTHON_BIN" "$REPO_ROOT/scripts/fetch_xcode_cloud_screenshots.py" comment-pr \ + --repo "$GITHUB_REPOSITORY" \ + --pr-number "$GITHUB_PULL_REQUEST" \ + --run-id "${CI_BUILD_ID:-unknown}" \ + --screenshots-dir "$OUTPUT_DIR/screenshots" +fi + +echo "ci_post_xcodebuild: screenshots available in $OUTPUT_DIR/screenshots" diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000..0921fae --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1,4 @@ +xcode-cloud-output/ +__pycache__/ +.pytest_cache/ +*.pyc diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..d9693e2 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,100 @@ +# Xcode Cloud screenshot fetcher + +Fetch UI test screenshots from Xcode Cloud after a build completes. + +## Why not a GitHub Action? + +A GitHub Action is **optional**, not required. It was only suggested earlier because `xcresulttool` needs macOS. + +This repo uses two paths instead: + +1. **`Bedtime/ci_scripts/ci_post_xcodebuild.sh`** (recommended on Xcode Cloud) + Runs on Apple's Mac right after tests finish. It reads `CI_RESULT_BUNDLE_PATH` directly, so there is no API round-trip and no second macOS runner. + +2. **`scripts/fetch_xcode_cloud_screenshots.py fetch`** + For webhook handlers, local machines, or Cursor Cloud Agents. Downloads the test result bundle via the App Store Connect API, then extracts screenshots if `xcresulttool` is available. + +Use a GitHub Action only if you want extraction to happen outside Xcode Cloud. + +## Setup + +```bash +cd scripts +python3 -m pip install -r requirements-dev.txt +``` + +### Credentials (Runtime Secrets in Cursor, or local env) + +```bash +export APP_STORE_CONNECT_KEY_ID="..." +export APP_STORE_CONNECT_ISSUER_ID="..." +export APP_STORE_CONNECT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" +``` + +Optional for PR comments from `ci_post_xcodebuild.sh`: + +```bash +export GITHUB_TOKEN="..." +export GITHUB_REPOSITORY="owner/repo" +export GITHUB_PULL_REQUEST="123" +``` + +## Usage + +Poll until complete, then fetch and extract: + +```bash +python3 scripts/fetch_xcode_cloud_screenshots.py wait-and-fetch --run-id BUILD_RUN_ID +``` + +Fetch a completed build: + +```bash +python3 scripts/fetch_xcode_cloud_screenshots.py fetch --run-id BUILD_RUN_ID +``` + +Download only (skip extraction on Linux): + +```bash +python3 scripts/fetch_xcode_cloud_screenshots.py fetch --run-id BUILD_RUN_ID --skip-extract +``` + +Extract from a local bundle (Xcode Cloud Mac or your laptop): + +```bash +python3 scripts/fetch_xcode_cloud_screenshots.py extract-local \ + --bundle-path /path/to/resultbundle.xcresult +``` + +Post a PR summary comment: + +```bash +python3 scripts/fetch_xcode_cloud_screenshots.py comment-pr \ + --repo owner/repo \ + --pr-number 123 \ + --run-id BUILD_RUN_ID \ + --screenshots-dir ./xcode-cloud-output/screenshots +``` + +## Tests + +```bash +cd scripts +python3 -m pytest +``` + +## Flow + +```text +Xcode Cloud test action finishes + ├─ ci_post_xcodebuild.sh (on Mac) + │ └─ extract from CI_RESULT_BUNDLE_PATH + │ + └─ webhook / manual fetch (anywhere) + └─ App Store Connect API → temporary downloadUrl + └─ download .xcresult + └─ xcresulttool export attachments (macOS) + └─ optional GitHub PR comment +``` + +Apple's `downloadUrl` values are short-lived. Do not embed them directly in PRs. Extract PNGs and upload or reference stable assets instead. diff --git a/scripts/fetch_xcode_cloud_screenshots.py b/scripts/fetch_xcode_cloud_screenshots.py new file mode 100644 index 0000000..aca6a45 --- /dev/null +++ b/scripts/fetch_xcode_cloud_screenshots.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Fetch Xcode Cloud screenshots after a build run completes.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.xcode_cloud.asc_auth import create_asc_token, credentials_from_env +from scripts.xcode_cloud.client import XcodeCloudClient +from scripts.xcode_cloud.extract import XcresultToolNotFoundError +from scripts.xcode_cloud.github_pr import build_screenshot_comment, post_pr_comment +from scripts.xcode_cloud.screenshots import ( + extract_screenshots_from_local_bundle, + fetch_screenshots_from_build_run, +) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Fetch Xcode Cloud test screenshots after a build completes." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + fetch_parser = subparsers.add_parser( + "fetch", + help="Download the test result bundle for a completed build run.", + ) + fetch_parser.add_argument("--run-id", required=True, help="ciBuildRuns ID") + fetch_parser.add_argument( + "--output-dir", + default="./xcode-cloud-output", + help="Directory for downloaded bundles and screenshots", + ) + fetch_parser.add_argument( + "--only-failures", + action="store_true", + help="Export only attachments associated with failing tests", + ) + fetch_parser.add_argument( + "--skip-extract", + action="store_true", + help="Download the bundle but do not run xcresulttool extraction", + ) + + wait_parser = subparsers.add_parser( + "wait-and-fetch", + help="Poll until the build run completes, then fetch screenshots.", + ) + wait_parser.add_argument("--run-id", required=True, help="ciBuildRuns ID") + wait_parser.add_argument( + "--output-dir", + default="./xcode-cloud-output", + help="Directory for downloaded bundles and screenshots", + ) + wait_parser.add_argument("--timeout-seconds", type=int, default=3600) + wait_parser.add_argument("--poll-interval-seconds", type=int, default=30) + wait_parser.add_argument("--only-failures", action="store_true") + + local_parser = subparsers.add_parser( + "extract-local", + help="Extract screenshots from a local .xcresult bundle.", + ) + local_parser.add_argument("--bundle-path", required=True) + local_parser.add_argument( + "--output-dir", + default="./xcode-cloud-output", + help="Directory for extracted screenshots", + ) + local_parser.add_argument("--only-failures", action="store_true") + + comment_parser = subparsers.add_parser( + "comment-pr", + help="Post a screenshot summary comment to a GitHub pull request.", + ) + comment_parser.add_argument("--repo", required=True, help="owner/repo") + comment_parser.add_argument("--pr-number", type=int, required=True) + comment_parser.add_argument("--run-id", required=True) + comment_parser.add_argument( + "--screenshots-dir", + required=True, + help="Directory containing extracted .png files", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + if args.command == "extract-local": + screenshots = extract_screenshots_from_local_bundle( + Path(args.bundle_path), + Path(args.output_dir), + only_failures=args.only_failures, + ) + _print_screenshots(screenshots) + return 0 + + if args.command == "comment-pr": + import os + + token = os.environ.get("GITHUB_TOKEN") + if not token: + parser.error("GITHUB_TOKEN is required for comment-pr") + + screenshots_dir = Path(args.screenshots_dir) + screenshots = sorted(screenshots_dir.rglob("*.png")) + body = build_screenshot_comment(screenshots, build_run_id=args.run_id) + post_pr_comment(args.repo, args.pr_number, body, token=token) + print(f"Posted PR comment to {args.repo}#{args.pr_number}") + return 0 + + credentials = credentials_from_env() + output_dir = Path(args.output_dir) + + with XcodeCloudClient(lambda: create_asc_token(credentials)) as client: + if args.command == "wait-and-fetch": + status = client.wait_for_build_run( + args.run_id, + timeout_seconds=args.timeout_seconds, + poll_interval_seconds=args.poll_interval_seconds, + ) + print( + f"Build run {status.run_id} completed with status " + f"{status.completion_status or 'UNKNOWN'}" + ) + + if args.command == "fetch" and args.skip_extract: + from scripts.xcode_cloud.screenshots import fetch_test_result_bundle + + _, artifact_id, bundle_path = fetch_test_result_bundle( + client, + args.run_id, + output_dir, + ) + print(f"Downloaded artifact {artifact_id} to {bundle_path}") + return 0 + + try: + result = fetch_screenshots_from_build_run( + client, + args.run_id, + output_dir, + only_failures=getattr(args, "only_failures", False), + ) + except XcresultToolNotFoundError as error: + print(str(error), file=sys.stderr) + return 2 + + print(f"Test action: {result.test_action_id}") + print(f"Artifact: {result.artifact_id}") + print(f"Bundle: {result.bundle_path}") + _print_screenshots(result.screenshot_paths) + return 0 + + +def _print_screenshots(screenshots: list[Path] | tuple[Path, ...]) -> None: + if not screenshots: + print("No screenshots found.") + return + print(f"Extracted {len(screenshots)} screenshot(s):") + for path in screenshots: + print(path) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/pytest.ini b/scripts/pytest.ini new file mode 100644 index 0000000..5cc30ab --- /dev/null +++ b/scripts/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tests +pythonpath = .. diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt new file mode 100644 index 0000000..7de14c9 --- /dev/null +++ b/scripts/requirements-dev.txt @@ -0,0 +1,4 @@ +pytest>=8.0 +httpx>=0.27 +PyJWT[crypto]>=2.8 +cryptography>=42.0 diff --git a/scripts/tests/conftest.py b/scripts/tests/conftest.py new file mode 100644 index 0000000..66f7988 --- /dev/null +++ b/scripts/tests/conftest.py @@ -0,0 +1,15 @@ +import json +from pathlib import Path + +import pytest + + +FIXTURES = Path(__file__).parent / "fixtures" + + +@pytest.fixture +def load_fixture(): + def _load(name: str): + return json.loads((FIXTURES / name).read_text()) + + return _load diff --git a/scripts/tests/fixtures/artifact_detail.json b/scripts/tests/fixtures/artifact_detail.json new file mode 100644 index 0000000..25513ac --- /dev/null +++ b/scripts/tests/fixtures/artifact_detail.json @@ -0,0 +1,11 @@ +{ + "data": { + "type": "ciArtifacts", + "id": "artifact-test-1", + "attributes": { + "fileType": "TEST_RESULT_BUNDLE", + "fileName": "TestResults.zip", + "downloadUrl": "https://example.com/TestResults.zip" + } + } +} diff --git a/scripts/tests/fixtures/artifacts_list.json b/scripts/tests/fixtures/artifacts_list.json new file mode 100644 index 0000000..adf2813 --- /dev/null +++ b/scripts/tests/fixtures/artifacts_list.json @@ -0,0 +1,20 @@ +{ + "data": [ + { + "type": "ciArtifacts", + "id": "artifact-log-1", + "attributes": { + "fileType": "LOG_BUNDLE", + "fileName": "logs.zip" + } + }, + { + "type": "ciArtifacts", + "id": "artifact-test-1", + "attributes": { + "fileType": "TEST_RESULT_BUNDLE", + "fileName": "TestResults.zip" + } + } + ] +} diff --git a/scripts/tests/fixtures/build_actions.json b/scripts/tests/fixtures/build_actions.json new file mode 100644 index 0000000..3ffeb2b --- /dev/null +++ b/scripts/tests/fixtures/build_actions.json @@ -0,0 +1,20 @@ +{ + "data": [ + { + "type": "ciBuildActions", + "id": "action-build-1", + "attributes": { + "name": "build", + "actionType": "ARCHIVE" + } + }, + { + "type": "ciBuildActions", + "id": "action-test-1", + "attributes": { + "name": "test", + "actionType": "TEST" + } + } + ] +} diff --git a/scripts/tests/fixtures/build_run_complete.json b/scripts/tests/fixtures/build_run_complete.json new file mode 100644 index 0000000..2d89720 --- /dev/null +++ b/scripts/tests/fixtures/build_run_complete.json @@ -0,0 +1,10 @@ +{ + "data": { + "type": "ciBuildRuns", + "id": "build-run-123", + "attributes": { + "executionProgress": "COMPLETE", + "completionStatus": "SUCCEEDED" + } + } +} diff --git a/scripts/tests/test_asc_auth.py b/scripts/tests/test_asc_auth.py new file mode 100644 index 0000000..5ebed07 --- /dev/null +++ b/scripts/tests/test_asc_auth.py @@ -0,0 +1,26 @@ +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec + +from scripts.xcode_cloud.asc_auth import AscCredentials, create_asc_token + + +def _generate_test_key() -> str: + private_key = ec.generate_private_key(ec.SECP256R1()) + return private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode() + + +def test_create_asc_token_contains_expected_claims(): + credentials = AscCredentials( + key_id="KEY123", + issuer_id="issuer-uuid", + private_key=_generate_test_key(), + ) + + token = create_asc_token(credentials, expiration_seconds=600) + + assert isinstance(token, str) + assert token.count(".") == 2 diff --git a/scripts/tests/test_client.py b/scripts/tests/test_client.py new file mode 100644 index 0000000..aa4a833 --- /dev/null +++ b/scripts/tests/test_client.py @@ -0,0 +1,37 @@ +import json + +import httpx + +from scripts.xcode_cloud.client import XcodeCloudClient + + +def test_get_build_run_status(load_fixture): + fixture = load_fixture("build_run_complete.json") + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path.endswith("/v1/ciBuildRuns/build-run-123") + return httpx.Response(200, json=fixture) + + client = httpx.Client(transport=httpx.MockTransport(handler)) + api = XcodeCloudClient(lambda: "token", client=client) + + status = api.get_build_run_status("build-run-123") + + assert status.execution_progress == "COMPLETE" + assert status.completion_status == "SUCCEEDED" + + +def test_list_build_actions(load_fixture): + fixture = load_fixture("build_actions.json") + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path.endswith("/v1/ciBuildRuns/build-run-123/actions") + return httpx.Response(200, json=fixture) + + client = httpx.Client(transport=httpx.MockTransport(handler)) + api = XcodeCloudClient(lambda: "token", client=client) + + actions = api.list_build_actions("build-run-123") + + assert len(actions) == 2 + assert actions[1]["attributes"]["actionType"] == "TEST" diff --git a/scripts/tests/test_extract.py b/scripts/tests/test_extract.py new file mode 100644 index 0000000..53c5c66 --- /dev/null +++ b/scripts/tests/test_extract.py @@ -0,0 +1,38 @@ +from pathlib import Path +from unittest.mock import patch + +import pytest + +from scripts.xcode_cloud.extract import ( + XcresultToolNotFoundError, + extract_attachments, +) + + +def test_extract_attachments_runs_xcresulttool(tmp_path): + bundle_path = tmp_path / "Test.xcresult" + bundle_path.mkdir() + output_dir = tmp_path / "out" + screenshot = output_dir / "shot.png" + + def fake_run(command, check, capture_output, text): + assert command[0] == "/usr/bin/xcresulttool" + assert command[1:4] == ["export", "attachments", "--path"] + output_dir.mkdir(parents=True, exist_ok=True) + screenshot.write_bytes(b"png") + return None + + with patch("scripts.xcode_cloud.extract.xcresulttool_path", return_value="/usr/bin/xcresulttool"): + with patch("scripts.xcode_cloud.extract.subprocess.run", side_effect=fake_run): + paths = extract_attachments(bundle_path, output_dir) + + assert paths == [screenshot] + + +def test_extract_attachments_requires_xcresulttool(tmp_path): + bundle_path = tmp_path / "Test.xcresult" + bundle_path.mkdir() + + with patch("scripts.xcode_cloud.extract.xcresulttool_path", return_value=None): + with pytest.raises(XcresultToolNotFoundError): + extract_attachments(bundle_path, tmp_path / "out") diff --git a/scripts/tests/test_github_pr.py b/scripts/tests/test_github_pr.py new file mode 100644 index 0000000..3796629 --- /dev/null +++ b/scripts/tests/test_github_pr.py @@ -0,0 +1,17 @@ +from pathlib import Path + +from scripts.xcode_cloud.github_pr import build_screenshot_comment + + +def test_build_screenshot_comment_lists_files(tmp_path): + shots = [tmp_path / "home.png", tmp_path / "settings.png"] + body = build_screenshot_comment(shots, build_run_id="run-1") + + assert "run-1" in body + assert "home.png" in body + assert "settings.png" in body + + +def test_build_screenshot_comment_handles_empty_list(): + body = build_screenshot_comment([], build_run_id="run-1") + assert "no screenshot attachments" in body.lower() diff --git a/scripts/tests/test_screenshots.py b/scripts/tests/test_screenshots.py new file mode 100644 index 0000000..6157cd8 --- /dev/null +++ b/scripts/tests/test_screenshots.py @@ -0,0 +1,67 @@ +import io +import zipfile +from pathlib import Path + +import httpx +import pytest + +from scripts.xcode_cloud.client import XcodeCloudClient, XcodeCloudError +from scripts.xcode_cloud.screenshots import ( + fetch_test_result_bundle, + find_test_action, + find_test_result_artifact, +) + + +def test_find_test_action(load_fixture): + actions = load_fixture("build_actions.json")["data"] + action = find_test_action(actions) + assert action["id"] == "action-test-1" + + +def test_find_test_result_artifact(load_fixture): + artifacts = load_fixture("artifacts_list.json")["data"] + artifact = find_test_result_artifact(artifacts) + assert artifact["id"] == "artifact-test-1" + + +def test_find_test_action_raises_when_missing(): + with pytest.raises(XcodeCloudError, match="No TEST build action"): + find_test_action([]) + + +def test_fetch_test_result_bundle_downloads_zip(tmp_path, load_fixture): + actions_fixture = load_fixture("build_actions.json") + artifacts_fixture = load_fixture("artifacts_list.json") + artifact_fixture = load_fixture("artifact_detail.json") + + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, "w") as archive: + archive.writestr("Test.xcresult/Info.plist", "plist") + zip_bytes = zip_buffer.getvalue() + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/actions"): + return httpx.Response(200, json=actions_fixture) + if request.url.path.endswith("/artifacts") and "ciBuildActions" in request.url.path: + return httpx.Response(200, json=artifacts_fixture) + if request.url.path.endswith("/ciArtifacts/artifact-test-1"): + return httpx.Response(200, json=artifact_fixture) + if request.url.host == "example.com": + return httpx.Response(200, content=zip_bytes) + raise AssertionError(f"Unexpected request: {request.url}") + + api_client = httpx.Client(transport=httpx.MockTransport(handler)) + download_client = httpx.Client(transport=httpx.MockTransport(handler)) + api = XcodeCloudClient(lambda: "token", client=api_client) + + _, artifact_id, bundle_path = fetch_test_result_bundle( + api, + "build-run-123", + tmp_path, + download_client=download_client, + ) + + assert artifact_id == "artifact-test-1" + assert bundle_path.suffix == ".xcresult" + assert bundle_path.exists() diff --git a/scripts/xcode_cloud/__init__.py b/scripts/xcode_cloud/__init__.py new file mode 100644 index 0000000..93ee89a --- /dev/null +++ b/scripts/xcode_cloud/__init__.py @@ -0,0 +1 @@ +"""Fetch Xcode Cloud test artifacts and extract UI test screenshots.""" diff --git a/scripts/xcode_cloud/asc_auth.py b/scripts/xcode_cloud/asc_auth.py new file mode 100644 index 0000000..3a63dd5 --- /dev/null +++ b/scripts/xcode_cloud/asc_auth.py @@ -0,0 +1,59 @@ +"""App Store Connect API JWT helpers.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass + +import jwt + + +@dataclass(frozen=True) +class AscCredentials: + key_id: str + issuer_id: str + private_key: str + + +def create_asc_token( + credentials: AscCredentials, + *, + expiration_seconds: int = 1200, +) -> str: + """Create a short-lived JWT for App Store Connect API requests.""" + now = int(time.time()) + headers = {"alg": "ES256", "kid": credentials.key_id, "typ": "JWT"} + payload = { + "iss": credentials.issuer_id, + "iat": now, + "exp": now + expiration_seconds, + "aud": "appstoreconnect-v1", + } + return jwt.encode(payload, credentials.private_key, algorithm="ES256", headers=headers) + + +def credentials_from_env() -> AscCredentials: + """Load App Store Connect credentials from standard environment variables.""" + import os + + key_id = os.environ.get("APP_STORE_CONNECT_KEY_ID") + issuer_id = os.environ.get("APP_STORE_CONNECT_ISSUER_ID") + private_key = os.environ.get("APP_STORE_CONNECT_PRIVATE_KEY") + + missing = [ + name + for name, value in ( + ("APP_STORE_CONNECT_KEY_ID", key_id), + ("APP_STORE_CONNECT_ISSUER_ID", issuer_id), + ("APP_STORE_CONNECT_PRIVATE_KEY", private_key), + ) + if not value + ] + if missing: + raise ValueError(f"Missing required environment variables: {', '.join(missing)}") + + return AscCredentials( + key_id=key_id, + issuer_id=issuer_id, + private_key=private_key.replace("\\n", "\n"), + ) diff --git a/scripts/xcode_cloud/client.py b/scripts/xcode_cloud/client.py new file mode 100644 index 0000000..56fe852 --- /dev/null +++ b/scripts/xcode_cloud/client.py @@ -0,0 +1,94 @@ +"""App Store Connect API client for Xcode Cloud resources.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Any, Callable +from urllib.parse import urljoin + +import httpx + + +class XcodeCloudError(RuntimeError): + """Raised when an Xcode Cloud API request fails.""" + + +@dataclass(frozen=True) +class BuildRunStatus: + run_id: str + execution_progress: str + completion_status: str | None + + +class XcodeCloudClient: + BASE_URL = "https://api.appstoreconnect.apple.com/v1/" + + def __init__( + self, + token_provider: Callable[[], str], + *, + client: httpx.Client | None = None, + ) -> None: + self._token_provider = token_provider + self._client = client or httpx.Client(timeout=60.0) + + def close(self) -> None: + self._client.close() + + def __enter__(self) -> XcodeCloudClient: + return self + + def __exit__(self, *args: object) -> None: + self.close() + + def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: + url = path if path.startswith("http") else urljoin(self.BASE_URL, path.lstrip("/")) + headers = kwargs.pop("headers", {}) + headers["Authorization"] = f"Bearer {self._token_provider()}" + response = self._client.request(method, url, headers=headers, **kwargs) + if response.status_code >= 400: + raise XcodeCloudError( + f"{method} {url} failed with {response.status_code}: {response.text}" + ) + if not response.content: + return {} + return response.json() + + def get_build_run(self, run_id: str) -> dict[str, Any]: + return self._request("GET", f"ciBuildRuns/{run_id}") + + def list_build_actions(self, run_id: str) -> list[dict[str, Any]]: + payload = self._request("GET", f"ciBuildRuns/{run_id}/actions") + return payload.get("data", []) + + def list_artifacts(self, action_id: str) -> list[dict[str, Any]]: + payload = self._request("GET", f"ciBuildActions/{action_id}/artifacts") + return payload.get("data", []) + + def get_artifact(self, artifact_id: str) -> dict[str, Any]: + return self._request("GET", f"ciArtifacts/{artifact_id}") + + def get_build_run_status(self, run_id: str) -> BuildRunStatus: + payload = self.get_build_run(run_id) + attributes = payload["data"]["attributes"] + return BuildRunStatus( + run_id=run_id, + execution_progress=attributes.get("executionProgress", ""), + completion_status=attributes.get("completionStatus"), + ) + + def wait_for_build_run( + self, + run_id: str, + *, + timeout_seconds: int = 3600, + poll_interval_seconds: int = 30, + ) -> BuildRunStatus: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + status = self.get_build_run_status(run_id) + if status.execution_progress == "COMPLETE": + return status + time.sleep(poll_interval_seconds) + raise TimeoutError(f"Timed out waiting for build run {run_id} to complete") diff --git a/scripts/xcode_cloud/extract.py b/scripts/xcode_cloud/extract.py new file mode 100644 index 0000000..0bb8c51 --- /dev/null +++ b/scripts/xcode_cloud/extract.py @@ -0,0 +1,49 @@ +"""Extract screenshot attachments from an .xcresult bundle.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + + +class XcresultToolNotFoundError(RuntimeError): + """Raised when xcresulttool is unavailable on the current machine.""" + + +def xcresulttool_path() -> str | None: + return shutil.which("xcresulttool") + + +def extract_attachments( + bundle_path: Path, + output_dir: Path, + *, + only_failures: bool = False, +) -> list[Path]: + """Export XCTest attachments from a result bundle using xcresulttool.""" + if not bundle_path.exists(): + raise FileNotFoundError(f"Result bundle not found: {bundle_path}") + + tool = xcresulttool_path() + if tool is None: + raise XcresultToolNotFoundError( + "xcresulttool was not found. Run this step on macOS with Xcode installed, " + "or use ci_scripts/ci_post_xcodebuild.sh inside Xcode Cloud." + ) + + output_dir.mkdir(parents=True, exist_ok=True) + command = [ + tool, + "export", + "attachments", + "--path", + str(bundle_path), + "--output-path", + str(output_dir), + ] + if only_failures: + command.append("--only-failures") + + subprocess.run(command, check=True, capture_output=True, text=True) + return sorted(path for path in output_dir.rglob("*.png") if path.is_file()) diff --git a/scripts/xcode_cloud/github_pr.py b/scripts/xcode_cloud/github_pr.py new file mode 100644 index 0000000..52fbcf5 --- /dev/null +++ b/scripts/xcode_cloud/github_pr.py @@ -0,0 +1,60 @@ +"""Optional GitHub pull request comment helpers.""" + +from __future__ import annotations + +from pathlib import Path + +import httpx + + +def build_screenshot_comment( + screenshot_paths: list[Path], + *, + build_run_id: str, + title: str = "Xcode Cloud screenshots", +) -> str: + if not screenshot_paths: + return ( + f"### {title}\n\n" + f"Build run `{build_run_id}` completed, but no screenshot attachments were found." + ) + + lines = [ + f"### {title}", + "", + f"Build run `{build_run_id}`", + "", + "| Screenshot |", + "| --- |", + ] + for path in screenshot_paths: + lines.append(f"| `{path.name}` |") + return "\n".join(lines) + + +def post_pr_comment( + repo: str, + pr_number: int, + body: str, + *, + token: str, + client: httpx.Client | None = None, +) -> dict: + """Post a markdown comment on a GitHub pull request.""" + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + response = http.post( + f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + json={"body": body}, + ) + response.raise_for_status() + return response.json() + finally: + if close_client: + http.close() diff --git a/scripts/xcode_cloud/screenshots.py b/scripts/xcode_cloud/screenshots.py new file mode 100644 index 0000000..4b9229c --- /dev/null +++ b/scripts/xcode_cloud/screenshots.py @@ -0,0 +1,157 @@ +"""Download Xcode Cloud test result bundles and extract screenshots.""" + +from __future__ import annotations + +import shutil +import zipfile +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +import httpx + +from scripts.xcode_cloud.client import XcodeCloudClient, XcodeCloudError +from scripts.xcode_cloud.extract import extract_attachments + + +TEST_ACTION_TYPES = {"TEST"} +TEST_RESULT_FILE_TYPES = {"TEST_RESULT_BUNDLE", "XCRESULT"} + + +@dataclass(frozen=True) +class ScreenshotFetchResult: + build_run_id: str + test_action_id: str + artifact_id: str + bundle_path: Path + screenshot_paths: tuple[Path, ...] + + +def find_test_action(actions: Iterable[dict]) -> dict: + for action in actions: + action_type = action.get("attributes", {}).get("actionType") + if action_type in TEST_ACTION_TYPES: + return action + raise XcodeCloudError("No TEST build action found for this build run") + + +def find_test_result_artifact(artifacts: Iterable[dict]) -> dict: + for artifact in artifacts: + file_type = artifact.get("attributes", {}).get("fileType") + if file_type in TEST_RESULT_FILE_TYPES: + return artifact + raise XcodeCloudError("No test result bundle artifact found for the TEST action") + + +def _download_file(url: str, destination: Path, *, client: httpx.Client | None = None) -> None: + http = client or httpx.Client(timeout=120.0, follow_redirects=True) + close_client = client is None + try: + with http.stream("GET", url) as response: + response.raise_for_status() + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("wb") as handle: + for chunk in response.iter_bytes(): + handle.write(chunk) + finally: + if close_client: + http.close() + + +def _prepare_xcresult_bundle(download_path: Path, output_dir: Path) -> Path: + if download_path.suffix == ".xcresult" and download_path.is_dir(): + return download_path + + if download_path.suffix == ".zip" or zipfile.is_zipfile(download_path): + extract_dir = output_dir / "extracted" + extract_dir.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(download_path) as archive: + archive.extractall(extract_dir) + candidates = list(extract_dir.rglob("*.xcresult")) + if not candidates: + raise XcodeCloudError(f"No .xcresult bundle found inside {download_path}") + return candidates[0] + + if download_path.name.endswith(".xcresult"): + if download_path.is_dir(): + return download_path + bundle_dir = output_dir / download_path.name + if bundle_dir.exists(): + shutil.rmtree(bundle_dir) + shutil.move(str(download_path), str(bundle_dir)) + return bundle_dir + + raise XcodeCloudError(f"Unsupported test result artifact format: {download_path}") + + +def fetch_test_result_bundle( + client: XcodeCloudClient, + build_run_id: str, + output_dir: Path, + *, + download_client: httpx.Client | None = None, +) -> tuple[str, str, Path]: + """Download the Xcode Cloud test result bundle for a completed build run.""" + actions = client.list_build_actions(build_run_id) + test_action = find_test_action(actions) + test_action_id = test_action["id"] + + artifacts = client.list_artifacts(test_action_id) + artifact = find_test_result_artifact(artifacts) + artifact_id = artifact["id"] + + artifact_payload = client.get_artifact(artifact_id) + download_url = artifact_payload["data"]["attributes"]["downloadUrl"] + file_name = artifact_payload["data"]["attributes"].get("fileName", f"{artifact_id}.zip") + + download_path = output_dir / file_name + _download_file(download_url, download_path, client=download_client) + bundle_path = _prepare_xcresult_bundle(download_path, output_dir) + return test_action_id, artifact_id, bundle_path + + +def fetch_screenshots_from_build_run( + client: XcodeCloudClient, + build_run_id: str, + output_dir: Path, + *, + only_failures: bool = False, + download_client: httpx.Client | None = None, +) -> ScreenshotFetchResult: + """Fetch a build run's test bundle and extract screenshot attachments.""" + screenshots_dir = output_dir / "screenshots" + test_action_id, artifact_id, bundle_path = fetch_test_result_bundle( + client, + build_run_id, + output_dir, + download_client=download_client, + ) + screenshot_paths = extract_attachments( + bundle_path, + screenshots_dir, + only_failures=only_failures, + ) + return ScreenshotFetchResult( + build_run_id=build_run_id, + test_action_id=test_action_id, + artifact_id=artifact_id, + bundle_path=bundle_path, + screenshot_paths=tuple(screenshot_paths), + ) + + +def extract_screenshots_from_local_bundle( + bundle_path: Path, + output_dir: Path, + *, + only_failures: bool = False, +) -> tuple[Path, ...]: + """Extract screenshots from a local .xcresult bundle (Xcode Cloud Mac path).""" + screenshots_dir = output_dir / "screenshots" + return tuple( + extract_attachments( + bundle_path, + screenshots_dir, + only_failures=only_failures, + ) + ) From 970dc8773c03c0b289c56a7054ecc55baea45192 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 28 Jun 2026 19:09:31 +0000 Subject: [PATCH 02/10] Add synchronous trigger-and-fetch for Xcode Cloud builds Expose a trigger-and-fetch CLI command that starts a workflow via the App Store Connect API, polls until the build run completes, and then downloads screenshots in one blocking invocation. Co-authored-by: Greg --- scripts/README.md | 10 +- scripts/fetch_xcode_cloud_screenshots.py | 67 ++++++++++--- scripts/tests/test_trigger.py | 120 +++++++++++++++++++++++ scripts/xcode_cloud/client.py | 44 ++++++++- scripts/xcode_cloud/trigger.py | 92 +++++++++++++++++ 5 files changed, 319 insertions(+), 14 deletions(-) create mode 100644 scripts/tests/test_trigger.py create mode 100644 scripts/xcode_cloud/trigger.py diff --git a/scripts/README.md b/scripts/README.md index d9693e2..b97ad77 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -41,7 +41,15 @@ export GITHUB_PULL_REQUEST="123" ## Usage -Poll until complete, then fetch and extract: +Trigger a build, block until it finishes, then fetch screenshots (fully synchronous): + +```bash +python3 scripts/fetch_xcode_cloud_screenshots.py trigger-and-fetch \ + --workflow-id WORKFLOW_ID \ + --branch main +``` + +Poll an already-triggered build until complete, then fetch: ```bash python3 scripts/fetch_xcode_cloud_screenshots.py wait-and-fetch --run-id BUILD_RUN_ID diff --git a/scripts/fetch_xcode_cloud_screenshots.py b/scripts/fetch_xcode_cloud_screenshots.py index aca6a45..eda66d1 100644 --- a/scripts/fetch_xcode_cloud_screenshots.py +++ b/scripts/fetch_xcode_cloud_screenshots.py @@ -18,7 +18,9 @@ from scripts.xcode_cloud.screenshots import ( extract_screenshots_from_local_bundle, fetch_screenshots_from_build_run, + fetch_test_result_bundle, ) +from scripts.xcode_cloud.trigger import trigger_and_wait def build_parser() -> argparse.ArgumentParser: @@ -62,6 +64,28 @@ def build_parser() -> argparse.ArgumentParser: wait_parser.add_argument("--poll-interval-seconds", type=int, default=30) wait_parser.add_argument("--only-failures", action="store_true") + trigger_parser = subparsers.add_parser( + "trigger-and-fetch", + help="Start an Xcode Cloud build, wait until it finishes, then fetch screenshots.", + ) + trigger_parser.add_argument("--workflow-id", required=True, help="ciWorkflows ID") + branch_group = trigger_parser.add_mutually_exclusive_group(required=True) + branch_group.add_argument("--branch", help="Branch name to build") + branch_group.add_argument("--git-reference-id", help="scmGitReferences ID") + trigger_parser.add_argument( + "--output-dir", + default="./xcode-cloud-output", + help="Directory for downloaded bundles and screenshots", + ) + trigger_parser.add_argument("--timeout-seconds", type=int, default=3600) + trigger_parser.add_argument("--poll-interval-seconds", type=int, default=30) + trigger_parser.add_argument("--only-failures", action="store_true") + trigger_parser.add_argument( + "--skip-extract", + action="store_true", + help="Download the bundle but do not run xcresulttool extraction", + ) + local_parser = subparsers.add_parser( "extract-local", help="Extract screenshots from a local .xcresult bundle.", @@ -118,34 +142,55 @@ def main(argv: list[str] | None = None) -> int: credentials = credentials_from_env() output_dir = Path(args.output_dir) + build_succeeded = True with XcodeCloudClient(lambda: create_asc_token(credentials)) as client: - if args.command == "wait-and-fetch": + run_id = getattr(args, "run_id", None) + + if args.command == "trigger-and-fetch": + trigger_result = trigger_and_wait( + client, + args.workflow_id, + git_reference_id=getattr(args, "git_reference_id", None), + branch=getattr(args, "branch", None), + timeout_seconds=args.timeout_seconds, + poll_interval_seconds=args.poll_interval_seconds, + ) + run_id = trigger_result.build_run_id + build_succeeded = trigger_result.status.completion_status == "SUCCEEDED" + print( + f"Triggered build run {run_id}; completed with status " + f"{trigger_result.status.completion_status or 'UNKNOWN'}" + ) + + elif args.command == "wait-and-fetch": status = client.wait_for_build_run( args.run_id, timeout_seconds=args.timeout_seconds, poll_interval_seconds=args.poll_interval_seconds, ) + run_id = status.run_id + build_succeeded = status.completion_status == "SUCCEEDED" print( f"Build run {status.run_id} completed with status " f"{status.completion_status or 'UNKNOWN'}" ) - if args.command == "fetch" and args.skip_extract: - from scripts.xcode_cloud.screenshots import fetch_test_result_bundle + assert run_id is not None + if getattr(args, "skip_extract", False): _, artifact_id, bundle_path = fetch_test_result_bundle( client, - args.run_id, + run_id, output_dir, ) print(f"Downloaded artifact {artifact_id} to {bundle_path}") - return 0 + return 0 if build_succeeded else 1 try: result = fetch_screenshots_from_build_run( client, - args.run_id, + run_id, output_dir, only_failures=getattr(args, "only_failures", False), ) @@ -153,11 +198,11 @@ def main(argv: list[str] | None = None) -> int: print(str(error), file=sys.stderr) return 2 - print(f"Test action: {result.test_action_id}") - print(f"Artifact: {result.artifact_id}") - print(f"Bundle: {result.bundle_path}") - _print_screenshots(result.screenshot_paths) - return 0 + print(f"Test action: {result.test_action_id}") + print(f"Artifact: {result.artifact_id}") + print(f"Bundle: {result.bundle_path}") + _print_screenshots(result.screenshot_paths) + return 0 if build_succeeded else 1 def _print_screenshots(screenshots: list[Path] | tuple[Path, ...]) -> None: diff --git a/scripts/tests/test_trigger.py b/scripts/tests/test_trigger.py new file mode 100644 index 0000000..e707917 --- /dev/null +++ b/scripts/tests/test_trigger.py @@ -0,0 +1,120 @@ +import httpx + +from scripts.xcode_cloud.client import XcodeCloudClient +from scripts.xcode_cloud.trigger import ( + git_reference_id_for_branch, + repository_id_for_workflow, + trigger_and_wait, + trigger_build_run, +) + + +def test_trigger_build_run_returns_run_id(): + create_response = {"data": {"type": "ciBuildRuns", "id": "new-run-1"}} + + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "POST" and request.url.path.endswith("/v1/ciBuildRuns"): + return httpx.Response(201, json=create_response) + raise AssertionError(f"Unexpected request: {request.method} {request.url}") + + client = httpx.Client(transport=httpx.MockTransport(handler)) + api = XcodeCloudClient(lambda: "token", client=client) + + run_id = trigger_build_run( + api, + "workflow-1", + git_reference_id="git-ref-1", + ) + + assert run_id == "new-run-1" + + +def test_git_reference_id_for_branch(): + references = { + "data": [ + { + "id": "git-ref-main", + "attributes": { + "kind": "BRANCH", + "name": "main", + "canonicalName": "refs/heads/main", + }, + } + ] + } + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/gitReferences"): + return httpx.Response(200, json=references) + raise AssertionError(f"Unexpected request: {request.url}") + + client = httpx.Client(transport=httpx.MockTransport(handler)) + api = XcodeCloudClient(lambda: "token", client=client) + + ref_id = git_reference_id_for_branch(api, "repo-1", "main") + assert ref_id == "git-ref-main" + + +def test_repository_id_for_workflow_from_included(): + workflow = { + "data": { + "id": "workflow-1", + "type": "ciWorkflows", + "relationships": {}, + }, + "included": [ + {"id": "repo-1", "type": "scmRepositories"}, + ], + } + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/ciWorkflows/workflow-1"): + return httpx.Response(200, json=workflow) + raise AssertionError(f"Unexpected request: {request.url}") + + client = httpx.Client(transport=httpx.MockTransport(handler)) + api = XcodeCloudClient(lambda: "token", client=client) + + repo_id = repository_id_for_workflow(api, "workflow-1") + assert repo_id == "repo-1" + + +def test_trigger_and_wait_polls_until_complete(monkeypatch): + calls = {"count": 0} + pending = { + "data": { + "id": "run-1", + "attributes": {"executionProgress": "RUNNING", "completionStatus": None}, + } + } + complete = { + "data": { + "id": "run-1", + "attributes": {"executionProgress": "COMPLETE", "completionStatus": "SUCCEEDED"}, + } + } + create_response = {"data": {"type": "ciBuildRuns", "id": "run-1"}} + + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "POST" and request.url.path.endswith("/v1/ciBuildRuns"): + return httpx.Response(201, json=create_response) + if request.url.path.endswith("/ciBuildRuns/run-1"): + calls["count"] += 1 + return httpx.Response(200, json=complete if calls["count"] > 1 else pending) + raise AssertionError(f"Unexpected request: {request.method} {request.url}") + + monkeypatch.setattr("scripts.xcode_cloud.client.time.sleep", lambda _: None) + + client = httpx.Client(transport=httpx.MockTransport(handler)) + api = XcodeCloudClient(lambda: "token", client=client) + + result = trigger_and_wait( + api, + "workflow-1", + git_reference_id="git-ref-1", + poll_interval_seconds=1, + ) + + assert result.build_run_id == "run-1" + assert result.status.completion_status == "SUCCEEDED" + assert calls["count"] >= 2 diff --git a/scripts/xcode_cloud/client.py b/scripts/xcode_cloud/client.py index 56fe852..efbdaf0 100644 --- a/scripts/xcode_cloud/client.py +++ b/scripts/xcode_cloud/client.py @@ -42,12 +42,20 @@ def __enter__(self) -> XcodeCloudClient: def __exit__(self, *args: object) -> None: self.close() - def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: + def _request( + self, + method: str, + path: str, + *, + allowed_statuses: set[int] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: url = path if path.startswith("http") else urljoin(self.BASE_URL, path.lstrip("/")) headers = kwargs.pop("headers", {}) headers["Authorization"] = f"Bearer {self._token_provider()}" response = self._client.request(method, url, headers=headers, **kwargs) - if response.status_code >= 400: + allowed = allowed_statuses or {200} + if response.status_code not in allowed: raise XcodeCloudError( f"{method} {url} failed with {response.status_code}: {response.text}" ) @@ -58,6 +66,38 @@ def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: def get_build_run(self, run_id: str) -> dict[str, Any]: return self._request("GET", f"ciBuildRuns/{run_id}") + def create_build_run(self, workflow_id: str, git_reference_id: str) -> dict[str, Any]: + body = { + "data": { + "type": "ciBuildRuns", + "relationships": { + "workflow": { + "data": {"type": "ciWorkflows", "id": workflow_id}, + }, + "sourceBranchOrTag": { + "data": {"type": "scmGitReferences", "id": git_reference_id}, + }, + }, + } + } + return self._request( + "POST", + "ciBuildRuns", + json=body, + allowed_statuses={201}, + ) + + def get_workflow(self, workflow_id: str, *, include_repository: bool = False) -> dict[str, Any]: + query = "include=repository" if include_repository else None + path = f"ciWorkflows/{workflow_id}" + if query: + path = f"{path}?{query}" + return self._request("GET", path) + + def list_git_references(self, repository_id: str) -> list[dict[str, Any]]: + payload = self._request("GET", f"scmRepositories/{repository_id}/gitReferences") + return payload.get("data", []) + def list_build_actions(self, run_id: str) -> list[dict[str, Any]]: payload = self._request("GET", f"ciBuildRuns/{run_id}/actions") return payload.get("data", []) diff --git a/scripts/xcode_cloud/trigger.py b/scripts/xcode_cloud/trigger.py new file mode 100644 index 0000000..0140632 --- /dev/null +++ b/scripts/xcode_cloud/trigger.py @@ -0,0 +1,92 @@ +"""Trigger Xcode Cloud builds and wait for completion.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.xcode_cloud.client import BuildRunStatus, XcodeCloudClient, XcodeCloudError + + +@dataclass(frozen=True) +class TriggerResult: + build_run_id: str + status: BuildRunStatus + + +def repository_id_for_workflow(client: XcodeCloudClient, workflow_id: str) -> str: + payload = client.get_workflow(workflow_id, include_repository=True) + included = payload.get("included", []) + for resource in included: + if resource.get("type") == "scmRepositories": + return resource["id"] + + relationships = payload.get("data", {}).get("relationships", {}) + repository = relationships.get("repository", {}).get("data") + if repository and repository.get("id"): + return repository["id"] + + raise XcodeCloudError(f"Could not resolve repository for workflow {workflow_id}") + + +def git_reference_id_for_branch( + client: XcodeCloudClient, + repository_id: str, + branch: str, +) -> str: + references = client.list_git_references(repository_id) + normalized = branch.removeprefix("refs/heads/") + for reference in references: + attributes = reference.get("attributes", {}) + if attributes.get("kind") != "BRANCH": + continue + name = attributes.get("name", "") + canonical = attributes.get("canonicalName", "") + if name == normalized or canonical == f"refs/heads/{normalized}": + return reference["id"] + + raise XcodeCloudError( + f"Could not find git reference for branch '{branch}' in repository {repository_id}" + ) + + +def trigger_build_run( + client: XcodeCloudClient, + workflow_id: str, + *, + git_reference_id: str | None = None, + branch: str | None = None, +) -> str: + if bool(git_reference_id) == bool(branch): + raise ValueError("Provide exactly one of git_reference_id or branch") + + resolved_reference_id = git_reference_id + if branch is not None: + repository_id = repository_id_for_workflow(client, workflow_id) + resolved_reference_id = git_reference_id_for_branch(client, repository_id, branch) + + assert resolved_reference_id is not None + payload = client.create_build_run(workflow_id, resolved_reference_id) + return payload["data"]["id"] + + +def trigger_and_wait( + client: XcodeCloudClient, + workflow_id: str, + *, + git_reference_id: str | None = None, + branch: str | None = None, + timeout_seconds: int = 3600, + poll_interval_seconds: int = 30, +) -> TriggerResult: + build_run_id = trigger_build_run( + client, + workflow_id, + git_reference_id=git_reference_id, + branch=branch, + ) + status = client.wait_for_build_run( + build_run_id, + timeout_seconds=timeout_seconds, + poll_interval_seconds=poll_interval_seconds, + ) + return TriggerResult(build_run_id=build_run_id, status=status) From 67d7b4f2ae123691b5e2c75511b012b339b9f50c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 28 Jun 2026 19:10:48 +0000 Subject: [PATCH 03/10] Upload Xcode Cloud screenshots to public S3 bucket from ci_post Document the macOS-only xcresulttool extraction requirement prominently and make the recommended PR path extract-on-Xcode-Cloud, upload to a public bucket, then embed stable URLs in GitHub comments. Co-authored-by: Greg --- Bedtime/ci_scripts/ci_post_xcodebuild.sh | 43 +++++--- scripts/README.md | 120 ++++++++++++----------- scripts/fetch_xcode_cloud_screenshots.py | 71 +++++++++++++- scripts/requirements-dev.txt | 1 + scripts/tests/test_github_pr.py | 18 +++- scripts/tests/test_upload.py | 53 ++++++++++ scripts/xcode_cloud/github_pr.py | 28 +++++- scripts/xcode_cloud/upload.py | 116 ++++++++++++++++++++++ 8 files changed, 368 insertions(+), 82 deletions(-) create mode 100644 scripts/tests/test_upload.py create mode 100644 scripts/xcode_cloud/upload.py diff --git a/Bedtime/ci_scripts/ci_post_xcodebuild.sh b/Bedtime/ci_scripts/ci_post_xcodebuild.sh index 9bbc244..4b51b34 100755 --- a/Bedtime/ci_scripts/ci_post_xcodebuild.sh +++ b/Bedtime/ci_scripts/ci_post_xcodebuild.sh @@ -2,10 +2,14 @@ set -eu # Runs on the Xcode Cloud Mac after xcodebuild finishes. -# Uses the local result bundle directly — no App Store Connect API download needed. +# Extracts screenshots locally (macOS + xcresulttool), uploads to a public bucket, +# then optionally posts a PR comment with embeddable image URLs. OUTPUT_DIR="${CI_DERIVED_DATA_PATH:-/tmp}/xcode-cloud-screenshots" +SCREENSHOTS_DIR="$OUTPUT_DIR/screenshots" +MANIFEST_PATH="$OUTPUT_DIR/screenshots-manifest.json" ONLY_FAILURES="${XCODE_CLOUD_SCREENSHOT_ONLY_FAILURES:-false}" +BUILD_ID="${CI_BUILD_ID:-unknown}" if [ -z "${CI_RESULT_BUNDLE_PATH:-}" ]; then echo "ci_post_xcodebuild: CI_RESULT_BUNDLE_PATH is not set; skipping screenshot export." @@ -19,25 +23,38 @@ fi mkdir -p "$OUTPUT_DIR" -ARGS="extract-local --bundle-path \"$CI_RESULT_BUNDLE_PATH\" --output-dir \"$OUTPUT_DIR\"" +REPO_ROOT="${CI_PRIMARY_REPOSITORY_PATH:-$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)}" +PYTHON_BIN="${PYTHON_BIN:-python3}" +FETCH_SCRIPT="$REPO_ROOT/scripts/fetch_xcode_cloud_screenshots.py" + +EXTRACT_ARGS="extract-local --bundle-path \"$CI_RESULT_BUNDLE_PATH\" --output-dir \"$OUTPUT_DIR\"" if [ "$ONLY_FAILURES" = "true" ]; then - ARGS="$ARGS --only-failures" + EXTRACT_ARGS="$EXTRACT_ARGS --only-failures" fi -REPO_ROOT="${CI_PRIMARY_REPOSITORY_PATH:-$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)}" -PYTHON_BIN="${PYTHON_BIN:-python3}" - echo "ci_post_xcodebuild: exporting screenshots from $CI_RESULT_BUNDLE_PATH" # shellcheck disable=SC2086 -"$PYTHON_BIN" "$REPO_ROOT/scripts/fetch_xcode_cloud_screenshots.py" $ARGS +"$PYTHON_BIN" "$FETCH_SCRIPT" $EXTRACT_ARGS + +if [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then + echo "ci_post_xcodebuild: uploading screenshots to s3://$SCREENSHOTS_S3_BUCKET" + "$PYTHON_BIN" -m pip install --quiet boto3 + "$PYTHON_BIN" "$FETCH_SCRIPT" upload-screenshots \ + --screenshots-dir "$SCREENSHOTS_DIR" \ + --build-id "$BUILD_ID" \ + --manifest "$MANIFEST_PATH" +fi if [ -n "${GITHUB_REPOSITORY:-}" ] && [ -n "${GITHUB_PULL_REQUEST:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then echo "ci_post_xcodebuild: posting screenshot summary to PR #${GITHUB_PULL_REQUEST}" - "$PYTHON_BIN" "$REPO_ROOT/scripts/fetch_xcode_cloud_screenshots.py" comment-pr \ - --repo "$GITHUB_REPOSITORY" \ - --pr-number "$GITHUB_PULL_REQUEST" \ - --run-id "${CI_BUILD_ID:-unknown}" \ - --screenshots-dir "$OUTPUT_DIR/screenshots" + COMMENT_ARGS="comment-pr --repo \"$GITHUB_REPOSITORY\" --pr-number \"$GITHUB_PULL_REQUEST\" --run-id \"$BUILD_ID\"" + if [ -f "$MANIFEST_PATH" ]; then + COMMENT_ARGS="$COMMENT_ARGS --manifest \"$MANIFEST_PATH\"" + else + COMMENT_ARGS="$COMMENT_ARGS --screenshots-dir \"$SCREENSHOTS_DIR\"" + fi + # shellcheck disable=SC2086 + "$PYTHON_BIN" "$FETCH_SCRIPT" $COMMENT_ARGS fi -echo "ci_post_xcodebuild: screenshots available in $OUTPUT_DIR/screenshots" +echo "ci_post_xcodebuild: screenshots available in $SCREENSHOTS_DIR" diff --git a/scripts/README.md b/scripts/README.md index b97ad77..a403636 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,107 +2,111 @@ Fetch UI test screenshots from Xcode Cloud after a build completes. -## Why not a GitHub Action? +## Recommended flow: Xcode Cloud uploads to a public bucket -A GitHub Action is **optional**, not required. It was only suggested earlier because `xcresulttool` needs macOS. +PR comments need **stable, public image URLs**. Apple's artifact `downloadUrl` values expire, and extraction requires **macOS + Xcode** (`xcresulttool`). -This repo uses two paths instead: +The recommended path is to do everything on the **Xcode Cloud Mac** in `Bedtime/ci_scripts/ci_post_xcodebuild.sh`: -1. **`Bedtime/ci_scripts/ci_post_xcodebuild.sh`** (recommended on Xcode Cloud) - Runs on Apple's Mac right after tests finish. It reads `CI_RESULT_BUNDLE_PATH` directly, so there is no API round-trip and no second macOS runner. +```text +UI tests finish on Xcode Cloud (macOS) + → extract PNGs from CI_RESULT_BUNDLE_PATH [macOS only] + → upload to public S3/R2 bucket [stable URLs] + → post PR comment with embedded images +``` -2. **`scripts/fetch_xcode_cloud_screenshots.py fetch`** - For webhook handlers, local machines, or Cursor Cloud Agents. Downloads the test result bundle via the App Store Connect API, then extracts screenshots if `xcresulttool` is available. +Anything that runs on Linux (Cursor Cloud Agent, webhook handler, `trigger-and-fetch` without `--skip-extract`) can **trigger** builds and **download** bundles, but cannot extract screenshots unless you move the `.xcresult` to a Mac. -Use a GitHub Action only if you want extraction to happen outside Xcode Cloud. +## macOS requirement (extraction) -## Setup +| Step | Runs on | Tool | +|------|---------|------| +| Trigger build | anywhere | App Store Connect API | +| Wait for completion | anywhere | App Store Connect API | +| Download `.xcresult` | anywhere | App Store Connect API | +| **Extract PNGs** | **macOS only** | `xcresulttool` (ships with Xcode) | +| Upload to public bucket | anywhere with AWS creds | `boto3` | +| Post PR comment | anywhere | GitHub API | -```bash -cd scripts -python3 -m pip install -r requirements-dev.txt -``` +On Linux, use `--skip-extract` to download the bundle only. Do not expect `fetch` or `trigger-and-fetch` to produce PNGs on a Linux agent. -### Credentials (Runtime Secrets in Cursor, or local env) +## Xcode Cloud setup (public bucket + PR embed) -```bash -export APP_STORE_CONNECT_KEY_ID="..." -export APP_STORE_CONNECT_ISSUER_ID="..." -export APP_STORE_CONNECT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" -``` - -Optional for PR comments from `ci_post_xcodebuild.sh`: +Add these as **Xcode Cloud workflow secrets**: ```bash -export GITHUB_TOKEN="..." -export GITHUB_REPOSITORY="owner/repo" -export GITHUB_PULL_REQUEST="123" +# S3 or S3-compatible bucket with public read on the prefix (bucket policy, or CDN in front) +SCREENSHOTS_S3_BUCKET="my-public-screenshots" +SCREENSHOTS_S3_PREFIX="bedtime" +SCREENSHOTS_PUBLIC_BASE_URL="https://my-public-screenshots.s3.amazonaws.com" # or CloudFront URL +AWS_ACCESS_KEY_ID="..." +AWS_SECRET_ACCESS_KEY="..." + +# Optional: PR comment with embedded images +GITHUB_TOKEN="..." +GITHUB_REPOSITORY="owner/repo" +GITHUB_PULL_REQUEST="123" # set via custom env / script if not automatic ``` -## Usage - -Trigger a build, block until it finishes, then fetch screenshots (fully synchronous): +For R2 or other S3-compatible storage: ```bash -python3 scripts/fetch_xcode_cloud_screenshots.py trigger-and-fetch \ - --workflow-id WORKFLOW_ID \ - --branch main +SCREENSHOTS_S3_ENDPOINT_URL="https://.r2.cloudflarestorage.com" +SCREENSHOTS_PUBLIC_BASE_URL="https://screenshots.example.com" ``` -Poll an already-triggered build until complete, then fetch: +`ci_post_xcodebuild.sh` will: -```bash -python3 scripts/fetch_xcode_cloud_screenshots.py wait-and-fetch --run-id BUILD_RUN_ID -``` +1. Extract screenshots from `CI_RESULT_BUNDLE_PATH` +2. Upload PNGs when `SCREENSHOTS_S3_BUCKET` is set +3. Post a PR comment with `![name](url)` markdown when GitHub env vars are set -Fetch a completed build: +## App Store Connect API (trigger / fetch from outside Xcode Cloud) ```bash -python3 scripts/fetch_xcode_cloud_screenshots.py fetch --run-id BUILD_RUN_ID +export APP_STORE_CONNECT_KEY_ID="..." +export APP_STORE_CONNECT_ISSUER_ID="..." +export APP_STORE_CONNECT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" ``` -Download only (skip extraction on Linux): +### Fully synchronous trigger + wait + fetch ```bash -python3 scripts/fetch_xcode_cloud_screenshots.py fetch --run-id BUILD_RUN_ID --skip-extract +python3 scripts/fetch_xcode_cloud_screenshots.py trigger-and-fetch \ + --workflow-id WORKFLOW_ID \ + --branch main ``` -Extract from a local bundle (Xcode Cloud Mac or your laptop): +On Linux, add `--skip-extract` and rely on `ci_post_xcodebuild.sh` for extraction + upload. + +### Wait for an existing build, then fetch ```bash -python3 scripts/fetch_xcode_cloud_screenshots.py extract-local \ - --bundle-path /path/to/resultbundle.xcresult +python3 scripts/fetch_xcode_cloud_screenshots.py wait-and-fetch --run-id BUILD_RUN_ID ``` -Post a PR summary comment: +### Manual upload + PR comment (after extraction on a Mac) ```bash +python3 scripts/fetch_xcode_cloud_screenshots.py upload-screenshots \ + --screenshots-dir ./xcode-cloud-output/screenshots \ + --build-id BUILD_RUN_ID + python3 scripts/fetch_xcode_cloud_screenshots.py comment-pr \ --repo owner/repo \ --pr-number 123 \ --run-id BUILD_RUN_ID \ - --screenshots-dir ./xcode-cloud-output/screenshots + --manifest ./xcode-cloud-output/screenshots-manifest.json ``` -## Tests +## Local development ```bash cd scripts +python3 -m pip install -r requirements-dev.txt python3 -m pytest ``` -## Flow - -```text -Xcode Cloud test action finishes - ├─ ci_post_xcodebuild.sh (on Mac) - │ └─ extract from CI_RESULT_BUNDLE_PATH - │ - └─ webhook / manual fetch (anywhere) - └─ App Store Connect API → temporary downloadUrl - └─ download .xcresult - └─ xcresulttool export attachments (macOS) - └─ optional GitHub PR comment -``` +## Why not a GitHub Action? -Apple's `downloadUrl` values are short-lived. Do not embed them directly in PRs. Extract PNGs and upload or reference stable assets instead. +A GitHub Action is optional. It only helps if you want macOS extraction **outside** Xcode Cloud. If Xcode Cloud already runs your UI tests, `ci_post_xcodebuild.sh` is the simpler place to extract and upload — no second macOS runner. diff --git a/scripts/fetch_xcode_cloud_screenshots.py b/scripts/fetch_xcode_cloud_screenshots.py index eda66d1..075bb8b 100644 --- a/scripts/fetch_xcode_cloud_screenshots.py +++ b/scripts/fetch_xcode_cloud_screenshots.py @@ -21,6 +21,13 @@ fetch_test_result_bundle, ) from scripts.xcode_cloud.trigger import trigger_and_wait +from scripts.xcode_cloud.upload import ( + UploadConfigError, + UploadedScreenshot, + upload_config_from_env, + upload_screenshots, + write_manifest, +) def build_parser() -> argparse.ArgumentParser: @@ -107,8 +114,23 @@ def build_parser() -> argparse.ArgumentParser: comment_parser.add_argument("--run-id", required=True) comment_parser.add_argument( "--screenshots-dir", - required=True, - help="Directory containing extracted .png files", + help="Directory containing extracted .png files (used when no manifest is provided)", + ) + comment_parser.add_argument( + "--manifest", + help="JSON manifest written by upload-screenshots (preferred for embedded images)", + ) + + upload_parser = subparsers.add_parser( + "upload-screenshots", + help="Upload extracted screenshots to a public S3 bucket.", + ) + upload_parser.add_argument("--screenshots-dir", required=True) + upload_parser.add_argument("--build-id", required=True) + upload_parser.add_argument( + "--manifest", + default="./xcode-cloud-output/screenshots-manifest.json", + help="Where to write the public URL manifest", ) return parser @@ -127,19 +149,58 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.command == "comment-pr": + import json import os token = os.environ.get("GITHUB_TOKEN") if not token: parser.error("GITHUB_TOKEN is required for comment-pr") - screenshots_dir = Path(args.screenshots_dir) - screenshots = sorted(screenshots_dir.rglob("*.png")) - body = build_screenshot_comment(screenshots, build_run_id=args.run_id) + uploaded: list[UploadedScreenshot] | None = None + screenshots: list[Path] = [] + if args.manifest: + manifest = json.loads(Path(args.manifest).read_text()) + uploaded = [ + UploadedScreenshot(name=item["name"], key=item["key"], url=item["url"]) + for item in manifest.get("screenshots", []) + ] + elif args.screenshots_dir: + screenshots = sorted(Path(args.screenshots_dir).rglob("*.png")) + else: + parser.error("comment-pr requires --manifest or --screenshots-dir") + + body = build_screenshot_comment( + screenshots, + build_run_id=args.run_id, + uploaded=uploaded, + ) post_pr_comment(args.repo, args.pr_number, body, token=token) print(f"Posted PR comment to {args.repo}#{args.pr_number}") return 0 + if args.command == "upload-screenshots": + try: + config = upload_config_from_env() + except UploadConfigError as error: + parser.error(str(error)) + + uploads = upload_screenshots( + Path(args.screenshots_dir), + build_id=args.build_id, + bucket=config["bucket"], + prefix=config["prefix"], + public_base_url=config["public_base_url"], + region=config["region"], + endpoint_url=config["endpoint_url"], + ) + manifest_path = Path(args.manifest) + write_manifest(manifest_path, args.build_id, uploads) + print(f"Uploaded {len(uploads)} screenshot(s)") + for item in uploads: + print(item.url) + print(f"Manifest: {manifest_path}") + return 0 + credentials = credentials_from_env() output_dir = Path(args.output_dir) build_succeeded = True diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt index 7de14c9..c0367e5 100644 --- a/scripts/requirements-dev.txt +++ b/scripts/requirements-dev.txt @@ -2,3 +2,4 @@ pytest>=8.0 httpx>=0.27 PyJWT[crypto]>=2.8 cryptography>=42.0 +boto3>=1.34 diff --git a/scripts/tests/test_github_pr.py b/scripts/tests/test_github_pr.py index 3796629..987fca2 100644 --- a/scripts/tests/test_github_pr.py +++ b/scripts/tests/test_github_pr.py @@ -1,15 +1,29 @@ from pathlib import Path from scripts.xcode_cloud.github_pr import build_screenshot_comment +from scripts.xcode_cloud.upload import UploadedScreenshot -def test_build_screenshot_comment_lists_files(tmp_path): +def test_build_screenshot_comment_lists_files_without_upload(tmp_path): shots = [tmp_path / "home.png", tmp_path / "settings.png"] body = build_screenshot_comment(shots, build_run_id="run-1") assert "run-1" in body assert "home.png" in body - assert "settings.png" in body + assert "public bucket" in body.lower() + + +def test_build_screenshot_comment_embeds_uploaded_images(): + uploaded = [ + UploadedScreenshot( + name="home.png", + key="bedtime/run-1/home.png", + url="https://cdn.example.com/bedtime/run-1/home.png", + ) + ] + body = build_screenshot_comment([], build_run_id="run-1", uploaded=uploaded) + + assert "![home.png](https://cdn.example.com/bedtime/run-1/home.png)" in body def test_build_screenshot_comment_handles_empty_list(): diff --git a/scripts/tests/test_upload.py b/scripts/tests/test_upload.py new file mode 100644 index 0000000..d5981b0 --- /dev/null +++ b/scripts/tests/test_upload.py @@ -0,0 +1,53 @@ +import pytest + +from scripts.xcode_cloud.upload import ( + UploadConfigError, + UploadedScreenshot, + object_key, + public_url_for_key, + upload_config_from_env, + write_manifest, +) + + +def test_object_key_includes_build_id_and_filename(): + assert object_key("bedtime", "run-1", "home.png") == "bedtime/run-1/home.png" + + +def test_public_url_for_key(): + url = public_url_for_key("https://cdn.example.com/shots", "bedtime/run-1/home.png") + assert url == "https://cdn.example.com/shots/bedtime/run-1/home.png" + + +def test_upload_config_from_env(monkeypatch): + monkeypatch.setenv("SCREENSHOTS_S3_BUCKET", "my-bucket") + monkeypatch.setenv("SCREENSHOTS_S3_PREFIX", "bedtime") + monkeypatch.setenv("SCREENSHOTS_PUBLIC_BASE_URL", "https://cdn.example.com") + + config = upload_config_from_env() + + assert config["bucket"] == "my-bucket" + assert config["prefix"] == "bedtime" + assert config["public_base_url"] == "https://cdn.example.com" + + +def test_upload_config_requires_bucket(monkeypatch): + monkeypatch.delenv("SCREENSHOTS_S3_BUCKET", raising=False) + with pytest.raises(UploadConfigError, match="SCREENSHOTS_S3_BUCKET"): + upload_config_from_env() + + +def test_write_manifest(tmp_path): + uploads = [ + UploadedScreenshot( + name="home.png", + key="bedtime/run-1/home.png", + url="https://cdn.example.com/bedtime/run-1/home.png", + ) + ] + manifest_path = tmp_path / "manifest.json" + write_manifest(manifest_path, "run-1", uploads) + + payload = manifest_path.read_text() + assert "home.png" in payload + assert "https://cdn.example.com/bedtime/run-1/home.png" in payload diff --git a/scripts/xcode_cloud/github_pr.py b/scripts/xcode_cloud/github_pr.py index 52fbcf5..551880d 100644 --- a/scripts/xcode_cloud/github_pr.py +++ b/scripts/xcode_cloud/github_pr.py @@ -6,13 +6,30 @@ import httpx +from scripts.xcode_cloud.upload import UploadedScreenshot + def build_screenshot_comment( - screenshot_paths: list[Path], - *, - build_run_id: str, - title: str = "Xcode Cloud screenshots", + screenshot_paths: list[Path], + *, + build_run_id: str, + title: str = "Xcode Cloud screenshots", + uploaded: list[UploadedScreenshot] | None = None, ) -> str: + if uploaded: + lines = [ + f"### {title}", + "", + f"Build run `{build_run_id}`", + "", + ] + for item in uploaded: + lines.append(f"**{item.name}**") + lines.append("") + lines.append(f"![{item.name}]({item.url})") + lines.append("") + return "\n".join(lines).rstrip() + if not screenshot_paths: return ( f"### {title}\n\n" @@ -24,6 +41,9 @@ def build_screenshot_comment( "", f"Build run `{build_run_id}`", "", + "Screenshots were extracted on the Xcode Cloud Mac but not uploaded to a public bucket.", + "Set `SCREENSHOTS_S3_BUCKET` (and related env vars) to embed images in PR comments.", + "", "| Screenshot |", "| --- |", ] diff --git a/scripts/xcode_cloud/upload.py b/scripts/xcode_cloud/upload.py new file mode 100644 index 0000000..58231f6 --- /dev/null +++ b/scripts/xcode_cloud/upload.py @@ -0,0 +1,116 @@ +"""Upload extracted screenshots to a public S3-compatible bucket.""" + +from __future__ import annotations + +import json +import mimetypes +import os +from dataclasses import dataclass +from pathlib import Path + + +class UploadConfigError(ValueError): + """Raised when required upload configuration is missing.""" + + +@dataclass(frozen=True) +class UploadedScreenshot: + name: str + key: str + url: str + + +def upload_config_from_env() -> dict[str, str]: + bucket = os.environ.get("SCREENSHOTS_S3_BUCKET", "").strip() + if not bucket: + raise UploadConfigError("SCREENSHOTS_S3_BUCKET is required for upload") + + public_base_url = os.environ.get("SCREENSHOTS_PUBLIC_BASE_URL", "").strip() + if not public_base_url: + region = os.environ.get("SCREENSHOTS_S3_REGION", "us-east-1").strip() + public_base_url = f"https://{bucket}.s3.{region}.amazonaws.com" + + return { + "bucket": bucket, + "prefix": os.environ.get("SCREENSHOTS_S3_PREFIX", "xcode-cloud-screenshots").strip("/"), + "public_base_url": public_base_url.rstrip("/"), + "region": os.environ.get("SCREENSHOTS_S3_REGION", "us-east-1").strip(), + "endpoint_url": os.environ.get("SCREENSHOTS_S3_ENDPOINT_URL", "").strip() or None, + } + + +def object_key(prefix: str, build_id: str, filename: str) -> str: + safe_name = Path(filename).name + return "/".join(part for part in (prefix, build_id, safe_name) if part) + + +def public_url_for_key(public_base_url: str, key: str) -> str: + return f"{public_base_url}/{key.lstrip('/')}" + + +def _s3_client(region: str, endpoint_url: str | None): + try: + import boto3 + except ImportError as error: + raise RuntimeError( + "boto3 is required for S3 upload. Install with: python3 -m pip install boto3" + ) from error + + return boto3.client("s3", region_name=region, endpoint_url=endpoint_url) + + +def upload_screenshots( + screenshots_dir: Path, + *, + build_id: str, + bucket: str, + prefix: str, + public_base_url: str, + region: str = "us-east-1", + endpoint_url: str | None = None, +) -> list[UploadedScreenshot]: + """Upload PNG screenshots and return stable public URLs.""" + screenshots = sorted(path for path in screenshots_dir.rglob("*.png") if path.is_file()) + if not screenshots: + return [] + + client = _s3_client(region, endpoint_url) + uploads: list[UploadedScreenshot] = [] + + for screenshot in screenshots: + key = object_key(prefix, build_id, screenshot.name) + content_type = mimetypes.guess_type(screenshot.name)[0] or "image/png" + extra_args = { + "ContentType": content_type, + "CacheControl": "public, max-age=31536000, immutable", + } + if os.environ.get("SCREENSHOTS_S3_USE_ACL", "false").lower() == "true": + extra_args["ACL"] = "public-read" + + client.upload_file( + str(screenshot), + bucket, + key, + ExtraArgs=extra_args, + ) + uploads.append( + UploadedScreenshot( + name=screenshot.name, + key=key, + url=public_url_for_key(public_base_url, key), + ) + ) + + return uploads + + +def write_manifest(path: Path, build_id: str, uploads: list[UploadedScreenshot]) -> None: + payload = { + "build_id": build_id, + "screenshots": [ + {"name": item.name, "key": item.key, "url": item.url} + for item in uploads + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2)) From cf5c986e194768d76812aca58f07c1777e688ea8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 28 Jun 2026 19:23:34 +0000 Subject: [PATCH 04/10] Add Imgur as the simple auth-less screenshot upload backend Imgur only needs a free Client-ID for anonymous uploads, which is much lighter than S3. Auto-detect prefers Imgur when IMGUR_CLIENT_ID is set. Co-authored-by: Greg --- Bedtime/ci_scripts/ci_post_xcodebuild.sh | 8 +- scripts/README.md | 99 ++++++++-------- scripts/fetch_xcode_cloud_screenshots.py | 24 ++-- scripts/tests/test_upload.py | 55 ++++++++- scripts/xcode_cloud/upload.py | 137 ++++++++++++++++++----- 5 files changed, 225 insertions(+), 98 deletions(-) diff --git a/Bedtime/ci_scripts/ci_post_xcodebuild.sh b/Bedtime/ci_scripts/ci_post_xcodebuild.sh index 4b51b34..a2acecd 100755 --- a/Bedtime/ci_scripts/ci_post_xcodebuild.sh +++ b/Bedtime/ci_scripts/ci_post_xcodebuild.sh @@ -36,9 +36,11 @@ echo "ci_post_xcodebuild: exporting screenshots from $CI_RESULT_BUNDLE_PATH" # shellcheck disable=SC2086 "$PYTHON_BIN" "$FETCH_SCRIPT" $EXTRACT_ARGS -if [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then - echo "ci_post_xcodebuild: uploading screenshots to s3://$SCREENSHOTS_S3_BUCKET" - "$PYTHON_BIN" -m pip install --quiet boto3 +if [ -n "${IMGUR_CLIENT_ID:-}" ] || [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then + echo "ci_post_xcodebuild: uploading screenshots to public image host" + if [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then + "$PYTHON_BIN" -m pip install --quiet boto3 + fi "$PYTHON_BIN" "$FETCH_SCRIPT" upload-screenshots \ --screenshots-dir "$SCREENSHOTS_DIR" \ --build-id "$BUILD_ID" \ diff --git a/scripts/README.md b/scripts/README.md index a403636..85471e1 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,66 +2,71 @@ Fetch UI test screenshots from Xcode Cloud after a build completes. -## Recommended flow: Xcode Cloud uploads to a public bucket +## Recommended flow: Imgur upload from Xcode Cloud -PR comments need **stable, public image URLs**. Apple's artifact `downloadUrl` values expire, and extraction requires **macOS + Xcode** (`xcresulttool`). - -The recommended path is to do everything on the **Xcode Cloud Mac** in `Bedtime/ci_scripts/ci_post_xcodebuild.sh`: +PR comments need **stable, public image URLs**. The simplest setup is **Imgur** — one free Client-ID, no AWS account, no bucket policies. ```text UI tests finish on Xcode Cloud (macOS) → extract PNGs from CI_RESULT_BUNDLE_PATH [macOS only] - → upload to public S3/R2 bucket [stable URLs] - → post PR comment with embedded images + → upload to Imgur (anonymous) [IMGUR_CLIENT_ID only] + → post PR comment with ![...](https://i.imgur.com/...) +``` + +Register an app at [api.imgur.com/oauth2/addclient](https://api.imgur.com/oauth2/addclient) (choose “anonymous usage without user authorization”), then add one Xcode Cloud secret: + +```bash +IMGUR_CLIENT_ID="your-client-id" +``` + +Optional PR comment secrets: + +```bash +GITHUB_TOKEN="..." +GITHUB_REPOSITORY="owner/repo" +GITHUB_PULL_REQUEST="123" ``` -Anything that runs on Linux (Cursor Cloud Agent, webhook handler, `trigger-and-fetch` without `--skip-extract`) can **trigger** builds and **download** bundles, but cannot extract screenshots unless you move the `.xcresult` to a Mac. +`Bedtime/ci_scripts/ci_post_xcodebuild.sh` handles extract → upload → comment automatically. + +### Imgur caveats + +- Images are **public** on Imgur +- Free tier has **rate limits** (~1,250 uploads/day per Client-ID) +- Imgur’s terms restrict **commercial** use on the free API — fine for side projects / internal CI, not for a production SaaS ## macOS requirement (extraction) | Step | Runs on | Tool | |------|---------|------| | Trigger build | anywhere | App Store Connect API | -| Wait for completion | anywhere | App Store Connect API | -| Download `.xcresult` | anywhere | App Store Connect API | +| Wait / download `.xcresult` | anywhere | App Store Connect API | | **Extract PNGs** | **macOS only** | `xcresulttool` (ships with Xcode) | -| Upload to public bucket | anywhere with AWS creds | `boto3` | -| Post PR comment | anywhere | GitHub API | +| Upload to Imgur | anywhere | `IMGUR_CLIENT_ID` | +| Post PR comment | anywhere | `GITHUB_TOKEN` | + +On Linux agents, use `--skip-extract` and let `ci_post_xcodebuild.sh` do extraction on the Xcode Cloud Mac. -On Linux, use `--skip-extract` to download the bundle only. Do not expect `fetch` or `trigger-and-fetch` to produce PNGs on a Linux agent. +## Alternatives to Imgur -## Xcode Cloud setup (public bucket + PR embed) +| Option | Auth needed | Notes | +|--------|-------------|-------| +| **Imgur** | Client-ID only | Easiest; recommended default | +| **S3 / R2** | AWS keys + bucket policy | More control; set `SCREENSHOTS_S3_BUCKET` | +| **GitHub drag-and-drop URLs** | Browser session | No stable API with `GITHUB_TOKEN` alone | -Add these as **Xcode Cloud workflow secrets**: +S3 is still supported if you set `SCREENSHOTS_UPLOAD_BACKEND=s3` or only configure S3 env vars. + +## Xcode Cloud secrets (S3 alternative) ```bash -# S3 or S3-compatible bucket with public read on the prefix (bucket policy, or CDN in front) SCREENSHOTS_S3_BUCKET="my-public-screenshots" -SCREENSHOTS_S3_PREFIX="bedtime" -SCREENSHOTS_PUBLIC_BASE_URL="https://my-public-screenshots.s3.amazonaws.com" # or CloudFront URL +SCREENSHOTS_PUBLIC_BASE_URL="https://cdn.example.com" AWS_ACCESS_KEY_ID="..." AWS_SECRET_ACCESS_KEY="..." - -# Optional: PR comment with embedded images -GITHUB_TOKEN="..." -GITHUB_REPOSITORY="owner/repo" -GITHUB_PULL_REQUEST="123" # set via custom env / script if not automatic -``` - -For R2 or other S3-compatible storage: - -```bash -SCREENSHOTS_S3_ENDPOINT_URL="https://.r2.cloudflarestorage.com" -SCREENSHOTS_PUBLIC_BASE_URL="https://screenshots.example.com" ``` -`ci_post_xcodebuild.sh` will: - -1. Extract screenshots from `CI_RESULT_BUNDLE_PATH` -2. Upload PNGs when `SCREENSHOTS_S3_BUCKET` is set -3. Post a PR comment with `![name](url)` markdown when GitHub env vars are set - -## App Store Connect API (trigger / fetch from outside Xcode Cloud) +## App Store Connect API (trigger from outside Xcode Cloud) ```bash export APP_STORE_CONNECT_KEY_ID="..." @@ -69,28 +74,22 @@ export APP_STORE_CONNECT_ISSUER_ID="..." export APP_STORE_CONNECT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" ``` -### Fully synchronous trigger + wait + fetch - ```bash +# Trigger, block until done, fetch (use --skip-extract on Linux) python3 scripts/fetch_xcode_cloud_screenshots.py trigger-and-fetch \ --workflow-id WORKFLOW_ID \ --branch main ``` -On Linux, add `--skip-extract` and rely on `ci_post_xcodebuild.sh` for extraction + upload. - -### Wait for an existing build, then fetch +## Manual upload + PR comment ```bash -python3 scripts/fetch_xcode_cloud_screenshots.py wait-and-fetch --run-id BUILD_RUN_ID -``` - -### Manual upload + PR comment (after extraction on a Mac) +export IMGUR_CLIENT_ID="..." -```bash python3 scripts/fetch_xcode_cloud_screenshots.py upload-screenshots \ --screenshots-dir ./xcode-cloud-output/screenshots \ - --build-id BUILD_RUN_ID + --build-id BUILD_RUN_ID \ + --backend imgur python3 scripts/fetch_xcode_cloud_screenshots.py comment-pr \ --repo owner/repo \ @@ -99,14 +98,10 @@ python3 scripts/fetch_xcode_cloud_screenshots.py comment-pr \ --manifest ./xcode-cloud-output/screenshots-manifest.json ``` -## Local development +## Tests ```bash cd scripts python3 -m pip install -r requirements-dev.txt python3 -m pytest ``` - -## Why not a GitHub Action? - -A GitHub Action is optional. It only helps if you want macOS extraction **outside** Xcode Cloud. If Xcode Cloud already runs your UI tests, `ci_post_xcodebuild.sh` is the simpler place to extract and upload — no second macOS runner. diff --git a/scripts/fetch_xcode_cloud_screenshots.py b/scripts/fetch_xcode_cloud_screenshots.py index 075bb8b..13eb10b 100644 --- a/scripts/fetch_xcode_cloud_screenshots.py +++ b/scripts/fetch_xcode_cloud_screenshots.py @@ -24,7 +24,6 @@ from scripts.xcode_cloud.upload import ( UploadConfigError, UploadedScreenshot, - upload_config_from_env, upload_screenshots, write_manifest, ) @@ -127,6 +126,12 @@ def build_parser() -> argparse.ArgumentParser: ) upload_parser.add_argument("--screenshots-dir", required=True) upload_parser.add_argument("--build-id", required=True) + upload_parser.add_argument( + "--backend", + choices=["auto", "imgur", "s3"], + default="auto", + help="Upload backend (default: auto-detect from env)", + ) upload_parser.add_argument( "--manifest", default="./xcode-cloud-output/screenshots-manifest.json", @@ -180,22 +185,17 @@ def main(argv: list[str] | None = None) -> int: if args.command == "upload-screenshots": try: - config = upload_config_from_env() + uploads = upload_screenshots( + Path(args.screenshots_dir), + build_id=args.build_id, + backend=args.backend, + ) except UploadConfigError as error: parser.error(str(error)) - uploads = upload_screenshots( - Path(args.screenshots_dir), - build_id=args.build_id, - bucket=config["bucket"], - prefix=config["prefix"], - public_base_url=config["public_base_url"], - region=config["region"], - endpoint_url=config["endpoint_url"], - ) manifest_path = Path(args.manifest) write_manifest(manifest_path, args.build_id, uploads) - print(f"Uploaded {len(uploads)} screenshot(s)") + print(f"Uploaded {len(uploads)} screenshot(s) via {args.backend}") for item in uploads: print(item.url) print(f"Manifest: {manifest_path}") diff --git a/scripts/tests/test_upload.py b/scripts/tests/test_upload.py index d5981b0..ef03931 100644 --- a/scripts/tests/test_upload.py +++ b/scripts/tests/test_upload.py @@ -1,3 +1,6 @@ +from pathlib import Path + +import httpx import pytest from scripts.xcode_cloud.upload import ( @@ -5,7 +8,9 @@ UploadedScreenshot, object_key, public_url_for_key, + upload_backend_from_env, upload_config_from_env, + upload_to_imgur, write_manifest, ) @@ -19,6 +24,25 @@ def test_public_url_for_key(): assert url == "https://cdn.example.com/shots/bedtime/run-1/home.png" +def test_upload_backend_prefers_imgur(monkeypatch): + monkeypatch.setenv("IMGUR_CLIENT_ID", "imgur-id") + monkeypatch.setenv("SCREENSHOTS_S3_BUCKET", "my-bucket") + assert upload_backend_from_env() == "imgur" + + +def test_upload_backend_uses_s3_when_configured(monkeypatch): + monkeypatch.delenv("IMGUR_CLIENT_ID", raising=False) + monkeypatch.setenv("SCREENSHOTS_S3_BUCKET", "my-bucket") + assert upload_backend_from_env() == "s3" + + +def test_upload_backend_requires_configuration(monkeypatch): + monkeypatch.delenv("IMGUR_CLIENT_ID", raising=False) + monkeypatch.delenv("SCREENSHOTS_S3_BUCKET", raising=False) + with pytest.raises(UploadConfigError, match="No upload backend"): + upload_backend_from_env() + + def test_upload_config_from_env(monkeypatch): monkeypatch.setenv("SCREENSHOTS_S3_BUCKET", "my-bucket") monkeypatch.setenv("SCREENSHOTS_S3_PREFIX", "bedtime") @@ -37,12 +61,37 @@ def test_upload_config_requires_bucket(monkeypatch): upload_config_from_env() +def test_upload_to_imgur(tmp_path): + image = tmp_path / "home.png" + image.write_bytes(b"fakepng") + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.host == "api.imgur.com" + assert request.headers["Authorization"] == "Client-ID test-client" + return httpx.Response( + 200, + json={ + "success": True, + "data": { + "id": "abc123", + "link": "https://i.imgur.com/abc123.png", + }, + }, + ) + + client = httpx.Client(transport=httpx.MockTransport(handler)) + uploaded = upload_to_imgur(image, client_id="test-client", http_client=client) + + assert uploaded.url == "https://i.imgur.com/abc123.png" + assert uploaded.key == "abc123" + + def test_write_manifest(tmp_path): uploads = [ UploadedScreenshot( name="home.png", - key="bedtime/run-1/home.png", - url="https://cdn.example.com/bedtime/run-1/home.png", + key="abc123", + url="https://i.imgur.com/abc123.png", ) ] manifest_path = tmp_path / "manifest.json" @@ -50,4 +99,4 @@ def test_write_manifest(tmp_path): payload = manifest_path.read_text() assert "home.png" in payload - assert "https://cdn.example.com/bedtime/run-1/home.png" in payload + assert "https://i.imgur.com/abc123.png" in payload diff --git a/scripts/xcode_cloud/upload.py b/scripts/xcode_cloud/upload.py index 58231f6..71022d0 100644 --- a/scripts/xcode_cloud/upload.py +++ b/scripts/xcode_cloud/upload.py @@ -1,12 +1,18 @@ -"""Upload extracted screenshots to a public S3-compatible bucket.""" +"""Upload extracted screenshots to a public image host.""" from __future__ import annotations +import base64 import json import mimetypes import os from dataclasses import dataclass from pathlib import Path +from typing import Literal + +import httpx + +UploadBackend = Literal["auto", "imgur", "s3"] class UploadConfigError(ValueError): @@ -20,10 +26,28 @@ class UploadedScreenshot: url: str +def upload_backend_from_env() -> UploadBackend: + explicit = os.environ.get("SCREENSHOTS_UPLOAD_BACKEND", "auto").strip().lower() + if explicit in {"imgur", "s3"}: + return explicit # type: ignore[return-value] + if explicit != "auto": + raise UploadConfigError( + "SCREENSHOTS_UPLOAD_BACKEND must be one of: auto, imgur, s3" + ) + if os.environ.get("IMGUR_CLIENT_ID", "").strip(): + return "imgur" + if os.environ.get("SCREENSHOTS_S3_BUCKET", "").strip(): + return "s3" + raise UploadConfigError( + "No upload backend configured. Set IMGUR_CLIENT_ID (simplest) or " + "SCREENSHOTS_S3_BUCKET." + ) + + def upload_config_from_env() -> dict[str, str]: bucket = os.environ.get("SCREENSHOTS_S3_BUCKET", "").strip() if not bucket: - raise UploadConfigError("SCREENSHOTS_S3_BUCKET is required for upload") + raise UploadConfigError("SCREENSHOTS_S3_BUCKET is required for S3 upload") public_base_url = os.environ.get("SCREENSHOTS_PUBLIC_BASE_URL", "").strip() if not public_base_url: @@ -59,8 +83,38 @@ def _s3_client(region: str, endpoint_url: str | None): return boto3.client("s3", region_name=region, endpoint_url=endpoint_url) -def upload_screenshots( - screenshots_dir: Path, +def upload_to_imgur( + screenshot: Path, + *, + client_id: str, + http_client: httpx.Client | None = None, +) -> UploadedScreenshot: + """Upload one image anonymously to Imgur using only a Client-ID.""" + http = http_client or httpx.Client(timeout=60.0) + close_client = http_client is None + try: + response = http.post( + "https://api.imgur.com/3/image", + headers={"Authorization": f"Client-ID {client_id}"}, + data={"image": base64.b64encode(screenshot.read_bytes()).decode("ascii")}, + ) + response.raise_for_status() + payload = response.json() + if not payload.get("success"): + raise RuntimeError(f"Imgur upload failed: {payload}") + data = payload["data"] + return UploadedScreenshot( + name=screenshot.name, + key=data.get("id", screenshot.name), + url=data["link"], + ) + finally: + if close_client: + http.close() + + +def upload_to_s3( + screenshot: Path, *, build_id: str, bucket: str, @@ -68,40 +122,67 @@ def upload_screenshots( public_base_url: str, region: str = "us-east-1", endpoint_url: str | None = None, +) -> UploadedScreenshot: + key = object_key(prefix, build_id, screenshot.name) + content_type = mimetypes.guess_type(screenshot.name)[0] or "image/png" + extra_args = { + "ContentType": content_type, + "CacheControl": "public, max-age=31536000, immutable", + } + if os.environ.get("SCREENSHOTS_S3_USE_ACL", "false").lower() == "true": + extra_args["ACL"] = "public-read" + + client = _s3_client(region, endpoint_url) + client.upload_file(str(screenshot), bucket, key, ExtraArgs=extra_args) + return UploadedScreenshot( + name=screenshot.name, + key=key, + url=public_url_for_key(public_base_url, key), + ) + + +def upload_screenshots( + screenshots_dir: Path, + *, + build_id: str, + backend: UploadBackend = "auto", + http_client: httpx.Client | None = None, ) -> list[UploadedScreenshot]: """Upload PNG screenshots and return stable public URLs.""" screenshots = sorted(path for path in screenshots_dir.rglob("*.png") if path.is_file()) if not screenshots: return [] - client = _s3_client(region, endpoint_url) + resolved_backend = backend if backend != "auto" else upload_backend_from_env() uploads: list[UploadedScreenshot] = [] - for screenshot in screenshots: - key = object_key(prefix, build_id, screenshot.name) - content_type = mimetypes.guess_type(screenshot.name)[0] or "image/png" - extra_args = { - "ContentType": content_type, - "CacheControl": "public, max-age=31536000, immutable", - } - if os.environ.get("SCREENSHOTS_S3_USE_ACL", "false").lower() == "true": - extra_args["ACL"] = "public-read" - - client.upload_file( - str(screenshot), - bucket, - key, - ExtraArgs=extra_args, - ) - uploads.append( - UploadedScreenshot( - name=screenshot.name, - key=key, - url=public_url_for_key(public_base_url, key), + if resolved_backend == "imgur": + client_id = os.environ.get("IMGUR_CLIENT_ID", "").strip() + if not client_id: + raise UploadConfigError("IMGUR_CLIENT_ID is required for Imgur upload") + for screenshot in screenshots: + uploads.append( + upload_to_imgur(screenshot, client_id=client_id, http_client=http_client) ) - ) + return uploads + + if resolved_backend == "s3": + config = upload_config_from_env() + for screenshot in screenshots: + uploads.append( + upload_to_s3( + screenshot, + build_id=build_id, + bucket=config["bucket"], + prefix=config["prefix"], + public_base_url=config["public_base_url"], + region=config["region"], + endpoint_url=config["endpoint_url"], + ) + ) + return uploads - return uploads + raise UploadConfigError(f"Unsupported upload backend: {resolved_backend}") def write_manifest(path: Path, build_id: str, uploads: list[UploadedScreenshot]) -> None: From a100642b87cc77ed8f8e3b5e0371e144d803983d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 29 Jun 2026 04:18:23 +0000 Subject: [PATCH 05/10] Add XCUITest screenshot tests and remove Python script tests Add BedtimeUITests with ScreenshotTests that capture home and settings screens via XCTAttachment (.keepAlways) for Xcode Cloud PR previews. Support -ui_testing launch argument with mock sleep data so simulator runs do not require HealthKit authorization. Remove the pytest suite for the Python fetch/upload helpers; those scripts remain as optional post-build tooling. Co-authored-by: Greg --- Bedtime/Bedtime.xcodeproj/project.pbxproj | 120 ++++++++++++++++++ Bedtime/Bedtime/ContentView.swift | 6 + Bedtime/Bedtime/Models/HealthKitManager.swift | 9 ++ .../Bedtime/Support/UITestingSupport.swift | 57 +++++++++ Bedtime/Bedtime/Views/SettingsView.swift | 1 + Bedtime/BedtimeUITests/ScreenshotTests.swift | 31 +++++ .../XCTestCase+Screenshots.swift | 10 ++ scripts/README.md | 112 +++++----------- scripts/pytest.ini | 3 - scripts/requirements-dev.txt | 5 - scripts/tests/conftest.py | 15 --- scripts/tests/fixtures/artifact_detail.json | 11 -- scripts/tests/fixtures/artifacts_list.json | 20 --- scripts/tests/fixtures/build_actions.json | 20 --- .../tests/fixtures/build_run_complete.json | 10 -- scripts/tests/test_asc_auth.py | 26 ---- scripts/tests/test_client.py | 37 ------ scripts/tests/test_extract.py | 38 ------ scripts/tests/test_github_pr.py | 31 ----- scripts/tests/test_screenshots.py | 67 ---------- scripts/tests/test_trigger.py | 120 ------------------ scripts/tests/test_upload.py | 102 --------------- 22 files changed, 266 insertions(+), 585 deletions(-) create mode 100644 Bedtime/Bedtime/Support/UITestingSupport.swift create mode 100644 Bedtime/BedtimeUITests/ScreenshotTests.swift create mode 100644 Bedtime/BedtimeUITests/XCTestCase+Screenshots.swift delete mode 100644 scripts/pytest.ini delete mode 100644 scripts/requirements-dev.txt delete mode 100644 scripts/tests/conftest.py delete mode 100644 scripts/tests/fixtures/artifact_detail.json delete mode 100644 scripts/tests/fixtures/artifacts_list.json delete mode 100644 scripts/tests/fixtures/build_actions.json delete mode 100644 scripts/tests/fixtures/build_run_complete.json delete mode 100644 scripts/tests/test_asc_auth.py delete mode 100644 scripts/tests/test_client.py delete mode 100644 scripts/tests/test_extract.py delete mode 100644 scripts/tests/test_github_pr.py delete mode 100644 scripts/tests/test_screenshots.py delete mode 100644 scripts/tests/test_trigger.py delete mode 100644 scripts/tests/test_upload.py diff --git a/Bedtime/Bedtime.xcodeproj/project.pbxproj b/Bedtime/Bedtime.xcodeproj/project.pbxproj index 386fa6d..c47e6ed 100644 --- a/Bedtime/Bedtime.xcodeproj/project.pbxproj +++ b/Bedtime/Bedtime.xcodeproj/project.pbxproj @@ -6,8 +6,19 @@ objectVersion = 77; objects = { +/* Begin PBXContainerItemProxy section */ + 373AAF072E91B7E900D8ED84 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 373AAED52E91B7E900D8ED84 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 373AAEDC2E91B7E900D8ED84; + remoteInfo = Bedtime; + }; +/* End PBXContainerItemProxy section */ + /* Begin PBXFileReference section */ 373AAEDD2E91B7E900D8ED84 /* Bedger.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Bedger.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 373AAF022E91B7E900D8ED84 /* BedtimeUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = BedtimeUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ /* Begin PBXFileSystemSynchronizedRootGroup section */ @@ -16,6 +27,11 @@ path = Bedtime; sourceTree = ""; }; + 373AAF012E91B7E900D8ED84 /* BedtimeUITests */ = { + isa = PBXFileSystemSynchronizedRootGroup; + path = BedtimeUITests; + sourceTree = ""; + }; /* End PBXFileSystemSynchronizedRootGroup section */ /* Begin PBXFrameworksBuildPhase section */ @@ -26,6 +42,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 373AAF052E91B7E900D8ED84 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ @@ -33,6 +56,7 @@ isa = PBXGroup; children = ( 373AAEDF2E91B7E900D8ED84 /* Bedtime */, + 373AAF012E91B7E900D8ED84 /* BedtimeUITests */, 373AAEDE2E91B7E900D8ED84 /* Products */, ); sourceTree = ""; @@ -41,6 +65,7 @@ isa = PBXGroup; children = ( 373AAEDD2E91B7E900D8ED84 /* Bedger.app */, + 373AAF022E91B7E900D8ED84 /* BedtimeUITests.xctest */, ); name = Products; sourceTree = ""; @@ -70,6 +95,29 @@ productReference = 373AAEDD2E91B7E900D8ED84 /* Bedger.app */; productType = "com.apple.product-type.application"; }; + 373AAF032E91B7E900D8ED84 /* BedtimeUITests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 373AAF092E91B7EA00D8ED84 /* Build configuration list for PBXNativeTarget "BedtimeUITests" */; + buildPhases = ( + 373AAF042E91B7E900D8ED84 /* Sources */, + 373AAF052E91B7E900D8ED84 /* Frameworks */, + 373AAF062E91B7E900D8ED84 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 373AAF082E91B7E900D8ED84 /* PBXTargetDependency */, + ); + fileSystemSynchronizedGroups = ( + 373AAF012E91B7E900D8ED84 /* BedtimeUITests */, + ); + name = BedtimeUITests; + packageProductDependencies = ( + ); + productName = BedtimeUITests; + productReference = 373AAF022E91B7E900D8ED84 /* BedtimeUITests.xctest */; + productType = "com.apple.product-type.bundle.ui-testing"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ @@ -83,6 +131,10 @@ 373AAEDC2E91B7E900D8ED84 = { CreatedOnToolsVersion = 26.0; }; + 373AAF032E91B7E900D8ED84 = { + CreatedOnToolsVersion = 26.0; + TestTargetID = 373AAEDC2E91B7E900D8ED84; + }; }; }; buildConfigurationList = 373AAED82E91B7E900D8ED84 /* Build configuration list for PBXProject "Bedtime" */; @@ -100,6 +152,7 @@ projectRoot = ""; targets = ( 373AAEDC2E91B7E900D8ED84 /* Bedtime */, + 373AAF032E91B7E900D8ED84 /* BedtimeUITests */, ); }; /* End PBXProject section */ @@ -112,6 +165,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 373AAF062E91B7E900D8ED84 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -122,8 +182,23 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 373AAF042E91B7E900D8ED84 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ +/* Begin PBXTargetDependency section */ + 373AAF082E91B7E900D8ED84 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 373AAEDC2E91B7E900D8ED84 /* Bedtime */; + targetProxy = 373AAF072E91B7E900D8ED84 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + /* Begin XCBuildConfiguration section */ 373AAEEC2E91B7EA00D8ED84 /* Debug */ = { isa = XCBuildConfiguration; @@ -322,6 +397,42 @@ }; name = Release; }; + 373AAF0A2E91B7EA00D8ED84 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 9J44X82D53; + GENERATE_INFOPLIST_FILE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.9.0; + PRODUCT_BUNDLE_IDENTIFIER = com.burnsides.bedtime.uitests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_TARGET_NAME = Bedtime; + }; + name = Debug; + }; + 373AAF0B2E91B7EA00D8ED84 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 9J44X82D53; + GENERATE_INFOPLIST_FILE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.9.0; + PRODUCT_BUNDLE_IDENTIFIER = com.burnsides.bedtime.uitests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_TARGET_NAME = Bedtime; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -343,6 +454,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + 373AAF092E91B7EA00D8ED84 /* Build configuration list for PBXNativeTarget "BedtimeUITests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 373AAF0A2E91B7EA00D8ED84 /* Debug */, + 373AAF0B2E91B7EA00D8ED84 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; /* End XCConfigurationList section */ }; rootObject = 373AAED52E91B7E900D8ED84 /* Project object */; diff --git a/Bedtime/Bedtime/ContentView.swift b/Bedtime/Bedtime/ContentView.swift index b1d87ce..70ac7db 100644 --- a/Bedtime/Bedtime/ContentView.swift +++ b/Bedtime/Bedtime/ContentView.swift @@ -93,6 +93,7 @@ struct ContentView: View { .frame(maxWidth: 600) .frame(maxWidth: .infinity) } + .accessibilityIdentifier("home_screen") .background(Color.backgroundBehindCards) .navigationTitle("Bedger") .toolbar { @@ -100,6 +101,7 @@ struct ContentView: View { Button("Settings", systemImage: "gear") { showingSettings.toggle() } + .accessibilityIdentifier("settings_button") } } .refreshable { @@ -125,6 +127,10 @@ struct ContentView: View { } } .task { + #if DEBUG + healthKitManager.prepareForUITestingIfNeeded() + guard !UITestingSupport.isActive else { return } + #endif try? await healthKitManager.fetchSleepData() } } diff --git a/Bedtime/Bedtime/Models/HealthKitManager.swift b/Bedtime/Bedtime/Models/HealthKitManager.swift index 5d5004e..dd9dbcf 100644 --- a/Bedtime/Bedtime/Models/HealthKitManager.swift +++ b/Bedtime/Bedtime/Models/HealthKitManager.swift @@ -158,4 +158,13 @@ class HealthKitManager: ObservableObject { self.sleepSessions = Dictionary(grouping: sessions) { $0.dateForGrouping } } + + #if DEBUG + func prepareForUITestingIfNeeded() { + guard UITestingSupport.isActive else { return } + isAuthorized = true + sleepSessions = UITestingSupport.mockSleepSessions() + errorMessage = nil + } + #endif } diff --git a/Bedtime/Bedtime/Support/UITestingSupport.swift b/Bedtime/Bedtime/Support/UITestingSupport.swift new file mode 100644 index 0000000..f4e59b5 --- /dev/null +++ b/Bedtime/Bedtime/Support/UITestingSupport.swift @@ -0,0 +1,57 @@ +#if DEBUG +import Foundation +import HealthKit + +enum UITestingSupport { + static var isActive: Bool { + ProcessInfo.processInfo.arguments.contains("-ui_testing") + } + + static func mockSleepSessions() -> [Date: [SleepSession]] { + let source = HKSourceRevision(source: HKSource.default(), version: nil) + let calendar = Calendar.current + let today = calendar.startOfDay(for: Date()) + + func session( + dayOffset: Int, + startHour: Int, + startMinute: Int, + endHour: Int, + endMinute: Int + ) -> SleepSession { + let base = calendar.date(byAdding: .day, value: dayOffset, to: today) ?? today + let start = calendar.date( + bySettingHour: startHour, + minute: startMinute, + second: 0, + of: base + ) ?? base + let endDayOffset = endHour < startHour ? 1 : 0 + let endBase = calendar.date(byAdding: .day, value: endDayOffset, to: base) ?? base + let end = calendar.date( + bySettingHour: endHour, + minute: endMinute, + second: 0, + of: endBase + ) ?? endBase + + return SleepSession( + startDate: start, + endDate: end, + sleepType: .asleepCore, + source: source + ) + } + + let lastNight = session(dayOffset: -1, startHour: 23, startMinute: 15, endHour: 7, endMinute: 5) + let twoNightsAgo = session(dayOffset: -2, startHour: 22, startMinute: 45, endHour: 6, endMinute: 30) + let threeNightsAgo = session(dayOffset: -3, startHour: 23, startMinute: 30, endHour: 7, endMinute: 45) + + return [ + lastNight.dateForGrouping: [lastNight], + twoNightsAgo.dateForGrouping: [twoNightsAgo], + threeNightsAgo.dateForGrouping: [threeNightsAgo], + ] + } +} +#endif diff --git a/Bedtime/Bedtime/Views/SettingsView.swift b/Bedtime/Bedtime/Views/SettingsView.swift index 6b53847..23c1a5e 100644 --- a/Bedtime/Bedtime/Views/SettingsView.swift +++ b/Bedtime/Bedtime/Views/SettingsView.swift @@ -154,6 +154,7 @@ struct SettingsView: View { } .navigationTitle("Settings") .navigationBarTitleDisplayMode(.inline) + .accessibilityIdentifier("settings_screen") .toolbar { if horizontalSizeClass == .compact { ToolbarItem(placement: .confirmationAction) { diff --git a/Bedtime/BedtimeUITests/ScreenshotTests.swift b/Bedtime/BedtimeUITests/ScreenshotTests.swift new file mode 100644 index 0000000..f75400d --- /dev/null +++ b/Bedtime/BedtimeUITests/ScreenshotTests.swift @@ -0,0 +1,31 @@ +import XCTest + +/// UI tests that capture screenshots for pull request previews. +/// +/// Screenshots are saved as XCTAttachments with `.keepAlways` so Xcode Cloud +/// includes them in the test result bundle. `ci_post_xcodebuild.sh` can then +/// upload them to Imgur (or another host) and embed the URLs in a PR comment. +final class ScreenshotTests: XCTestCase { + + override func setUpWithError() throws { + continueAfterFailure = false + } + + @MainActor + func testPullRequestScreenshots() throws { + let app = XCUIApplication() + app.launchArguments.append("-ui_testing") + app.launch() + + XCTAssertTrue(app.scrollViews["home_screen"].waitForExistence(timeout: 10)) + attachScreenshot(named: "01-home", in: app) + + app.buttons["settings_button"].tap() + XCTAssertTrue(app.navigationBars["Settings"].waitForExistence(timeout: 5)) + attachScreenshot(named: "02-settings", in: app) + + if app.buttons["Done"].exists { + app.buttons["Done"].tap() + } + } +} diff --git a/Bedtime/BedtimeUITests/XCTestCase+Screenshots.swift b/Bedtime/BedtimeUITests/XCTestCase+Screenshots.swift new file mode 100644 index 0000000..0683e50 --- /dev/null +++ b/Bedtime/BedtimeUITests/XCTestCase+Screenshots.swift @@ -0,0 +1,10 @@ +import XCTest + +extension XCTestCase { + func attachScreenshot(named name: String, in app: XCUIApplication, file: StaticString = #filePath, line: UInt = #line) { + let attachment = XCTAttachment(screenshot: app.screenshot()) + attachment.name = name + attachment.lifetime = .keepAlways + add(attachment) + } +} diff --git a/scripts/README.md b/scripts/README.md index 85471e1..d08a4c6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,107 +1,59 @@ -# Xcode Cloud screenshot fetcher +# Xcode Cloud screenshots -Fetch UI test screenshots from Xcode Cloud after a build completes. +## What generates the screenshots -## Recommended flow: Imgur upload from Xcode Cloud +**`BedtimeUITests/ScreenshotTests.swift`** — XCUITest UI tests that capture PNG attachments for pull request previews. -PR comments need **stable, public image URLs**. The simplest setup is **Imgur** — one free Client-ID, no AWS account, no bucket policies. +The test launches the app with `-ui_testing` (mock sleep data, no HealthKit prompt) and saves screenshots with `XCTAttachment` lifetime `.keepAlways` so Xcode Cloud keeps them in the test result bundle. -```text -UI tests finish on Xcode Cloud (macOS) - → extract PNGs from CI_RESULT_BUNDLE_PATH [macOS only] - → upload to Imgur (anonymous) [IMGUR_CLIENT_ID only] - → post PR comment with ![...](https://i.imgur.com/...) -``` +Add `BedtimeUITests` to your Xcode Cloud workflow's **Test** action. In the test plan, set **Screenshots** to **On, and keep all** if you want images even when tests pass. -Register an app at [api.imgur.com/oauth2/addclient](https://api.imgur.com/oauth2/addclient) (choose “anonymous usage without user authorization”), then add one Xcode Cloud secret: +### Run locally ```bash -IMGUR_CLIENT_ID="your-client-id" -``` - -Optional PR comment secrets: - -```bash -GITHUB_TOKEN="..." -GITHUB_REPOSITORY="owner/repo" -GITHUB_PULL_REQUEST="123" +xcodebuild test \ + -project Bedtime/Bedtime.xcodeproj \ + -scheme Bedtime \ + -destination 'platform=iOS Simulator,name=iPhone 16' \ + -only-testing:BedtimeUITests/ScreenshotTests ``` -`Bedtime/ci_scripts/ci_post_xcodebuild.sh` handles extract → upload → comment automatically. - -### Imgur caveats - -- Images are **public** on Imgur -- Free tier has **rate limits** (~1,250 uploads/day per Client-ID) -- Imgur’s terms restrict **commercial** use on the free API — fine for side projects / internal CI, not for a production SaaS - -## macOS requirement (extraction) - -| Step | Runs on | Tool | -|------|---------|------| -| Trigger build | anywhere | App Store Connect API | -| Wait / download `.xcresult` | anywhere | App Store Connect API | -| **Extract PNGs** | **macOS only** | `xcresulttool` (ships with Xcode) | -| Upload to Imgur | anywhere | `IMGUR_CLIENT_ID` | -| Post PR comment | anywhere | `GITHUB_TOKEN` | - -On Linux agents, use `--skip-extract` and let `ci_post_xcodebuild.sh` do extraction on the Xcode Cloud Mac. +## What happens after tests (Xcode Cloud Mac) -## Alternatives to Imgur +`Bedtime/ci_scripts/ci_post_xcodebuild.sh`: -| Option | Auth needed | Notes | -|--------|-------------|-------| -| **Imgur** | Client-ID only | Easiest; recommended default | -| **S3 / R2** | AWS keys + bucket policy | More control; set `SCREENSHOTS_S3_BUCKET` | -| **GitHub drag-and-drop URLs** | Browser session | No stable API with `GITHUB_TOKEN` alone | +1. Extracts PNGs from `CI_RESULT_BUNDLE_PATH` via `xcresulttool` +2. Uploads to **Imgur** (`IMGUR_CLIENT_ID`) or **S3** (`SCREENSHOTS_S3_BUCKET`) +3. Optionally posts a PR comment with embedded image URLs -S3 is still supported if you set `SCREENSHOTS_UPLOAD_BACKEND=s3` or only configure S3 env vars. +### Simplest upload setup (Imgur) -## Xcode Cloud secrets (S3 alternative) +Register at [api.imgur.com/oauth2/addclient](https://api.imgur.com/oauth2/addclient), then add one Xcode Cloud secret: ```bash -SCREENSHOTS_S3_BUCKET="my-public-screenshots" -SCREENSHOTS_PUBLIC_BASE_URL="https://cdn.example.com" -AWS_ACCESS_KEY_ID="..." -AWS_SECRET_ACCESS_KEY="..." +IMGUR_CLIENT_ID="your-client-id" ``` -## App Store Connect API (trigger from outside Xcode Cloud) +Optional PR comment: ```bash -export APP_STORE_CONNECT_KEY_ID="..." -export APP_STORE_CONNECT_ISSUER_ID="..." -export APP_STORE_CONNECT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----" +GITHUB_TOKEN="..." +GITHUB_REPOSITORY="owner/repo" +GITHUB_PULL_REQUEST="123" ``` +## Optional: fetch scripts (outside Xcode Cloud) + +`scripts/fetch_xcode_cloud_screenshots.py` can trigger builds, wait for completion, and download result bundles via the App Store Connect API. Extraction still requires macOS (`xcresulttool`). + ```bash -# Trigger, block until done, fetch (use --skip-extract on Linux) +export APP_STORE_CONNECT_KEY_ID=... +export APP_STORE_CONNECT_ISSUER_ID=... +export APP_STORE_CONNECT_PRIVATE_KEY='...' + python3 scripts/fetch_xcode_cloud_screenshots.py trigger-and-fetch \ --workflow-id WORKFLOW_ID \ --branch main ``` -## Manual upload + PR comment - -```bash -export IMGUR_CLIENT_ID="..." - -python3 scripts/fetch_xcode_cloud_screenshots.py upload-screenshots \ - --screenshots-dir ./xcode-cloud-output/screenshots \ - --build-id BUILD_RUN_ID \ - --backend imgur - -python3 scripts/fetch_xcode_cloud_screenshots.py comment-pr \ - --repo owner/repo \ - --pr-number 123 \ - --run-id BUILD_RUN_ID \ - --manifest ./xcode-cloud-output/screenshots-manifest.json -``` - -## Tests - -```bash -cd scripts -python3 -m pip install -r requirements-dev.txt -python3 -m pytest -``` +On Linux, use `--skip-extract` — let `ci_post_xcodebuild.sh` handle extraction on Apple's Mac. diff --git a/scripts/pytest.ini b/scripts/pytest.ini deleted file mode 100644 index 5cc30ab..0000000 --- a/scripts/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -testpaths = tests -pythonpath = .. diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt deleted file mode 100644 index c0367e5..0000000 --- a/scripts/requirements-dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -pytest>=8.0 -httpx>=0.27 -PyJWT[crypto]>=2.8 -cryptography>=42.0 -boto3>=1.34 diff --git a/scripts/tests/conftest.py b/scripts/tests/conftest.py deleted file mode 100644 index 66f7988..0000000 --- a/scripts/tests/conftest.py +++ /dev/null @@ -1,15 +0,0 @@ -import json -from pathlib import Path - -import pytest - - -FIXTURES = Path(__file__).parent / "fixtures" - - -@pytest.fixture -def load_fixture(): - def _load(name: str): - return json.loads((FIXTURES / name).read_text()) - - return _load diff --git a/scripts/tests/fixtures/artifact_detail.json b/scripts/tests/fixtures/artifact_detail.json deleted file mode 100644 index 25513ac..0000000 --- a/scripts/tests/fixtures/artifact_detail.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "data": { - "type": "ciArtifacts", - "id": "artifact-test-1", - "attributes": { - "fileType": "TEST_RESULT_BUNDLE", - "fileName": "TestResults.zip", - "downloadUrl": "https://example.com/TestResults.zip" - } - } -} diff --git a/scripts/tests/fixtures/artifacts_list.json b/scripts/tests/fixtures/artifacts_list.json deleted file mode 100644 index adf2813..0000000 --- a/scripts/tests/fixtures/artifacts_list.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "data": [ - { - "type": "ciArtifacts", - "id": "artifact-log-1", - "attributes": { - "fileType": "LOG_BUNDLE", - "fileName": "logs.zip" - } - }, - { - "type": "ciArtifacts", - "id": "artifact-test-1", - "attributes": { - "fileType": "TEST_RESULT_BUNDLE", - "fileName": "TestResults.zip" - } - } - ] -} diff --git a/scripts/tests/fixtures/build_actions.json b/scripts/tests/fixtures/build_actions.json deleted file mode 100644 index 3ffeb2b..0000000 --- a/scripts/tests/fixtures/build_actions.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "data": [ - { - "type": "ciBuildActions", - "id": "action-build-1", - "attributes": { - "name": "build", - "actionType": "ARCHIVE" - } - }, - { - "type": "ciBuildActions", - "id": "action-test-1", - "attributes": { - "name": "test", - "actionType": "TEST" - } - } - ] -} diff --git a/scripts/tests/fixtures/build_run_complete.json b/scripts/tests/fixtures/build_run_complete.json deleted file mode 100644 index 2d89720..0000000 --- a/scripts/tests/fixtures/build_run_complete.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "data": { - "type": "ciBuildRuns", - "id": "build-run-123", - "attributes": { - "executionProgress": "COMPLETE", - "completionStatus": "SUCCEEDED" - } - } -} diff --git a/scripts/tests/test_asc_auth.py b/scripts/tests/test_asc_auth.py deleted file mode 100644 index 5ebed07..0000000 --- a/scripts/tests/test_asc_auth.py +++ /dev/null @@ -1,26 +0,0 @@ -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import ec - -from scripts.xcode_cloud.asc_auth import AscCredentials, create_asc_token - - -def _generate_test_key() -> str: - private_key = ec.generate_private_key(ec.SECP256R1()) - return private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption(), - ).decode() - - -def test_create_asc_token_contains_expected_claims(): - credentials = AscCredentials( - key_id="KEY123", - issuer_id="issuer-uuid", - private_key=_generate_test_key(), - ) - - token = create_asc_token(credentials, expiration_seconds=600) - - assert isinstance(token, str) - assert token.count(".") == 2 diff --git a/scripts/tests/test_client.py b/scripts/tests/test_client.py deleted file mode 100644 index aa4a833..0000000 --- a/scripts/tests/test_client.py +++ /dev/null @@ -1,37 +0,0 @@ -import json - -import httpx - -from scripts.xcode_cloud.client import XcodeCloudClient - - -def test_get_build_run_status(load_fixture): - fixture = load_fixture("build_run_complete.json") - - def handler(request: httpx.Request) -> httpx.Response: - assert request.url.path.endswith("/v1/ciBuildRuns/build-run-123") - return httpx.Response(200, json=fixture) - - client = httpx.Client(transport=httpx.MockTransport(handler)) - api = XcodeCloudClient(lambda: "token", client=client) - - status = api.get_build_run_status("build-run-123") - - assert status.execution_progress == "COMPLETE" - assert status.completion_status == "SUCCEEDED" - - -def test_list_build_actions(load_fixture): - fixture = load_fixture("build_actions.json") - - def handler(request: httpx.Request) -> httpx.Response: - assert request.url.path.endswith("/v1/ciBuildRuns/build-run-123/actions") - return httpx.Response(200, json=fixture) - - client = httpx.Client(transport=httpx.MockTransport(handler)) - api = XcodeCloudClient(lambda: "token", client=client) - - actions = api.list_build_actions("build-run-123") - - assert len(actions) == 2 - assert actions[1]["attributes"]["actionType"] == "TEST" diff --git a/scripts/tests/test_extract.py b/scripts/tests/test_extract.py deleted file mode 100644 index 53c5c66..0000000 --- a/scripts/tests/test_extract.py +++ /dev/null @@ -1,38 +0,0 @@ -from pathlib import Path -from unittest.mock import patch - -import pytest - -from scripts.xcode_cloud.extract import ( - XcresultToolNotFoundError, - extract_attachments, -) - - -def test_extract_attachments_runs_xcresulttool(tmp_path): - bundle_path = tmp_path / "Test.xcresult" - bundle_path.mkdir() - output_dir = tmp_path / "out" - screenshot = output_dir / "shot.png" - - def fake_run(command, check, capture_output, text): - assert command[0] == "/usr/bin/xcresulttool" - assert command[1:4] == ["export", "attachments", "--path"] - output_dir.mkdir(parents=True, exist_ok=True) - screenshot.write_bytes(b"png") - return None - - with patch("scripts.xcode_cloud.extract.xcresulttool_path", return_value="/usr/bin/xcresulttool"): - with patch("scripts.xcode_cloud.extract.subprocess.run", side_effect=fake_run): - paths = extract_attachments(bundle_path, output_dir) - - assert paths == [screenshot] - - -def test_extract_attachments_requires_xcresulttool(tmp_path): - bundle_path = tmp_path / "Test.xcresult" - bundle_path.mkdir() - - with patch("scripts.xcode_cloud.extract.xcresulttool_path", return_value=None): - with pytest.raises(XcresultToolNotFoundError): - extract_attachments(bundle_path, tmp_path / "out") diff --git a/scripts/tests/test_github_pr.py b/scripts/tests/test_github_pr.py deleted file mode 100644 index 987fca2..0000000 --- a/scripts/tests/test_github_pr.py +++ /dev/null @@ -1,31 +0,0 @@ -from pathlib import Path - -from scripts.xcode_cloud.github_pr import build_screenshot_comment -from scripts.xcode_cloud.upload import UploadedScreenshot - - -def test_build_screenshot_comment_lists_files_without_upload(tmp_path): - shots = [tmp_path / "home.png", tmp_path / "settings.png"] - body = build_screenshot_comment(shots, build_run_id="run-1") - - assert "run-1" in body - assert "home.png" in body - assert "public bucket" in body.lower() - - -def test_build_screenshot_comment_embeds_uploaded_images(): - uploaded = [ - UploadedScreenshot( - name="home.png", - key="bedtime/run-1/home.png", - url="https://cdn.example.com/bedtime/run-1/home.png", - ) - ] - body = build_screenshot_comment([], build_run_id="run-1", uploaded=uploaded) - - assert "![home.png](https://cdn.example.com/bedtime/run-1/home.png)" in body - - -def test_build_screenshot_comment_handles_empty_list(): - body = build_screenshot_comment([], build_run_id="run-1") - assert "no screenshot attachments" in body.lower() diff --git a/scripts/tests/test_screenshots.py b/scripts/tests/test_screenshots.py deleted file mode 100644 index 6157cd8..0000000 --- a/scripts/tests/test_screenshots.py +++ /dev/null @@ -1,67 +0,0 @@ -import io -import zipfile -from pathlib import Path - -import httpx -import pytest - -from scripts.xcode_cloud.client import XcodeCloudClient, XcodeCloudError -from scripts.xcode_cloud.screenshots import ( - fetch_test_result_bundle, - find_test_action, - find_test_result_artifact, -) - - -def test_find_test_action(load_fixture): - actions = load_fixture("build_actions.json")["data"] - action = find_test_action(actions) - assert action["id"] == "action-test-1" - - -def test_find_test_result_artifact(load_fixture): - artifacts = load_fixture("artifacts_list.json")["data"] - artifact = find_test_result_artifact(artifacts) - assert artifact["id"] == "artifact-test-1" - - -def test_find_test_action_raises_when_missing(): - with pytest.raises(XcodeCloudError, match="No TEST build action"): - find_test_action([]) - - -def test_fetch_test_result_bundle_downloads_zip(tmp_path, load_fixture): - actions_fixture = load_fixture("build_actions.json") - artifacts_fixture = load_fixture("artifacts_list.json") - artifact_fixture = load_fixture("artifact_detail.json") - - zip_buffer = io.BytesIO() - with zipfile.ZipFile(zip_buffer, "w") as archive: - archive.writestr("Test.xcresult/Info.plist", "plist") - zip_bytes = zip_buffer.getvalue() - - def handler(request: httpx.Request) -> httpx.Response: - if request.url.path.endswith("/actions"): - return httpx.Response(200, json=actions_fixture) - if request.url.path.endswith("/artifacts") and "ciBuildActions" in request.url.path: - return httpx.Response(200, json=artifacts_fixture) - if request.url.path.endswith("/ciArtifacts/artifact-test-1"): - return httpx.Response(200, json=artifact_fixture) - if request.url.host == "example.com": - return httpx.Response(200, content=zip_bytes) - raise AssertionError(f"Unexpected request: {request.url}") - - api_client = httpx.Client(transport=httpx.MockTransport(handler)) - download_client = httpx.Client(transport=httpx.MockTransport(handler)) - api = XcodeCloudClient(lambda: "token", client=api_client) - - _, artifact_id, bundle_path = fetch_test_result_bundle( - api, - "build-run-123", - tmp_path, - download_client=download_client, - ) - - assert artifact_id == "artifact-test-1" - assert bundle_path.suffix == ".xcresult" - assert bundle_path.exists() diff --git a/scripts/tests/test_trigger.py b/scripts/tests/test_trigger.py deleted file mode 100644 index e707917..0000000 --- a/scripts/tests/test_trigger.py +++ /dev/null @@ -1,120 +0,0 @@ -import httpx - -from scripts.xcode_cloud.client import XcodeCloudClient -from scripts.xcode_cloud.trigger import ( - git_reference_id_for_branch, - repository_id_for_workflow, - trigger_and_wait, - trigger_build_run, -) - - -def test_trigger_build_run_returns_run_id(): - create_response = {"data": {"type": "ciBuildRuns", "id": "new-run-1"}} - - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "POST" and request.url.path.endswith("/v1/ciBuildRuns"): - return httpx.Response(201, json=create_response) - raise AssertionError(f"Unexpected request: {request.method} {request.url}") - - client = httpx.Client(transport=httpx.MockTransport(handler)) - api = XcodeCloudClient(lambda: "token", client=client) - - run_id = trigger_build_run( - api, - "workflow-1", - git_reference_id="git-ref-1", - ) - - assert run_id == "new-run-1" - - -def test_git_reference_id_for_branch(): - references = { - "data": [ - { - "id": "git-ref-main", - "attributes": { - "kind": "BRANCH", - "name": "main", - "canonicalName": "refs/heads/main", - }, - } - ] - } - - def handler(request: httpx.Request) -> httpx.Response: - if request.url.path.endswith("/gitReferences"): - return httpx.Response(200, json=references) - raise AssertionError(f"Unexpected request: {request.url}") - - client = httpx.Client(transport=httpx.MockTransport(handler)) - api = XcodeCloudClient(lambda: "token", client=client) - - ref_id = git_reference_id_for_branch(api, "repo-1", "main") - assert ref_id == "git-ref-main" - - -def test_repository_id_for_workflow_from_included(): - workflow = { - "data": { - "id": "workflow-1", - "type": "ciWorkflows", - "relationships": {}, - }, - "included": [ - {"id": "repo-1", "type": "scmRepositories"}, - ], - } - - def handler(request: httpx.Request) -> httpx.Response: - if request.url.path.endswith("/ciWorkflows/workflow-1"): - return httpx.Response(200, json=workflow) - raise AssertionError(f"Unexpected request: {request.url}") - - client = httpx.Client(transport=httpx.MockTransport(handler)) - api = XcodeCloudClient(lambda: "token", client=client) - - repo_id = repository_id_for_workflow(api, "workflow-1") - assert repo_id == "repo-1" - - -def test_trigger_and_wait_polls_until_complete(monkeypatch): - calls = {"count": 0} - pending = { - "data": { - "id": "run-1", - "attributes": {"executionProgress": "RUNNING", "completionStatus": None}, - } - } - complete = { - "data": { - "id": "run-1", - "attributes": {"executionProgress": "COMPLETE", "completionStatus": "SUCCEEDED"}, - } - } - create_response = {"data": {"type": "ciBuildRuns", "id": "run-1"}} - - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "POST" and request.url.path.endswith("/v1/ciBuildRuns"): - return httpx.Response(201, json=create_response) - if request.url.path.endswith("/ciBuildRuns/run-1"): - calls["count"] += 1 - return httpx.Response(200, json=complete if calls["count"] > 1 else pending) - raise AssertionError(f"Unexpected request: {request.method} {request.url}") - - monkeypatch.setattr("scripts.xcode_cloud.client.time.sleep", lambda _: None) - - client = httpx.Client(transport=httpx.MockTransport(handler)) - api = XcodeCloudClient(lambda: "token", client=client) - - result = trigger_and_wait( - api, - "workflow-1", - git_reference_id="git-ref-1", - poll_interval_seconds=1, - ) - - assert result.build_run_id == "run-1" - assert result.status.completion_status == "SUCCEEDED" - assert calls["count"] >= 2 diff --git a/scripts/tests/test_upload.py b/scripts/tests/test_upload.py deleted file mode 100644 index ef03931..0000000 --- a/scripts/tests/test_upload.py +++ /dev/null @@ -1,102 +0,0 @@ -from pathlib import Path - -import httpx -import pytest - -from scripts.xcode_cloud.upload import ( - UploadConfigError, - UploadedScreenshot, - object_key, - public_url_for_key, - upload_backend_from_env, - upload_config_from_env, - upload_to_imgur, - write_manifest, -) - - -def test_object_key_includes_build_id_and_filename(): - assert object_key("bedtime", "run-1", "home.png") == "bedtime/run-1/home.png" - - -def test_public_url_for_key(): - url = public_url_for_key("https://cdn.example.com/shots", "bedtime/run-1/home.png") - assert url == "https://cdn.example.com/shots/bedtime/run-1/home.png" - - -def test_upload_backend_prefers_imgur(monkeypatch): - monkeypatch.setenv("IMGUR_CLIENT_ID", "imgur-id") - monkeypatch.setenv("SCREENSHOTS_S3_BUCKET", "my-bucket") - assert upload_backend_from_env() == "imgur" - - -def test_upload_backend_uses_s3_when_configured(monkeypatch): - monkeypatch.delenv("IMGUR_CLIENT_ID", raising=False) - monkeypatch.setenv("SCREENSHOTS_S3_BUCKET", "my-bucket") - assert upload_backend_from_env() == "s3" - - -def test_upload_backend_requires_configuration(monkeypatch): - monkeypatch.delenv("IMGUR_CLIENT_ID", raising=False) - monkeypatch.delenv("SCREENSHOTS_S3_BUCKET", raising=False) - with pytest.raises(UploadConfigError, match="No upload backend"): - upload_backend_from_env() - - -def test_upload_config_from_env(monkeypatch): - monkeypatch.setenv("SCREENSHOTS_S3_BUCKET", "my-bucket") - monkeypatch.setenv("SCREENSHOTS_S3_PREFIX", "bedtime") - monkeypatch.setenv("SCREENSHOTS_PUBLIC_BASE_URL", "https://cdn.example.com") - - config = upload_config_from_env() - - assert config["bucket"] == "my-bucket" - assert config["prefix"] == "bedtime" - assert config["public_base_url"] == "https://cdn.example.com" - - -def test_upload_config_requires_bucket(monkeypatch): - monkeypatch.delenv("SCREENSHOTS_S3_BUCKET", raising=False) - with pytest.raises(UploadConfigError, match="SCREENSHOTS_S3_BUCKET"): - upload_config_from_env() - - -def test_upload_to_imgur(tmp_path): - image = tmp_path / "home.png" - image.write_bytes(b"fakepng") - - def handler(request: httpx.Request) -> httpx.Response: - assert request.url.host == "api.imgur.com" - assert request.headers["Authorization"] == "Client-ID test-client" - return httpx.Response( - 200, - json={ - "success": True, - "data": { - "id": "abc123", - "link": "https://i.imgur.com/abc123.png", - }, - }, - ) - - client = httpx.Client(transport=httpx.MockTransport(handler)) - uploaded = upload_to_imgur(image, client_id="test-client", http_client=client) - - assert uploaded.url == "https://i.imgur.com/abc123.png" - assert uploaded.key == "abc123" - - -def test_write_manifest(tmp_path): - uploads = [ - UploadedScreenshot( - name="home.png", - key="abc123", - url="https://i.imgur.com/abc123.png", - ) - ] - manifest_path = tmp_path / "manifest.json" - write_manifest(manifest_path, "run-1", uploads) - - payload = manifest_path.read_text() - assert "home.png" in payload - assert "https://i.imgur.com/abc123.png" in payload From a5de7646fbdb4a84256ab839b0a12d9698f7a330 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 29 Jun 2026 04:44:13 +0000 Subject: [PATCH 06/10] Prefer git-push screenshot triggers over API polling Add trigger-screenshots.sh for branch/tag pushes that start a dedicated screenshots/* Xcode Cloud workflow without App Store Connect credentials. Parse What to test notes from commit messages and auto-detect PR context from CI_PULL_REQUEST_* or screenshots/pr-N branch names in ci_post_xcodebuild. Co-authored-by: Greg --- Bedtime/ci_scripts/ci_post_xcodebuild.sh | 46 ++++++++- scripts/README.md | 87 ++++++++++------ scripts/fetch_xcode_cloud_screenshots.py | 5 + scripts/trigger-screenshots.sh | 123 +++++++++++++++++++++++ scripts/xcode_cloud/github_pr.py | 3 + 5 files changed, 227 insertions(+), 37 deletions(-) create mode 100755 scripts/trigger-screenshots.sh diff --git a/Bedtime/ci_scripts/ci_post_xcodebuild.sh b/Bedtime/ci_scripts/ci_post_xcodebuild.sh index a2acecd..3f00472 100755 --- a/Bedtime/ci_scripts/ci_post_xcodebuild.sh +++ b/Bedtime/ci_scripts/ci_post_xcodebuild.sh @@ -2,8 +2,7 @@ set -eu # Runs on the Xcode Cloud Mac after xcodebuild finishes. -# Extracts screenshots locally (macOS + xcresulttool), uploads to a public bucket, -# then optionally posts a PR comment with embeddable image URLs. +# Extracts screenshots, uploads to Imgur/S3, posts a PR comment. OUTPUT_DIR="${CI_DERIVED_DATA_PATH:-/tmp}/xcode-cloud-screenshots" SCREENSHOTS_DIR="$OUTPUT_DIR/screenshots" @@ -47,16 +46,53 @@ if [ -n "${IMGUR_CLIENT_ID:-}" ] || [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then --manifest "$MANIFEST_PATH" fi -if [ -n "${GITHUB_REPOSITORY:-}" ] && [ -n "${GITHUB_PULL_REQUEST:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then - echo "ci_post_xcodebuild: posting screenshot summary to PR #${GITHUB_PULL_REQUEST}" - COMMENT_ARGS="comment-pr --repo \"$GITHUB_REPOSITORY\" --pr-number \"$GITHUB_PULL_REQUEST\" --run-id \"$BUILD_ID\"" +# PR context: Xcode Cloud sets CI_PULL_REQUEST_* on PR builds; git-triggered +# screenshot branches can encode the PR number as screenshots/pr-42. +PR_NUMBER="${GITHUB_PULL_REQUEST:-${CI_PULL_REQUEST_NUMBER:-}}" +REPO_SLUG="${GITHUB_REPOSITORY:-${CI_PULL_REQUEST_TARGET_REPO:-}}" + +if [ -z "$PR_NUMBER" ] && [ -n "${CI_BRANCH:-}" ]; then + case "$CI_BRANCH" in + screenshots/pr-*) + PR_NUMBER="${CI_BRANCH#screenshots/pr-}" + ;; + esac +fi + +if [ -z "$REPO_SLUG" ] && [ -n "${CI_PULL_REQUEST_TARGET_REPO:-}" ]; then + REPO_SLUG="$CI_PULL_REQUEST_TARGET_REPO" +fi + +WHAT_TO_TEST="" +if [ -d "$REPO_ROOT/.git" ]; then + WHAT_TO_TEST="$(git -C "$REPO_ROOT" log -1 --format=%B 2>/dev/null | awk ' + /^What to test:/ { capture=1; next } + capture { print } + ' | sed '/^[[:space:]]*$/d' || true)" +fi + +if [ -n "$REPO_SLUG" ] && [ -n "$PR_NUMBER" ] && [ -n "${GITHUB_TOKEN:-}" ]; then + echo "ci_post_xcodebuild: posting screenshot summary to PR #${PR_NUMBER}" + COMMENT_ARGS="comment-pr --repo \"$REPO_SLUG\" --pr-number \"$PR_NUMBER\" --run-id \"$BUILD_ID\"" if [ -f "$MANIFEST_PATH" ]; then COMMENT_ARGS="$COMMENT_ARGS --manifest \"$MANIFEST_PATH\"" else COMMENT_ARGS="$COMMENT_ARGS --screenshots-dir \"$SCREENSHOTS_DIR\"" fi + if [ -n "$WHAT_TO_TEST" ]; then + WHAT_TO_TEST_FILE="$OUTPUT_DIR/what-to-test.txt" + printf '%s\n' "$WHAT_TO_TEST" > "$WHAT_TO_TEST_FILE" + COMMENT_ARGS="$COMMENT_ARGS --what-to-test-file \"$WHAT_TO_TEST_FILE\"" + fi # shellcheck disable=SC2086 "$PYTHON_BIN" "$FETCH_SCRIPT" $COMMENT_ARGS +elif [ -f "$MANIFEST_PATH" ]; then + echo "ci_post_xcodebuild: public screenshot URLs" + "$PYTHON_BIN" - "$MANIFEST_PATH" <<'PY' +import json, sys +for item in json.load(open(sys.argv[1]))["screenshots"]: + print(item["url"]) +PY fi echo "ci_post_xcodebuild: screenshots available in $SCREENSHOTS_DIR" diff --git a/scripts/README.md b/scripts/README.md index d08a4c6..8ce52ab 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,59 +1,82 @@ # Xcode Cloud screenshots -## What generates the screenshots +## Recommended: git-triggered workflow (no API polling) + +The cleanest setup avoids App Store Connect API credentials entirely: + +```text +git push screenshots/pr-42 → Xcode Cloud starts (branch/tag start condition) + → BedtimeUITests capture PNGs + → ci_post_xcodebuild uploads to Imgur + → PR comment with embedded images + "What to test" notes +``` + +### 1. Create a dedicated Xcode Cloud workflow -**`BedtimeUITests/ScreenshotTests.swift`** — XCUITest UI tests that capture PNG attachments for pull request previews. +In Xcode or App Store Connect, add a **Screenshots** workflow with: -The test launches the app with `-ui_testing` (mock sleep data, no HealthKit prompt) and saves screenshots with `XCTAttachment` lifetime `.keepAlways` so Xcode Cloud keeps them in the test result bundle. +| Setting | Value | +|---------|--------| +| Start condition | **Branch changes** → branches beginning with `screenshots/` | +| | and/or **Tag changes** → tags beginning with `screenshots/` | +| | and/or **Pull request changes** (runs on every PR automatically) | +| Test action | Scheme `Bedtime`, include `BedtimeUITests` | +| Test plan | Screenshots: **On, and keep all** | -Add `BedtimeUITests` to your Xcode Cloud workflow's **Test** action. In the test plan, set **Screenshots** to **On, and keep all** if you want images even when tests pass. +Keep this workflow separate from your main CI so screenshot runs do not block merges. -### Run locally +### 2. Xcode Cloud secrets (only two) ```bash -xcodebuild test \ - -project Bedtime/Bedtime.xcodeproj \ - -scheme Bedtime \ - -destination 'platform=iOS Simulator,name=iPhone 16' \ - -only-testing:BedtimeUITests/ScreenshotTests +IMGUR_CLIENT_ID=... # public image URLs +GITHUB_TOKEN=... # PR comments only ``` -## What happens after tests (Xcode Cloud Mac) +No `APP_STORE_CONNECT_*` keys needed for the git-trigger path. + +### 3. Trigger from your machine or a Cursor agent -`Bedtime/ci_scripts/ci_post_xcodebuild.sh`: +```bash +# On-demand screenshots for PR 42 with notes +./scripts/trigger-screenshots.sh --pr 42 --notes "Verify home + settings after sleep bank changes" -1. Extracts PNGs from `CI_RESULT_BUNDLE_PATH` via `xcresulttool` -2. Uploads to **Imgur** (`IMGUR_CLIENT_ID`) or **S3** (`SCREENSHOTS_S3_BUCKET`) -3. Optionally posts a PR comment with embedded image URLs +# Or push a tag (if the workflow uses tag start conditions) +./scripts/trigger-screenshots.sh --tag release-1.0 --notes "Release candidate UI pass" +``` -### Simplest upload setup (Imgur) +Commit/tag message format (parsed automatically): -Register at [api.imgur.com/oauth2/addclient](https://api.imgur.com/oauth2/addclient), then add one Xcode Cloud secret: +```text +screenshots: trigger -```bash -IMGUR_CLIENT_ID="your-client-id" +What to test: +- Home screen with mock sleep data +- Settings sliders and wake time picker ``` -Optional PR comment: +### 4. PR builds (zero extra trigger) + +If the workflow includes a **Pull request** start condition, every PR push already runs screenshots. Xcode Cloud sets `CI_PULL_REQUEST_NUMBER` and `CI_PULL_REQUEST_TARGET_REPO` — no manual `GITHUB_PULL_REQUEST` env var needed. + +## What generates the screenshots + +**`BedtimeUITests/ScreenshotTests.swift`** — XCUITest that saves `XCTAttachment` PNGs with `.keepAlways`. ```bash -GITHUB_TOKEN="..." -GITHUB_REPOSITORY="owner/repo" -GITHUB_PULL_REQUEST="123" +xcodebuild test \ + -project Bedtime/Bedtime.xcodeproj \ + -scheme Bedtime \ + -destination 'platform=iOS Simulator,name=iPhone 16' \ + -only-testing:BedtimeUITests/ScreenshotTests ``` -## Optional: fetch scripts (outside Xcode Cloud) +## Optional: App Store Connect API path -`scripts/fetch_xcode_cloud_screenshots.py` can trigger builds, wait for completion, and download result bundles via the App Store Connect API. Extraction still requires macOS (`xcresulttool`). +Only needed if you cannot push git refs (e.g. trigger from a system with no git write access): ```bash -export APP_STORE_CONNECT_KEY_ID=... -export APP_STORE_CONNECT_ISSUER_ID=... -export APP_STORE_CONNECT_PRIVATE_KEY='...' - python3 scripts/fetch_xcode_cloud_screenshots.py trigger-and-fetch \ - --workflow-id WORKFLOW_ID \ - --branch main + --workflow-id WORKFLOW_ID --branch main ``` -On Linux, use `--skip-extract` — let `ci_post_xcodebuild.sh` handle extraction on Apple's Mac. +This still polls until the build completes. Prefer git push when possible. diff --git a/scripts/fetch_xcode_cloud_screenshots.py b/scripts/fetch_xcode_cloud_screenshots.py index 13eb10b..1cb4a5f 100644 --- a/scripts/fetch_xcode_cloud_screenshots.py +++ b/scripts/fetch_xcode_cloud_screenshots.py @@ -137,6 +137,10 @@ def build_parser() -> argparse.ArgumentParser: default="./xcode-cloud-output/screenshots-manifest.json", help="Where to write the public URL manifest", ) + comment_parser.add_argument( + "--what-to-test-file", + help="Text file with What to test notes for the PR comment", + ) return parser @@ -178,6 +182,7 @@ def main(argv: list[str] | None = None) -> int: screenshots, build_run_id=args.run_id, uploaded=uploaded, + what_to_test=_read_what_to_test(args), ) post_pr_comment(args.repo, args.pr_number, body, token=token) print(f"Posted PR comment to {args.repo}#{args.pr_number}") diff --git a/scripts/trigger-screenshots.sh b/scripts/trigger-screenshots.sh new file mode 100755 index 0000000..2b7189d --- /dev/null +++ b/scripts/trigger-screenshots.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Trigger an Xcode Cloud screenshot workflow via git push (no App Store Connect API). + +Create a dedicated workflow in Xcode Cloud with a start condition for either: + - branches beginning with screenshots/ + - tags beginning with screenshots/ + +Usage: + scripts/trigger-screenshots.sh --pr 42 --notes "Check home and settings" + scripts/trigger-screenshots.sh --branch feature/foo --notes "Dark mode pass" + scripts/trigger-screenshots.sh --tag release-1.2.0 --notes "Release screenshots" + +Options: + --pr NUMBER Target PR number (creates branch screenshots/pr-NUMBER) + --branch NAME Push to screenshots/NAME instead + --tag NAME Create annotated tag screenshots/NAME (use tag start condition) + --notes TEXT "What to test" notes included in the commit/tag message + --dry-run Print the git commands without running them + -h, --help Show this help + +The commit/tag message uses this format: + + screenshots: trigger + + What to test: + +EOF +} + +PR="" +BRANCH="" +TAG="" +NOTES="" +DRY_RUN=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --pr) + PR="$2" + shift 2 + ;; + --branch) + BRANCH="$2" + shift 2 + ;; + --tag) + TAG="$2" + shift 2 + ;; + --notes) + NOTES="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -n "$PR" && ( -n "$BRANCH" || -n "$TAG" ) ]]; then + echo "Use only one of --pr, --branch, or --tag" >&2 + exit 1 +fi + +if [[ -z "$PR" && -z "$BRANCH" && -z "$TAG" ]]; then + echo "Provide --pr, --branch, or --tag" >&2 + usage >&2 + exit 1 +fi + +if [[ -n "$PR" ]]; then + REF="screenshots/pr-${PR}" +elif [[ -n "$BRANCH" ]]; then + REF="screenshots/${BRANCH}" +else + REF="screenshots/${TAG}" +fi + +MESSAGE=$'screenshots: trigger\n' +if [[ -n "$NOTES" ]]; then + MESSAGE+=$'\nWhat to test:\n'"${NOTES}"$'\n' +fi + +run() { + if [[ "$DRY_RUN" == true ]]; then + printf '+' + printf ' %q' "$@" + printf '\n' + else + "$@" + fi +} + +if [[ -n "$TAG" ]]; then + run git tag -fa "$REF" -m "$MESSAGE" + run git push origin "$REF" --force + echo "Pushed tag $REF" +else + CURRENT_BRANCH="$(git branch --show-current)" + run git checkout -B "$REF" + run git commit --allow-empty -m "$MESSAGE" + run git push -u origin "$REF" + if [[ -n "$CURRENT_BRANCH" ]]; then + run git checkout "$CURRENT_BRANCH" + fi + echo "Pushed branch $REF" +fi + +echo "Xcode Cloud should start the screenshots workflow shortly." diff --git a/scripts/xcode_cloud/github_pr.py b/scripts/xcode_cloud/github_pr.py index 551880d..bd335ca 100644 --- a/scripts/xcode_cloud/github_pr.py +++ b/scripts/xcode_cloud/github_pr.py @@ -15,6 +15,7 @@ def build_screenshot_comment( build_run_id: str, title: str = "Xcode Cloud screenshots", uploaded: list[UploadedScreenshot] | None = None, + what_to_test: str | None = None, ) -> str: if uploaded: lines = [ @@ -23,6 +24,8 @@ def build_screenshot_comment( f"Build run `{build_run_id}`", "", ] + if what_to_test: + lines.extend(["**What to test**", "", what_to_test.strip(), ""]) for item in uploaded: lines.append(f"**{item.name}**") lines.append("") From ffceaee6b7b1fc27c9bea04c49bfb7cd1a278bab Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 29 Jun 2026 05:12:27 +0000 Subject: [PATCH 07/10] Add sticky PR screenshot comments with before/after diffs Compare new screenshots against a per-PR baseline stored on the screenshot-baselines branch. Only changed and new screenshots appear in the PR comment table; unchanged runs get a short no-changes note. The same comment is updated in place each build via a hidden marker. Co-authored-by: Greg --- Bedtime/ci_scripts/ci_post_xcodebuild.sh | 56 +++----- scripts/README.md | 22 ++- scripts/fetch_xcode_cloud_screenshots.py | 54 ++++--- scripts/requirements.txt | 2 + scripts/xcode_cloud/baseline.py | 155 ++++++++++++++++++++ scripts/xcode_cloud/compare.py | 136 ++++++++++++++++++ scripts/xcode_cloud/github_pr.py | 175 +++++++++++++++++++++-- scripts/xcode_cloud/pr_report.py | 106 ++++++++++++++ 8 files changed, 633 insertions(+), 73 deletions(-) create mode 100644 scripts/requirements.txt create mode 100644 scripts/xcode_cloud/baseline.py create mode 100644 scripts/xcode_cloud/compare.py create mode 100644 scripts/xcode_cloud/pr_report.py diff --git a/Bedtime/ci_scripts/ci_post_xcodebuild.sh b/Bedtime/ci_scripts/ci_post_xcodebuild.sh index 3f00472..b345677 100755 --- a/Bedtime/ci_scripts/ci_post_xcodebuild.sh +++ b/Bedtime/ci_scripts/ci_post_xcodebuild.sh @@ -2,11 +2,11 @@ set -eu # Runs on the Xcode Cloud Mac after xcodebuild finishes. -# Extracts screenshots, uploads to Imgur/S3, posts a PR comment. +# Extracts screenshots, compares to the previous PR baseline, uploads diffs, +# and updates a single sticky PR comment with before/after tables. OUTPUT_DIR="${CI_DERIVED_DATA_PATH:-/tmp}/xcode-cloud-screenshots" SCREENSHOTS_DIR="$OUTPUT_DIR/screenshots" -MANIFEST_PATH="$OUTPUT_DIR/screenshots-manifest.json" ONLY_FAILURES="${XCODE_CLOUD_SCREENSHOT_ONLY_FAILURES:-false}" BUILD_ID="${CI_BUILD_ID:-unknown}" @@ -35,19 +35,6 @@ echo "ci_post_xcodebuild: exporting screenshots from $CI_RESULT_BUNDLE_PATH" # shellcheck disable=SC2086 "$PYTHON_BIN" "$FETCH_SCRIPT" $EXTRACT_ARGS -if [ -n "${IMGUR_CLIENT_ID:-}" ] || [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then - echo "ci_post_xcodebuild: uploading screenshots to public image host" - if [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then - "$PYTHON_BIN" -m pip install --quiet boto3 - fi - "$PYTHON_BIN" "$FETCH_SCRIPT" upload-screenshots \ - --screenshots-dir "$SCREENSHOTS_DIR" \ - --build-id "$BUILD_ID" \ - --manifest "$MANIFEST_PATH" -fi - -# PR context: Xcode Cloud sets CI_PULL_REQUEST_* on PR builds; git-triggered -# screenshot branches can encode the PR number as screenshots/pr-42. PR_NUMBER="${GITHUB_PULL_REQUEST:-${CI_PULL_REQUEST_NUMBER:-}}" REPO_SLUG="${GITHUB_REPOSITORY:-${CI_PULL_REQUEST_TARGET_REPO:-}}" @@ -59,10 +46,6 @@ if [ -z "$PR_NUMBER" ] && [ -n "${CI_BRANCH:-}" ]; then esac fi -if [ -z "$REPO_SLUG" ] && [ -n "${CI_PULL_REQUEST_TARGET_REPO:-}" ]; then - REPO_SLUG="$CI_PULL_REQUEST_TARGET_REPO" -fi - WHAT_TO_TEST="" if [ -d "$REPO_ROOT/.git" ]; then WHAT_TO_TEST="$(git -C "$REPO_ROOT" log -1 --format=%B 2>/dev/null | awk ' @@ -71,28 +54,27 @@ if [ -d "$REPO_ROOT/.git" ]; then ' | sed '/^[[:space:]]*$/d' || true)" fi -if [ -n "$REPO_SLUG" ] && [ -n "$PR_NUMBER" ] && [ -n "${GITHUB_TOKEN:-}" ]; then - echo "ci_post_xcodebuild: posting screenshot summary to PR #${PR_NUMBER}" - COMMENT_ARGS="comment-pr --repo \"$REPO_SLUG\" --pr-number \"$PR_NUMBER\" --run-id \"$BUILD_ID\"" - if [ -f "$MANIFEST_PATH" ]; then - COMMENT_ARGS="$COMMENT_ARGS --manifest \"$MANIFEST_PATH\"" - else - COMMENT_ARGS="$COMMENT_ARGS --screenshots-dir \"$SCREENSHOTS_DIR\"" - fi +if [ -n "$REPO_SLUG" ] && [ -n "$PR_NUMBER" ] && [ -n "${GITHUB_TOKEN:-}" ] && [ -n "${IMGUR_CLIENT_ID:-}" ]; then + echo "ci_post_xcodebuild: publishing sticky screenshot diff comment on PR #${PR_NUMBER}" + "$PYTHON_BIN" -m pip install --quiet -r "$REPO_ROOT/scripts/requirements.txt" + COMMENT_ARGS="comment-pr --repo \"$REPO_SLUG\" --pr-number \"$PR_NUMBER\" --run-id \"$BUILD_ID\" --screenshots-dir \"$SCREENSHOTS_DIR\"" if [ -n "$WHAT_TO_TEST" ]; then - WHAT_TO_TEST_FILE="$OUTPUT_DIR/what-to-test.txt" - printf '%s\n' "$WHAT_TO_TEST" > "$WHAT_TO_TEST_FILE" - COMMENT_ARGS="$COMMENT_ARGS --what-to-test-file \"$WHAT_TO_TEST_FILE\"" + WHAT_TO_TEST_FILE="$OUTPUT_DIR/what-to-test.txt" + printf '%s\n' "$WHAT_TO_TEST" > "$WHAT_TO_TEST_FILE" + COMMENT_ARGS="$COMMENT_ARGS --what-to-test-file \"$WHAT_TO_TEST_FILE\"" fi # shellcheck disable=SC2086 "$PYTHON_BIN" "$FETCH_SCRIPT" $COMMENT_ARGS -elif [ -f "$MANIFEST_PATH" ]; then - echo "ci_post_xcodebuild: public screenshot URLs" - "$PYTHON_BIN" - "$MANIFEST_PATH" <<'PY' -import json, sys -for item in json.load(open(sys.argv[1]))["screenshots"]: - print(item["url"]) -PY +elif [ -n "${IMGUR_CLIENT_ID:-}" ] || [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then + echo "ci_post_xcodebuild: uploading screenshots without PR diff comment" + if [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then + "$PYTHON_BIN" -m pip install --quiet boto3 + fi + "$PYTHON_BIN" -m pip install --quiet -r "$REPO_ROOT/scripts/requirements.txt" + "$PYTHON_BIN" "$FETCH_SCRIPT" upload-screenshots \ + --screenshots-dir "$SCREENSHOTS_DIR" \ + --build-id "$BUILD_ID" \ + --manifest "$OUTPUT_DIR/screenshots-manifest.json" fi echo "ci_post_xcodebuild: screenshots available in $SCREENSHOTS_DIR" diff --git a/scripts/README.md b/scripts/README.md index 8ce52ab..1e19e99 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -7,8 +7,8 @@ The cleanest setup avoids App Store Connect API credentials entirely: ```text git push screenshots/pr-42 → Xcode Cloud starts (branch/tag start condition) → BedtimeUITests capture PNGs - → ci_post_xcodebuild uploads to Imgur - → PR comment with embedded images + "What to test" notes + → ci_post_xcodebuild compares to previous baseline + → updates one sticky PR comment (before/after for changed screenshots only) ``` ### 1. Create a dedicated Xcode Cloud workflow @@ -25,15 +25,29 @@ In Xcode or App Store Connect, add a **Screenshots** workflow with: Keep this workflow separate from your main CI so screenshot runs do not block merges. -### 2. Xcode Cloud secrets (only two) +### 2. Xcode Cloud secrets ```bash IMGUR_CLIENT_ID=... # public image URLs -GITHUB_TOKEN=... # PR comments only +GITHUB_TOKEN=... # sticky PR comment + screenshot-baselines branch ``` +`GITHUB_TOKEN` needs permission to write issue comments and repository contents (for the per-PR baseline manifest on branch `screenshot-baselines`). + No `APP_STORE_CONNECT_*` keys needed for the git-trigger path. +### Sticky PR comment with before/after diffs + +Each PR keeps **one** screenshot comment (updated in place, not a new comment per build). + +1. Load the previous baseline from `screenshot-baselines` → `prs/{number}/manifest.json` +2. Download previous images from the stored Imgur URLs +3. Compare pixel-by-pixel against the new screenshots +4. Embed a before/after table for **changed** and **new** screenshots only +5. Save the new baseline manifest for the next run + +First run on a PR shows all screenshots as **new**. Later runs only surface diffs. + ### 3. Trigger from your machine or a Cursor agent ```bash diff --git a/scripts/fetch_xcode_cloud_screenshots.py b/scripts/fetch_xcode_cloud_screenshots.py index 1cb4a5f..4371602 100644 --- a/scripts/fetch_xcode_cloud_screenshots.py +++ b/scripts/fetch_xcode_cloud_screenshots.py @@ -14,7 +14,8 @@ from scripts.xcode_cloud.asc_auth import create_asc_token, credentials_from_env from scripts.xcode_cloud.client import XcodeCloudClient from scripts.xcode_cloud.extract import XcresultToolNotFoundError -from scripts.xcode_cloud.github_pr import build_screenshot_comment, post_pr_comment +from scripts.xcode_cloud.github_pr import build_screenshot_comment, upsert_pr_comment +from scripts.xcode_cloud.pr_report import publish_screenshot_pr_report from scripts.xcode_cloud.screenshots import ( extract_screenshots_from_local_bundle, fetch_screenshots_from_build_run, @@ -165,28 +166,39 @@ def main(argv: list[str] | None = None) -> int: if not token: parser.error("GITHUB_TOKEN is required for comment-pr") - uploaded: list[UploadedScreenshot] | None = None - screenshots: list[Path] = [] + what_to_test = _read_what_to_test(args) + + if args.screenshots_dir: + result = publish_screenshot_pr_report( + args.repo, + args.pr_number, + args.run_id, + Path(args.screenshots_dir), + token=token, + what_to_test=what_to_test, + ) + print( + f"Updated PR comment on {args.repo}#{args.pr_number}: {result['comment_url']}" + ) + return 0 + if args.manifest: manifest = json.loads(Path(args.manifest).read_text()) uploaded = [ UploadedScreenshot(name=item["name"], key=item["key"], url=item["url"]) for item in manifest.get("screenshots", []) ] - elif args.screenshots_dir: - screenshots = sorted(Path(args.screenshots_dir).rglob("*.png")) - else: - parser.error("comment-pr requires --manifest or --screenshots-dir") - - body = build_screenshot_comment( - screenshots, - build_run_id=args.run_id, - uploaded=uploaded, - what_to_test=_read_what_to_test(args), - ) - post_pr_comment(args.repo, args.pr_number, body, token=token) - print(f"Posted PR comment to {args.repo}#{args.pr_number}") - return 0 + body = build_screenshot_comment( + [], + build_run_id=args.run_id, + uploaded=uploaded, + what_to_test=what_to_test, + ) + upsert_pr_comment(args.repo, args.pr_number, body, token=token) + print(f"Updated PR comment on {args.repo}#{args.pr_number}") + return 0 + + parser.error("comment-pr requires --screenshots-dir or --manifest") if args.command == "upload-screenshots": try: @@ -280,5 +292,13 @@ def _print_screenshots(screenshots: list[Path] | tuple[Path, ...]) -> None: print(path) +def _read_what_to_test(args: argparse.Namespace) -> str | None: + path = getattr(args, "what_to_test_file", None) + if not path: + return None + text = Path(path).read_text().strip() + return text or None + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..62804ac --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,2 @@ +httpx>=0.27 +Pillow>=10.0 diff --git a/scripts/xcode_cloud/baseline.py b/scripts/xcode_cloud/baseline.py new file mode 100644 index 0000000..bff11c6 --- /dev/null +++ b/scripts/xcode_cloud/baseline.py @@ -0,0 +1,155 @@ +"""Store per-PR screenshot baselines via the GitHub Contents API.""" + +from __future__ import annotations + +import base64 +import json +from pathlib import Path +from typing import Any + +import httpx + +DEFAULT_BASELINE_BRANCH = "screenshot-baselines" +DEFAULT_BASELINE_ROOT = "prs" + + +def baseline_manifest_path(pr_number: int, *, root: str = DEFAULT_BASELINE_ROOT) -> str: + return f"{root}/{pr_number}/manifest.json" + + +def baseline_dir_for_pr(pr_number: int, cache_root: Path) -> Path: + return cache_root / f"pr-{pr_number}" + + +def _github_headers(token: str) -> dict[str, str]: + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + + +def fetch_baseline_manifest( + repo: str, + pr_number: int, + *, + token: str, + branch: str = DEFAULT_BASELINE_BRANCH, + client: httpx.Client | None = None, +) -> dict[str, Any] | None: + http = client or httpx.Client(timeout=60.0) + close_client = client is None + path = baseline_manifest_path(pr_number) + try: + response = http.get( + f"https://api.github.com/repos/{repo}/contents/{path}", + params={"ref": branch}, + headers=_github_headers(token), + ) + if response.status_code == 404: + return None + response.raise_for_status() + payload = response.json() + content = base64.b64decode(payload["content"]).decode("utf-8") + manifest = json.loads(content) + manifest["_sha"] = payload.get("sha") + return manifest + finally: + if close_client: + http.close() + + +def download_baseline_images( + manifest: dict[str, Any], + destination: Path, + *, + client: httpx.Client | None = None, +) -> dict[str, str]: + http = client or httpx.Client(timeout=60.0, follow_redirects=True) + close_client = client is None + destination.mkdir(parents=True, exist_ok=True) + urls: dict[str, str] = {} + try: + for item in manifest.get("screenshots", []): + name = item["name"] + url = item["url"] + urls[name] = url + target = destination / name + response = http.get(url) + response.raise_for_status() + target.write_bytes(response.content) + return urls + finally: + if close_client: + http.close() + + +def write_baseline_manifest( + repo: str, + pr_number: int, + manifest: dict[str, Any], + *, + token: str, + branch: str = DEFAULT_BASELINE_BRANCH, + client: httpx.Client | None = None, +) -> None: + http = client or httpx.Client(timeout=60.0) + close_client = client is None + path = baseline_manifest_path(pr_number) + body = {key: value for key, value in manifest.items() if not key.startswith("_")} + encoded = base64.b64encode(json.dumps(body, indent=2).encode("utf-8")).decode("ascii") + + payload: dict[str, Any] = { + "message": f"Update screenshot baseline for PR #{pr_number}", + "content": encoded, + "branch": branch, + } + existing_sha = manifest.get("_sha") + if existing_sha: + payload["sha"] = existing_sha + + try: + response = http.put( + f"https://api.github.com/repos/{repo}/contents/{path}", + headers=_github_headers(token), + json=payload, + ) + if response.status_code == 404 and "_sha" not in manifest: + _ensure_baseline_branch(repo, branch=branch, token=token, client=http) + response = http.put( + f"https://api.github.com/repos/{repo}/contents/{path}", + headers=_github_headers(token), + json=payload, + ) + response.raise_for_status() + finally: + if close_client: + http.close() + + +def _ensure_baseline_branch( + repo: str, + *, + branch: str, + token: str, + client: httpx.Client, +) -> None: + default_branch_response = client.get( + f"https://api.github.com/repos/{repo}", + headers=_github_headers(token), + ) + default_branch_response.raise_for_status() + default_branch = default_branch_response.json()["default_branch"] + ref_response = client.get( + f"https://api.github.com/repos/{repo}/git/ref/heads/{default_branch}", + headers=_github_headers(token), + ) + ref_response.raise_for_status() + sha = ref_response.json()["object"]["sha"] + create_response = client.post( + f"https://api.github.com/repos/{repo}/git/refs", + headers=_github_headers(token), + json={"ref": f"refs/heads/{branch}", "sha": sha}, + ) + if create_response.status_code not in {201, 422}: + create_response.raise_for_status() diff --git a/scripts/xcode_cloud/compare.py b/scripts/xcode_cloud/compare.py new file mode 100644 index 0000000..44fa6f9 --- /dev/null +++ b/scripts/xcode_cloud/compare.py @@ -0,0 +1,136 @@ +"""Compare UI test screenshots against a previous baseline.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + + +class ScreenshotChange(str, Enum): + NEW = "new" + CHANGED = "changed" + UNCHANGED = "unchanged" + REMOVED = "removed" + + +@dataclass(frozen=True) +class ScreenshotComparison: + name: str + change: ScreenshotChange + before_path: Path | None = None + after_path: Path | None = None + before_url: str | None = None + after_url: str | None = None + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _pixel_diff_ratio(left: Path, right: Path) -> float: + try: + from PIL import Image, ImageChops + except ImportError as error: + raise RuntimeError( + "Pillow is required for screenshot comparison. Install with: pip install Pillow" + ) from error + + with Image.open(left) as left_image, Image.open(right) as right_image: + if left_image.size != right_image.size: + right_image = right_image.resize(left_image.size) + + left_rgb = left_image.convert("RGB") + right_rgb = right_image.convert("RGB") + diff = ImageChops.difference(left_rgb, right_rgb) + histogram = diff.histogram() + # RGB histogram: 256 bins per channel + differing_pixels = sum( + histogram[index] + for channel in range(3) + for index in range(1, 256) + ) + total_pixels = left_rgb.size[0] * left_rgb.size[1] + return differing_pixels / (total_pixels * 3) + + +def screenshots_differ( + before: Path, + after: Path, + *, + pixel_threshold: float = 0.001, +) -> bool: + if file_sha256(before) == file_sha256(after): + return False + return _pixel_diff_ratio(before, after) > pixel_threshold + + +def compare_screenshot_sets( + current_dir: Path, + baseline_dir: Path | None, + *, + baseline_urls: dict[str, str] | None = None, + pixel_threshold: float = 0.001, +) -> list[ScreenshotComparison]: + current_paths = { + path.name: path for path in sorted(current_dir.rglob("*.png")) if path.is_file() + } + baseline_paths: dict[str, Path] = {} + if baseline_dir and baseline_dir.exists(): + baseline_paths = { + path.name: path for path in sorted(baseline_dir.rglob("*.png")) if path.is_file() + } + + baseline_urls = baseline_urls or {} + comparisons: list[ScreenshotComparison] = [] + + for name, after_path in current_paths.items(): + before_path = baseline_paths.get(name) + before_url = baseline_urls.get(name) + if before_path is None: + comparisons.append( + ScreenshotComparison( + name=name, + change=ScreenshotChange.NEW, + after_path=after_path, + after_url=None, + ) + ) + continue + + if screenshots_differ(before_path, after_path, pixel_threshold=pixel_threshold): + comparisons.append( + ScreenshotComparison( + name=name, + change=ScreenshotChange.CHANGED, + before_path=before_path, + after_path=after_path, + before_url=before_url, + after_url=None, + ) + ) + else: + comparisons.append( + ScreenshotComparison( + name=name, + change=ScreenshotChange.UNCHANGED, + before_path=before_path, + after_path=after_path, + before_url=before_url, + after_url=before_url, + ) + ) + + for name, before_path in baseline_paths.items(): + if name not in current_paths: + comparisons.append( + ScreenshotComparison( + name=name, + change=ScreenshotChange.REMOVED, + before_path=before_path, + before_url=baseline_urls.get(name), + ) + ) + + return comparisons diff --git a/scripts/xcode_cloud/github_pr.py b/scripts/xcode_cloud/github_pr.py index bd335ca..09ad83a 100644 --- a/scripts/xcode_cloud/github_pr.py +++ b/scripts/xcode_cloud/github_pr.py @@ -1,4 +1,4 @@ -"""Optional GitHub pull request comment helpers.""" +"""GitHub pull request screenshot comments.""" from __future__ import annotations @@ -6,19 +6,32 @@ import httpx +from scripts.xcode_cloud.compare import ScreenshotChange, ScreenshotComparison from scripts.xcode_cloud.upload import UploadedScreenshot +COMMENT_MARKER = "" + def build_screenshot_comment( screenshot_paths: list[Path], *, build_run_id: str, - title: str = "Xcode Cloud screenshots", + title: str = "UI screenshots", uploaded: list[UploadedScreenshot] | None = None, what_to_test: str | None = None, + comparisons: list[ScreenshotComparison] | None = None, ) -> str: + if comparisons is not None: + return _build_diff_comment( + comparisons, + build_run_id=build_run_id, + title=title, + what_to_test=what_to_test, + ) + if uploaded: lines = [ + COMMENT_MARKER, f"### {title}", "", f"Build run `{build_run_id}`", @@ -35,17 +48,18 @@ def build_screenshot_comment( if not screenshot_paths: return ( + f"{COMMENT_MARKER}\n\n" f"### {title}\n\n" f"Build run `{build_run_id}` completed, but no screenshot attachments were found." ) lines = [ + COMMENT_MARKER, f"### {title}", "", f"Build run `{build_run_id}`", "", - "Screenshots were extracted on the Xcode Cloud Mac but not uploaded to a public bucket.", - "Set `SCREENSHOTS_S3_BUCKET` (and related env vars) to embed images in PR comments.", + "Screenshots were extracted but not uploaded. Configure `IMGUR_CLIENT_ID` to embed images.", "", "| Screenshot |", "| --- |", @@ -55,7 +69,94 @@ def build_screenshot_comment( return "\n".join(lines) -def post_pr_comment( +def _build_diff_comment( + comparisons: list[ScreenshotComparison], + *, + build_run_id: str, + title: str, + what_to_test: str | None, +) -> str: + changed = [item for item in comparisons if item.change == ScreenshotChange.CHANGED] + new = [item for item in comparisons if item.change == ScreenshotChange.NEW] + unchanged = [item for item in comparisons if item.change == ScreenshotChange.UNCHANGED] + removed = [item for item in comparisons if item.change == ScreenshotChange.REMOVED] + + lines = [ + COMMENT_MARKER, + f"### {title}", + "", + f"Build run `{build_run_id}`", + "", + ( + f"{len(changed)} changed, {len(new)} new, {len(unchanged)} unchanged, " + f"{len(removed)} removed" + ), + "", + ] + + if what_to_test: + lines.extend(["**What to test**", "", what_to_test.strip(), ""]) + + if not changed and not new: + lines.append("No screenshot changes since the last run on this PR.") + return "\n".join(lines).rstrip() + + lines.extend(["", "| Screenshot | Before | After |", "| --- | --- | --- |"]) + + for item in [*changed, *new]: + before = _image_cell(item.before_url, item.name, "before") + after = _image_cell(item.after_url, item.name, "after") + label = item.name + if item.change == ScreenshotChange.NEW: + label = f"{item.name} (new)" + lines.append(f"| {label} | {before} | {after} |") + + if removed: + lines.extend(["", "**Removed screenshots**", ""]) + for item in removed: + lines.append(f"- `{item.name}`") + + return "\n".join(lines).rstrip() + + +def _image_cell(url: str | None, name: str, role: str) -> str: + if not url: + return "—" + return f"![{name} {role}]({url})" + + +def find_pr_comment_id( + repo: str, + pr_number: int, + *, + token: str, + client: httpx.Client | None = None, +) -> int | None: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + page = 1 + while True: + response = http.get( + f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments", + params={"per_page": 100, "page": page}, + headers=_github_headers(token), + ) + response.raise_for_status() + comments = response.json() + if not comments: + return None + for comment in comments: + if COMMENT_MARKER in comment.get("body", ""): + return comment["id"] + page += 1 + finally: + if close_client: + http.close() + return None + + +def upsert_pr_comment( repo: str, pr_number: int, body: str, @@ -63,21 +164,65 @@ def post_pr_comment( token: str, client: httpx.Client | None = None, ) -> dict: - """Post a markdown comment on a GitHub pull request.""" http = client or httpx.Client(timeout=30.0) close_client = client is None try: - response = http.post( - f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments", - headers={ - "Authorization": f"Bearer {token}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - }, - json={"body": body}, - ) + comment_id = find_pr_comment_id(repo, pr_number, token=token, client=http) + headers = _github_headers(token) + if comment_id is not None: + response = http.patch( + f"https://api.github.com/repos/{repo}/issues/comments/{comment_id}", + headers=headers, + json={"body": body}, + ) + else: + response = http.post( + f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments", + headers=headers, + json={"body": body}, + ) response.raise_for_status() return response.json() finally: if close_client: http.close() + + +def post_pr_comment( + repo: str, + pr_number: int, + body: str, + *, + token: str, + client: httpx.Client | None = None, +) -> dict: + return upsert_pr_comment(repo, pr_number, body, token=token, client=client) + + +def _github_headers(token: str) -> dict[str, str]: + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + + +def attach_upload_urls( + comparisons: list[ScreenshotComparison], + uploads: list[UploadedScreenshot], +) -> list[ScreenshotComparison]: + upload_by_name = {item.name: item.url for item in uploads} + updated: list[ScreenshotComparison] = [] + for item in comparisons: + after_url = upload_by_name.get(item.name, item.after_url) + updated.append( + ScreenshotComparison( + name=item.name, + change=item.change, + before_path=item.before_path, + after_path=item.after_path, + before_url=item.before_url, + after_url=after_url, + ) + ) + return updated diff --git a/scripts/xcode_cloud/pr_report.py b/scripts/xcode_cloud/pr_report.py new file mode 100644 index 0000000..374c43b --- /dev/null +++ b/scripts/xcode_cloud/pr_report.py @@ -0,0 +1,106 @@ +"""Publish sticky PR screenshot reports with before/after diffs.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import httpx + +from scripts.xcode_cloud.baseline import ( + baseline_dir_for_pr, + download_baseline_images, + fetch_baseline_manifest, + write_baseline_manifest, +) +from scripts.xcode_cloud.compare import compare_screenshot_sets, file_sha256 +from scripts.xcode_cloud.github_pr import ( + attach_upload_urls, + build_screenshot_comment, + upsert_pr_comment, +) +from scripts.xcode_cloud.upload import UploadBackend, upload_screenshots + + +def publish_screenshot_pr_report( + repo: str, + pr_number: int, + build_run_id: str, + screenshots_dir: Path, + *, + token: str, + what_to_test: str | None = None, + upload_backend: UploadBackend = "auto", + cache_root: Path | None = None, +) -> dict[str, Any]: + cache_root = cache_root or screenshots_dir.parent / "baseline-cache" + baseline_dir = baseline_dir_for_pr(pr_number, cache_root) + + with httpx.Client(timeout=60.0, follow_redirects=True) as client: + previous_manifest = fetch_baseline_manifest( + repo, + pr_number, + token=token, + client=client, + ) + baseline_urls: dict[str, str] = {} + if previous_manifest: + baseline_urls = download_baseline_images( + previous_manifest, + baseline_dir, + client=client, + ) + + comparisons = compare_screenshot_sets( + screenshots_dir, + baseline_dir if baseline_urls else None, + baseline_urls=baseline_urls, + ) + + uploads = upload_screenshots( + screenshots_dir, + build_id=build_run_id, + backend=upload_backend, + http_client=client, + ) + comparisons = attach_upload_urls(comparisons, uploads) + + body = build_screenshot_comment( + [], + build_run_id=build_run_id, + comparisons=comparisons, + what_to_test=what_to_test, + ) + comment = upsert_pr_comment(repo, pr_number, body, token=token, client=client) + + upload_by_name = {item.name: item for item in uploads} + manifest_screenshots = [] + for path in sorted(screenshots_dir.rglob("*.png")): + if not path.is_file(): + continue + uploaded = upload_by_name[path.name] + manifest_screenshots.append( + { + "name": path.name, + "key": uploaded.key, + "url": uploaded.url, + "sha256": file_sha256(path), + } + ) + + new_manifest: dict[str, Any] = { + "build_id": build_run_id, + "pr_number": pr_number, + "screenshots": manifest_screenshots, + } + if previous_manifest and previous_manifest.get("_sha"): + new_manifest["_sha"] = previous_manifest["_sha"] + + write_baseline_manifest(repo, pr_number, new_manifest, token=token, client=client) + + return { + "comment_id": comment["id"], + "comment_url": comment["html_url"], + "upload_count": len(uploads), + "comparison_count": len(comparisons), + } From 54ec05336075e511f81ab165c7aaba553754c574 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 29 Jun 2026 05:26:00 +0000 Subject: [PATCH 08/10] Use commit comments for Imgur screenshot baselines Replace the screenshot-baselines JSON branch with sticky GitHub commit comments. Main builds embed Imgur URLs in the comment metadata; PR builds compare against CI_PULL_REQUEST_TARGET_COMMIT and post a before/after table on the PR head commit. Co-authored-by: Greg --- Bedtime/ci_scripts/ci_post_xcodebuild.sh | 29 ++-- scripts/README.md | 53 +++--- scripts/fetch_xcode_cloud_screenshots.py | 46 +++-- scripts/xcode_cloud/baseline.py | 155 ----------------- scripts/xcode_cloud/commit_report.py | 89 ++++++++++ scripts/xcode_cloud/github_comments.py | 209 +++++++++++++++++++++++ scripts/xcode_cloud/github_commit.py | 124 ++++++++++++++ scripts/xcode_cloud/github_pr.py | 160 +---------------- scripts/xcode_cloud/pr_report.py | 106 ------------ 9 files changed, 508 insertions(+), 463 deletions(-) delete mode 100644 scripts/xcode_cloud/baseline.py create mode 100644 scripts/xcode_cloud/commit_report.py create mode 100644 scripts/xcode_cloud/github_comments.py create mode 100644 scripts/xcode_cloud/github_commit.py delete mode 100644 scripts/xcode_cloud/pr_report.py diff --git a/Bedtime/ci_scripts/ci_post_xcodebuild.sh b/Bedtime/ci_scripts/ci_post_xcodebuild.sh index b345677..d24f1a1 100755 --- a/Bedtime/ci_scripts/ci_post_xcodebuild.sh +++ b/Bedtime/ci_scripts/ci_post_xcodebuild.sh @@ -2,8 +2,8 @@ set -eu # Runs on the Xcode Cloud Mac after xcodebuild finishes. -# Extracts screenshots, compares to the previous PR baseline, uploads diffs, -# and updates a single sticky PR comment with before/after tables. +# Extracts screenshots, compares to the main-branch baseline commit comment, +# uploads to Imgur, and updates a sticky comment on this commit. OUTPUT_DIR="${CI_DERIVED_DATA_PATH:-/tmp}/xcode-cloud-screenshots" SCREENSHOTS_DIR="$OUTPUT_DIR/screenshots" @@ -35,16 +35,9 @@ echo "ci_post_xcodebuild: exporting screenshots from $CI_RESULT_BUNDLE_PATH" # shellcheck disable=SC2086 "$PYTHON_BIN" "$FETCH_SCRIPT" $EXTRACT_ARGS -PR_NUMBER="${GITHUB_PULL_REQUEST:-${CI_PULL_REQUEST_NUMBER:-}}" -REPO_SLUG="${GITHUB_REPOSITORY:-${CI_PULL_REQUEST_TARGET_REPO:-}}" - -if [ -z "$PR_NUMBER" ] && [ -n "${CI_BRANCH:-}" ]; then - case "$CI_BRANCH" in - screenshots/pr-*) - PR_NUMBER="${CI_BRANCH#screenshots/pr-}" - ;; - esac -fi +COMMIT_SHA="${CI_COMMIT:-}" +REPO_SLUG="${GITHUB_REPOSITORY:-${CI_PULL_REQUEST_TARGET_REPO:-${CI_PULL_REQUEST_SOURCE_REPO:-}}}" +BASELINE_COMMIT="${CI_PULL_REQUEST_TARGET_COMMIT:-}" WHAT_TO_TEST="" if [ -d "$REPO_ROOT/.git" ]; then @@ -54,10 +47,14 @@ if [ -d "$REPO_ROOT/.git" ]; then ' | sed '/^[[:space:]]*$/d' || true)" fi -if [ -n "$REPO_SLUG" ] && [ -n "$PR_NUMBER" ] && [ -n "${GITHUB_TOKEN:-}" ] && [ -n "${IMGUR_CLIENT_ID:-}" ]; then - echo "ci_post_xcodebuild: publishing sticky screenshot diff comment on PR #${PR_NUMBER}" +if [ -n "$REPO_SLUG" ] && [ -n "$COMMIT_SHA" ] && [ -n "${GITHUB_TOKEN:-}" ] && [ -n "${IMGUR_CLIENT_ID:-}" ]; then + echo "ci_post_xcodebuild: publishing sticky screenshot comment on commit ${COMMIT_SHA}" "$PYTHON_BIN" -m pip install --quiet -r "$REPO_ROOT/scripts/requirements.txt" - COMMENT_ARGS="comment-pr --repo \"$REPO_SLUG\" --pr-number \"$PR_NUMBER\" --run-id \"$BUILD_ID\" --screenshots-dir \"$SCREENSHOTS_DIR\"" + COMMENT_ARGS="comment-commit --repo \"$REPO_SLUG\" --commit-sha \"$COMMIT_SHA\" --run-id \"$BUILD_ID\" --screenshots-dir \"$SCREENSHOTS_DIR\"" + if [ -n "$BASELINE_COMMIT" ] && [ "$BASELINE_COMMIT" != "$COMMIT_SHA" ]; then + COMMENT_ARGS="$COMMENT_ARGS --baseline-commit \"$BASELINE_COMMIT\"" + echo "ci_post_xcodebuild: comparing against baseline commit ${BASELINE_COMMIT}" + fi if [ -n "$WHAT_TO_TEST" ]; then WHAT_TO_TEST_FILE="$OUTPUT_DIR/what-to-test.txt" printf '%s\n' "$WHAT_TO_TEST" > "$WHAT_TO_TEST_FILE" @@ -66,7 +63,7 @@ if [ -n "$REPO_SLUG" ] && [ -n "$PR_NUMBER" ] && [ -n "${GITHUB_TOKEN:-}" ] && [ # shellcheck disable=SC2086 "$PYTHON_BIN" "$FETCH_SCRIPT" $COMMENT_ARGS elif [ -n "${IMGUR_CLIENT_ID:-}" ] || [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then - echo "ci_post_xcodebuild: uploading screenshots without PR diff comment" + echo "ci_post_xcodebuild: uploading screenshots without commit comment" if [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then "$PYTHON_BIN" -m pip install --quiet boto3 fi diff --git a/scripts/README.md b/scripts/README.md index 1e19e99..9a45790 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -5,10 +5,9 @@ The cleanest setup avoids App Store Connect API credentials entirely: ```text -git push screenshots/pr-42 → Xcode Cloud starts (branch/tag start condition) - → BedtimeUITests capture PNGs - → ci_post_xcodebuild compares to previous baseline - → updates one sticky PR comment (before/after for changed screenshots only) +main push → screenshots on commit, Imgur URLs in commit comment +PR push → compare vs main baseline commit comment, sticky comment on PR commit +git push screenshots/pr-42 → on-demand screenshot run (branch start condition) ``` ### 1. Create a dedicated Xcode Cloud workflow @@ -17,9 +16,9 @@ In Xcode or App Store Connect, add a **Screenshots** workflow with: | Setting | Value | |---------|--------| -| Start condition | **Branch changes** → branches beginning with `screenshots/` | -| | and/or **Tag changes** → tags beginning with `screenshots/` | -| | and/or **Pull request changes** (runs on every PR automatically) | +| Start condition | **Branch changes** → `main` (keeps baseline fresh) | +| | and/or branches beginning with `screenshots/` | +| | and/or **Pull request changes** | | Test action | Scheme `Bedtime`, include `BedtimeUITests` | | Test plan | Screenshots: **On, and keep all** | @@ -29,24 +28,31 @@ Keep this workflow separate from your main CI so screenshot runs do not block me ```bash IMGUR_CLIENT_ID=... # public image URLs -GITHUB_TOKEN=... # sticky PR comment + screenshot-baselines branch +GITHUB_TOKEN=... # sticky commit comments ``` -`GITHUB_TOKEN` needs permission to write issue comments and repository contents (for the per-PR baseline manifest on branch `screenshot-baselines`). +`GITHUB_TOKEN` needs permission to create and update **commit comments** (`repo` scope or fine-grained equivalent). -No `APP_STORE_CONNECT_*` keys needed for the git-trigger path. +No `APP_STORE_CONNECT_*` keys needed for the git-trigger path. No S3, no separate baseline branch. -### Sticky PR comment with before/after diffs +### Sticky commit comments with before/after diffs -Each PR keeps **one** screenshot comment (updated in place, not a new comment per build). +Screenshots are reported as a **commit comment** on the built commit (`github.com/{owner}/{repo}/commit/{sha}`). No PR required. -1. Load the previous baseline from `screenshot-baselines` → `prs/{number}/manifest.json` -2. Download previous images from the stored Imgur URLs -3. Compare pixel-by-pixel against the new screenshots -4. Embed a before/after table for **changed** and **new** screenshots only -5. Save the new baseline manifest for the next run +**Main builds** (`CI_COMMIT` on `main`): -First run on a PR shows all screenshots as **new**. Later runs only surface diffs. +1. Upload screenshots to Imgur +2. Post/update one sticky comment on that commit +3. Embed Imgur URLs in a hidden metadata block inside the comment (for baseline lookup) + +**PR builds**: + +1. Read Imgur URLs from the commit comment on `CI_PULL_REQUEST_TARGET_COMMIT` (main HEAD) +2. Download those images as the "before" baseline +3. Compare pixel-by-pixel against new screenshots +4. Post/update sticky comment on `CI_COMMIT` (PR head) with before/after table for changed/new only + +First PR run after a fresh main baseline shows all screenshots as **new**. Unchanged PR re-runs show "no changes compared to `{main_sha}`". ### 3. Trigger from your machine or a Cursor agent @@ -68,9 +74,16 @@ What to test: - Settings sliders and wake time picker ``` -### 4. PR builds (zero extra trigger) +### 4. Manual CLI -If the workflow includes a **Pull request** start condition, every PR push already runs screenshots. Xcode Cloud sets `CI_PULL_REQUEST_NUMBER` and `CI_PULL_REQUEST_TARGET_REPO` — no manual `GITHUB_PULL_REQUEST` env var needed. +```bash +python3 scripts/fetch_xcode_cloud_screenshots.py comment-commit \ + --repo owner/repo \ + --commit-sha abc123def456 \ + --baseline-commit mainsha789 \ + --run-id build-1 \ + --screenshots-dir ./screenshots +``` ## What generates the screenshots diff --git a/scripts/fetch_xcode_cloud_screenshots.py b/scripts/fetch_xcode_cloud_screenshots.py index 4371602..ff26b2c 100644 --- a/scripts/fetch_xcode_cloud_screenshots.py +++ b/scripts/fetch_xcode_cloud_screenshots.py @@ -14,8 +14,8 @@ from scripts.xcode_cloud.asc_auth import create_asc_token, credentials_from_env from scripts.xcode_cloud.client import XcodeCloudClient from scripts.xcode_cloud.extract import XcresultToolNotFoundError -from scripts.xcode_cloud.github_pr import build_screenshot_comment, upsert_pr_comment -from scripts.xcode_cloud.pr_report import publish_screenshot_pr_report +from scripts.xcode_cloud.commit_report import publish_screenshot_commit_report +from scripts.xcode_cloud.github_comments import build_screenshot_comment from scripts.xcode_cloud.screenshots import ( extract_screenshots_from_local_bundle, fetch_screenshots_from_build_run, @@ -106,11 +106,15 @@ def build_parser() -> argparse.ArgumentParser: local_parser.add_argument("--only-failures", action="store_true") comment_parser = subparsers.add_parser( - "comment-pr", - help="Post a screenshot summary comment to a GitHub pull request.", + "comment-commit", + help="Post or update a sticky screenshot comment on a Git commit.", ) comment_parser.add_argument("--repo", required=True, help="owner/repo") - comment_parser.add_argument("--pr-number", type=int, required=True) + comment_parser.add_argument("--commit-sha", required=True, help="Commit to comment on") + comment_parser.add_argument( + "--baseline-commit", + help="Baseline commit to compare against (e.g. main HEAD for PR builds)", + ) comment_parser.add_argument("--run-id", required=True) comment_parser.add_argument( "--screenshots-dir", @@ -118,7 +122,7 @@ def build_parser() -> argparse.ArgumentParser: ) comment_parser.add_argument( "--manifest", - help="JSON manifest written by upload-screenshots (preferred for embedded images)", + help="JSON manifest written by upload-screenshots (legacy, no diff)", ) upload_parser = subparsers.add_parser( @@ -140,7 +144,7 @@ def build_parser() -> argparse.ArgumentParser: ) comment_parser.add_argument( "--what-to-test-file", - help="Text file with What to test notes for the PR comment", + help="Text file with What to test notes for the commit comment", ) return parser @@ -158,27 +162,29 @@ def main(argv: list[str] | None = None) -> int: _print_screenshots(screenshots) return 0 - if args.command == "comment-pr": + if args.command == "comment-commit": import json import os token = os.environ.get("GITHUB_TOKEN") if not token: - parser.error("GITHUB_TOKEN is required for comment-pr") + parser.error("GITHUB_TOKEN is required for comment-commit") what_to_test = _read_what_to_test(args) if args.screenshots_dir: - result = publish_screenshot_pr_report( + result = publish_screenshot_commit_report( args.repo, - args.pr_number, + args.commit_sha, args.run_id, Path(args.screenshots_dir), token=token, + baseline_commit_sha=args.baseline_commit, what_to_test=what_to_test, ) print( - f"Updated PR comment on {args.repo}#{args.pr_number}: {result['comment_url']}" + f"Updated commit comment on {args.repo}@{args.commit_sha[:7]}: " + f"{result['comment_url']}" ) return 0 @@ -188,17 +194,27 @@ def main(argv: list[str] | None = None) -> int: UploadedScreenshot(name=item["name"], key=item["key"], url=item["url"]) for item in manifest.get("screenshots", []) ] + screenshot_urls = {item.name: item.url for item in uploaded} body = build_screenshot_comment( [], build_run_id=args.run_id, + commit_sha=args.commit_sha, uploaded=uploaded, what_to_test=what_to_test, + screenshot_urls=screenshot_urls, + ) + from scripts.xcode_cloud.github_commit import upsert_commit_comment + + upsert_commit_comment( + args.repo, + args.commit_sha, + body, + token=token, ) - upsert_pr_comment(args.repo, args.pr_number, body, token=token) - print(f"Updated PR comment on {args.repo}#{args.pr_number}") + print(f"Updated commit comment on {args.repo}@{args.commit_sha[:7]}") return 0 - parser.error("comment-pr requires --screenshots-dir or --manifest") + parser.error("comment-commit requires --screenshots-dir or --manifest") if args.command == "upload-screenshots": try: diff --git a/scripts/xcode_cloud/baseline.py b/scripts/xcode_cloud/baseline.py deleted file mode 100644 index bff11c6..0000000 --- a/scripts/xcode_cloud/baseline.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Store per-PR screenshot baselines via the GitHub Contents API.""" - -from __future__ import annotations - -import base64 -import json -from pathlib import Path -from typing import Any - -import httpx - -DEFAULT_BASELINE_BRANCH = "screenshot-baselines" -DEFAULT_BASELINE_ROOT = "prs" - - -def baseline_manifest_path(pr_number: int, *, root: str = DEFAULT_BASELINE_ROOT) -> str: - return f"{root}/{pr_number}/manifest.json" - - -def baseline_dir_for_pr(pr_number: int, cache_root: Path) -> Path: - return cache_root / f"pr-{pr_number}" - - -def _github_headers(token: str) -> dict[str, str]: - return { - "Authorization": f"Bearer {token}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } - - -def fetch_baseline_manifest( - repo: str, - pr_number: int, - *, - token: str, - branch: str = DEFAULT_BASELINE_BRANCH, - client: httpx.Client | None = None, -) -> dict[str, Any] | None: - http = client or httpx.Client(timeout=60.0) - close_client = client is None - path = baseline_manifest_path(pr_number) - try: - response = http.get( - f"https://api.github.com/repos/{repo}/contents/{path}", - params={"ref": branch}, - headers=_github_headers(token), - ) - if response.status_code == 404: - return None - response.raise_for_status() - payload = response.json() - content = base64.b64decode(payload["content"]).decode("utf-8") - manifest = json.loads(content) - manifest["_sha"] = payload.get("sha") - return manifest - finally: - if close_client: - http.close() - - -def download_baseline_images( - manifest: dict[str, Any], - destination: Path, - *, - client: httpx.Client | None = None, -) -> dict[str, str]: - http = client or httpx.Client(timeout=60.0, follow_redirects=True) - close_client = client is None - destination.mkdir(parents=True, exist_ok=True) - urls: dict[str, str] = {} - try: - for item in manifest.get("screenshots", []): - name = item["name"] - url = item["url"] - urls[name] = url - target = destination / name - response = http.get(url) - response.raise_for_status() - target.write_bytes(response.content) - return urls - finally: - if close_client: - http.close() - - -def write_baseline_manifest( - repo: str, - pr_number: int, - manifest: dict[str, Any], - *, - token: str, - branch: str = DEFAULT_BASELINE_BRANCH, - client: httpx.Client | None = None, -) -> None: - http = client or httpx.Client(timeout=60.0) - close_client = client is None - path = baseline_manifest_path(pr_number) - body = {key: value for key, value in manifest.items() if not key.startswith("_")} - encoded = base64.b64encode(json.dumps(body, indent=2).encode("utf-8")).decode("ascii") - - payload: dict[str, Any] = { - "message": f"Update screenshot baseline for PR #{pr_number}", - "content": encoded, - "branch": branch, - } - existing_sha = manifest.get("_sha") - if existing_sha: - payload["sha"] = existing_sha - - try: - response = http.put( - f"https://api.github.com/repos/{repo}/contents/{path}", - headers=_github_headers(token), - json=payload, - ) - if response.status_code == 404 and "_sha" not in manifest: - _ensure_baseline_branch(repo, branch=branch, token=token, client=http) - response = http.put( - f"https://api.github.com/repos/{repo}/contents/{path}", - headers=_github_headers(token), - json=payload, - ) - response.raise_for_status() - finally: - if close_client: - http.close() - - -def _ensure_baseline_branch( - repo: str, - *, - branch: str, - token: str, - client: httpx.Client, -) -> None: - default_branch_response = client.get( - f"https://api.github.com/repos/{repo}", - headers=_github_headers(token), - ) - default_branch_response.raise_for_status() - default_branch = default_branch_response.json()["default_branch"] - ref_response = client.get( - f"https://api.github.com/repos/{repo}/git/ref/heads/{default_branch}", - headers=_github_headers(token), - ) - ref_response.raise_for_status() - sha = ref_response.json()["object"]["sha"] - create_response = client.post( - f"https://api.github.com/repos/{repo}/git/refs", - headers=_github_headers(token), - json={"ref": f"refs/heads/{branch}", "sha": sha}, - ) - if create_response.status_code not in {201, 422}: - create_response.raise_for_status() diff --git a/scripts/xcode_cloud/commit_report.py b/scripts/xcode_cloud/commit_report.py new file mode 100644 index 0000000..66dfd52 --- /dev/null +++ b/scripts/xcode_cloud/commit_report.py @@ -0,0 +1,89 @@ +"""Publish sticky commit screenshot reports with before/after diffs.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import httpx + +from scripts.xcode_cloud.compare import compare_screenshot_sets +from scripts.xcode_cloud.github_comments import attach_upload_urls, build_screenshot_comment +from scripts.xcode_cloud.github_commit import ( + download_baseline_images, + fetch_screenshot_urls_from_commit, + upsert_commit_comment, +) +from scripts.xcode_cloud.upload import UploadBackend, upload_screenshots + + +def publish_screenshot_commit_report( + repo: str, + commit_sha: str, + build_run_id: str, + screenshots_dir: Path, + *, + token: str, + baseline_commit_sha: str | None = None, + what_to_test: str | None = None, + upload_backend: UploadBackend = "auto", + cache_root: Path | None = None, +) -> dict[str, Any]: + cache_root = cache_root or screenshots_dir.parent / "baseline-cache" + baseline_dir = cache_root / f"baseline-{baseline_commit_sha or 'none'}" + + with httpx.Client(timeout=60.0, follow_redirects=True) as client: + baseline_urls: dict[str, str] = {} + if baseline_commit_sha and baseline_commit_sha != commit_sha: + baseline_urls = fetch_screenshot_urls_from_commit( + repo, + baseline_commit_sha, + token=token, + client=client, + ) + if baseline_urls: + download_baseline_images( + baseline_urls, + baseline_dir, + client=client, + ) + + comparisons = compare_screenshot_sets( + screenshots_dir, + baseline_dir if baseline_urls else None, + baseline_urls=baseline_urls, + ) + + uploads = upload_screenshots( + screenshots_dir, + build_id=build_run_id, + backend=upload_backend, + http_client=client, + ) + comparisons = attach_upload_urls(comparisons, uploads) + screenshot_urls = {item.name: item.url for item in uploads} + + body = build_screenshot_comment( + [], + build_run_id=build_run_id, + commit_sha=commit_sha, + baseline_commit_sha=baseline_commit_sha, + comparisons=comparisons, + what_to_test=what_to_test, + screenshot_urls=screenshot_urls, + ) + comment = upsert_commit_comment( + repo, + commit_sha, + body, + token=token, + client=client, + ) + + return { + "comment_id": comment["id"], + "comment_url": comment["html_url"], + "upload_count": len(uploads), + "comparison_count": len(comparisons), + "baseline_commit_sha": baseline_commit_sha, + } diff --git a/scripts/xcode_cloud/github_comments.py b/scripts/xcode_cloud/github_comments.py new file mode 100644 index 0000000..3705074 --- /dev/null +++ b/scripts/xcode_cloud/github_comments.py @@ -0,0 +1,209 @@ +"""Shared GitHub screenshot comment formatting and URL metadata.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +from scripts.xcode_cloud.compare import ScreenshotChange, ScreenshotComparison +from scripts.xcode_cloud.upload import UploadedScreenshot + +COMMENT_MARKER = "" +URLS_MARKER_START = "" + + +def github_headers(token: str) -> dict[str, str]: + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + + +def short_sha(commit_sha: str) -> str: + return commit_sha[:7] + + +def embed_screenshot_urls(body: str, urls: dict[str, str]) -> str: + stripped = strip_screenshot_urls_block(body).rstrip() + payload = json.dumps(urls, separators=(",", ":"), sort_keys=True) + block = f"{URLS_MARKER_START}\n{payload}\n{URLS_MARKER_END}" + return f"{stripped}\n\n{block}" + + +def strip_screenshot_urls_block(body: str) -> str: + pattern = re.compile( + rf"\n*{re.escape(URLS_MARKER_START)}.*?{re.escape(URLS_MARKER_END)}", + flags=re.DOTALL, + ) + return pattern.sub("", body) + + +def parse_screenshot_urls(body: str) -> dict[str, str]: + match = re.search( + rf"{re.escape(URLS_MARKER_START)}\s*(\{{.*?\}})\s*{re.escape(URLS_MARKER_END)}", + body, + flags=re.DOTALL, + ) + if not match: + return {} + try: + payload = json.loads(match.group(1)) + except json.JSONDecodeError: + return {} + if not isinstance(payload, dict): + return {} + return {str(name): str(url) for name, url in payload.items()} + + +def build_screenshot_comment( + screenshot_paths: list[Path], + *, + build_run_id: str, + commit_sha: str | None = None, + baseline_commit_sha: str | None = None, + title: str = "UI screenshots", + uploaded: list[UploadedScreenshot] | None = None, + what_to_test: str | None = None, + comparisons: list[ScreenshotComparison] | None = None, + screenshot_urls: dict[str, str] | None = None, +) -> str: + if comparisons is not None: + body = _build_diff_comment( + comparisons, + build_run_id=build_run_id, + commit_sha=commit_sha, + baseline_commit_sha=baseline_commit_sha, + title=title, + what_to_test=what_to_test, + ) + elif uploaded: + lines = [ + COMMENT_MARKER, + f"### {title}", + "", + f"Commit `{short_sha(commit_sha)}`" if commit_sha else f"Build run `{build_run_id}`", + "", + ] + if what_to_test: + lines.extend(["**What to test**", "", what_to_test.strip(), ""]) + for item in uploaded: + lines.append(f"**{item.name}**") + lines.append("") + lines.append(f"![{item.name}]({item.url})") + lines.append("") + body = "\n".join(lines).rstrip() + elif not screenshot_paths: + body = ( + f"{COMMENT_MARKER}\n\n" + f"### {title}\n\n" + f"Build run `{build_run_id}` completed, but no screenshot attachments were found." + ) + else: + lines = [ + COMMENT_MARKER, + f"### {title}", + "", + f"Build run `{build_run_id}`", + "", + "Screenshots were extracted but not uploaded. Configure `IMGUR_CLIENT_ID` to embed images.", + "", + "| Screenshot |", + "| --- |", + ] + for path in screenshot_paths: + lines.append(f"| `{path.name}` |") + body = "\n".join(lines) + + if screenshot_urls: + body = embed_screenshot_urls(body, screenshot_urls) + return body + + +def _build_diff_comment( + comparisons: list[ScreenshotComparison], + *, + build_run_id: str, + commit_sha: str | None, + baseline_commit_sha: str | None, + title: str, + what_to_test: str | None, +) -> str: + changed = [item for item in comparisons if item.change == ScreenshotChange.CHANGED] + new = [item for item in comparisons if item.change == ScreenshotChange.NEW] + unchanged = [item for item in comparisons if item.change == ScreenshotChange.UNCHANGED] + removed = [item for item in comparisons if item.change == ScreenshotChange.REMOVED] + + header = f"Commit `{short_sha(commit_sha)}`" if commit_sha else f"Build run `{build_run_id}`" + lines = [ + COMMENT_MARKER, + f"### {title}", + "", + header, + "", + ] + if baseline_commit_sha: + lines.append(f"Compared to `{short_sha(baseline_commit_sha)}`") + lines.append("") + lines.append( + f"{len(changed)} changed, {len(new)} new, {len(unchanged)} unchanged, {len(removed)} removed" + ) + lines.append("") + + if what_to_test: + lines.extend(["**What to test**", "", what_to_test.strip(), ""]) + + if not changed and not new: + if baseline_commit_sha: + lines.append( + f"No screenshot changes compared to `{short_sha(baseline_commit_sha)}`." + ) + else: + lines.append("No screenshot changes detected.") + return "\n".join(lines).rstrip() + + lines.extend(["", "| Screenshot | Before | After |", "| --- | --- | --- |"]) + + for item in [*changed, *new]: + before = _image_cell(item.before_url, item.name, "before") + after = _image_cell(item.after_url, item.name, "after") + label = item.name + if item.change == ScreenshotChange.NEW: + label = f"{item.name} (new)" + lines.append(f"| {label} | {before} | {after} |") + + if removed: + lines.extend(["", "**Removed screenshots**", ""]) + for item in removed: + lines.append(f"- `{item.name}`") + + return "\n".join(lines).rstrip() + + +def _image_cell(url: str | None, name: str, role: str) -> str: + if not url: + return "—" + return f"![{name} {role}]({url})" + + +def attach_upload_urls( + comparisons: list[ScreenshotComparison], + uploads: list[UploadedScreenshot], +) -> list[ScreenshotComparison]: + upload_by_name = {item.name: item.url for item in uploads} + updated: list[ScreenshotComparison] = [] + for item in comparisons: + after_url = upload_by_name.get(item.name, item.after_url) + updated.append( + ScreenshotComparison( + name=item.name, + change=item.change, + before_path=item.before_path, + after_path=item.after_path, + before_url=item.before_url, + after_url=after_url, + ) + ) + return updated diff --git a/scripts/xcode_cloud/github_commit.py b/scripts/xcode_cloud/github_commit.py new file mode 100644 index 0000000..3f7bd79 --- /dev/null +++ b/scripts/xcode_cloud/github_commit.py @@ -0,0 +1,124 @@ +"""GitHub commit comments for screenshot reports.""" + +from __future__ import annotations + +from pathlib import Path + +import httpx + +from scripts.xcode_cloud.github_comments import COMMENT_MARKER, github_headers, parse_screenshot_urls + + +def find_commit_comment_id( + repo: str, + commit_sha: str, + *, + token: str, + client: httpx.Client | None = None, +) -> int | None: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + response = http.get( + f"https://api.github.com/repos/{repo}/commits/{commit_sha}/comments", + headers=github_headers(token), + ) + if response.status_code == 404: + return None + response.raise_for_status() + for comment in response.json(): + if COMMENT_MARKER in comment.get("body", ""): + return comment["id"] + return None + finally: + if close_client: + http.close() + + +def upsert_commit_comment( + repo: str, + commit_sha: str, + body: str, + *, + token: str, + client: httpx.Client | None = None, +) -> dict: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + comment_id = find_commit_comment_id( + repo, + commit_sha, + token=token, + client=http, + ) + headers = github_headers(token) + if comment_id is not None: + response = http.patch( + f"https://api.github.com/repos/{repo}/comments/{comment_id}", + headers=headers, + json={"body": body}, + ) + else: + response = http.post( + f"https://api.github.com/repos/{repo}/commits/{commit_sha}/comments", + headers=headers, + json={"body": body}, + ) + response.raise_for_status() + return response.json() + finally: + if close_client: + http.close() + + +def fetch_screenshot_urls_from_commit( + repo: str, + commit_sha: str, + *, + token: str, + client: httpx.Client | None = None, +) -> dict[str, str]: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + response = http.get( + f"https://api.github.com/repos/{repo}/commits/{commit_sha}/comments", + headers=github_headers(token), + ) + if response.status_code == 404: + return {} + response.raise_for_status() + for comment in response.json(): + if COMMENT_MARKER not in comment.get("body", ""): + continue + urls = parse_screenshot_urls(comment["body"]) + if urls: + return urls + return {} + finally: + if close_client: + http.close() + + +def download_baseline_images( + urls: dict[str, str], + destination: Path, + *, + client: httpx.Client | None = None, +) -> dict[str, str]: + http = client or httpx.Client(timeout=60.0, follow_redirects=True) + close_client = client is None + destination.mkdir(parents=True, exist_ok=True) + downloaded: dict[str, str] = {} + try: + for name, url in urls.items(): + target = destination / name + response = http.get(url) + response.raise_for_status() + target.write_bytes(response.content) + downloaded[name] = url + return downloaded + finally: + if close_client: + http.close() diff --git a/scripts/xcode_cloud/github_pr.py b/scripts/xcode_cloud/github_pr.py index 09ad83a..4fc58a7 100644 --- a/scripts/xcode_cloud/github_pr.py +++ b/scripts/xcode_cloud/github_pr.py @@ -1,128 +1,15 @@ -"""GitHub pull request screenshot comments.""" +"""GitHub pull request screenshot comments (legacy manifest path).""" from __future__ import annotations -from pathlib import Path - import httpx -from scripts.xcode_cloud.compare import ScreenshotChange, ScreenshotComparison -from scripts.xcode_cloud.upload import UploadedScreenshot - -COMMENT_MARKER = "" - - -def build_screenshot_comment( - screenshot_paths: list[Path], - *, - build_run_id: str, - title: str = "UI screenshots", - uploaded: list[UploadedScreenshot] | None = None, - what_to_test: str | None = None, - comparisons: list[ScreenshotComparison] | None = None, -) -> str: - if comparisons is not None: - return _build_diff_comment( - comparisons, - build_run_id=build_run_id, - title=title, - what_to_test=what_to_test, - ) - - if uploaded: - lines = [ - COMMENT_MARKER, - f"### {title}", - "", - f"Build run `{build_run_id}`", - "", - ] - if what_to_test: - lines.extend(["**What to test**", "", what_to_test.strip(), ""]) - for item in uploaded: - lines.append(f"**{item.name}**") - lines.append("") - lines.append(f"![{item.name}]({item.url})") - lines.append("") - return "\n".join(lines).rstrip() - - if not screenshot_paths: - return ( - f"{COMMENT_MARKER}\n\n" - f"### {title}\n\n" - f"Build run `{build_run_id}` completed, but no screenshot attachments were found." - ) - - lines = [ - COMMENT_MARKER, - f"### {title}", - "", - f"Build run `{build_run_id}`", - "", - "Screenshots were extracted but not uploaded. Configure `IMGUR_CLIENT_ID` to embed images.", - "", - "| Screenshot |", - "| --- |", - ] - for path in screenshot_paths: - lines.append(f"| `{path.name}` |") - return "\n".join(lines) - - -def _build_diff_comment( - comparisons: list[ScreenshotComparison], - *, - build_run_id: str, - title: str, - what_to_test: str | None, -) -> str: - changed = [item for item in comparisons if item.change == ScreenshotChange.CHANGED] - new = [item for item in comparisons if item.change == ScreenshotChange.NEW] - unchanged = [item for item in comparisons if item.change == ScreenshotChange.UNCHANGED] - removed = [item for item in comparisons if item.change == ScreenshotChange.REMOVED] - - lines = [ - COMMENT_MARKER, - f"### {title}", - "", - f"Build run `{build_run_id}`", - "", - ( - f"{len(changed)} changed, {len(new)} new, {len(unchanged)} unchanged, " - f"{len(removed)} removed" - ), - "", - ] - - if what_to_test: - lines.extend(["**What to test**", "", what_to_test.strip(), ""]) - - if not changed and not new: - lines.append("No screenshot changes since the last run on this PR.") - return "\n".join(lines).rstrip() - - lines.extend(["", "| Screenshot | Before | After |", "| --- | --- | --- |"]) - - for item in [*changed, *new]: - before = _image_cell(item.before_url, item.name, "before") - after = _image_cell(item.after_url, item.name, "after") - label = item.name - if item.change == ScreenshotChange.NEW: - label = f"{item.name} (new)" - lines.append(f"| {label} | {before} | {after} |") - - if removed: - lines.extend(["", "**Removed screenshots**", ""]) - for item in removed: - lines.append(f"- `{item.name}`") - - return "\n".join(lines).rstrip() - - -def _image_cell(url: str | None, name: str, role: str) -> str: - if not url: - return "—" - return f"![{name} {role}]({url})" +from scripts.xcode_cloud.github_comments import ( + COMMENT_MARKER, + attach_upload_urls, + build_screenshot_comment, + github_headers, +) def find_pr_comment_id( @@ -140,7 +27,7 @@ def find_pr_comment_id( response = http.get( f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments", params={"per_page": 100, "page": page}, - headers=_github_headers(token), + headers=github_headers(token), ) response.raise_for_status() comments = response.json() @@ -168,7 +55,7 @@ def upsert_pr_comment( close_client = client is None try: comment_id = find_pr_comment_id(repo, pr_number, token=token, client=http) - headers = _github_headers(token) + headers = github_headers(token) if comment_id is not None: response = http.patch( f"https://api.github.com/repos/{repo}/issues/comments/{comment_id}", @@ -197,32 +84,3 @@ def post_pr_comment( client: httpx.Client | None = None, ) -> dict: return upsert_pr_comment(repo, pr_number, body, token=token, client=client) - - -def _github_headers(token: str) -> dict[str, str]: - return { - "Authorization": f"Bearer {token}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } - - -def attach_upload_urls( - comparisons: list[ScreenshotComparison], - uploads: list[UploadedScreenshot], -) -> list[ScreenshotComparison]: - upload_by_name = {item.name: item.url for item in uploads} - updated: list[ScreenshotComparison] = [] - for item in comparisons: - after_url = upload_by_name.get(item.name, item.after_url) - updated.append( - ScreenshotComparison( - name=item.name, - change=item.change, - before_path=item.before_path, - after_path=item.after_path, - before_url=item.before_url, - after_url=after_url, - ) - ) - return updated diff --git a/scripts/xcode_cloud/pr_report.py b/scripts/xcode_cloud/pr_report.py deleted file mode 100644 index 374c43b..0000000 --- a/scripts/xcode_cloud/pr_report.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Publish sticky PR screenshot reports with before/after diffs.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import httpx - -from scripts.xcode_cloud.baseline import ( - baseline_dir_for_pr, - download_baseline_images, - fetch_baseline_manifest, - write_baseline_manifest, -) -from scripts.xcode_cloud.compare import compare_screenshot_sets, file_sha256 -from scripts.xcode_cloud.github_pr import ( - attach_upload_urls, - build_screenshot_comment, - upsert_pr_comment, -) -from scripts.xcode_cloud.upload import UploadBackend, upload_screenshots - - -def publish_screenshot_pr_report( - repo: str, - pr_number: int, - build_run_id: str, - screenshots_dir: Path, - *, - token: str, - what_to_test: str | None = None, - upload_backend: UploadBackend = "auto", - cache_root: Path | None = None, -) -> dict[str, Any]: - cache_root = cache_root or screenshots_dir.parent / "baseline-cache" - baseline_dir = baseline_dir_for_pr(pr_number, cache_root) - - with httpx.Client(timeout=60.0, follow_redirects=True) as client: - previous_manifest = fetch_baseline_manifest( - repo, - pr_number, - token=token, - client=client, - ) - baseline_urls: dict[str, str] = {} - if previous_manifest: - baseline_urls = download_baseline_images( - previous_manifest, - baseline_dir, - client=client, - ) - - comparisons = compare_screenshot_sets( - screenshots_dir, - baseline_dir if baseline_urls else None, - baseline_urls=baseline_urls, - ) - - uploads = upload_screenshots( - screenshots_dir, - build_id=build_run_id, - backend=upload_backend, - http_client=client, - ) - comparisons = attach_upload_urls(comparisons, uploads) - - body = build_screenshot_comment( - [], - build_run_id=build_run_id, - comparisons=comparisons, - what_to_test=what_to_test, - ) - comment = upsert_pr_comment(repo, pr_number, body, token=token, client=client) - - upload_by_name = {item.name: item for item in uploads} - manifest_screenshots = [] - for path in sorted(screenshots_dir.rglob("*.png")): - if not path.is_file(): - continue - uploaded = upload_by_name[path.name] - manifest_screenshots.append( - { - "name": path.name, - "key": uploaded.key, - "url": uploaded.url, - "sha256": file_sha256(path), - } - ) - - new_manifest: dict[str, Any] = { - "build_id": build_run_id, - "pr_number": pr_number, - "screenshots": manifest_screenshots, - } - if previous_manifest and previous_manifest.get("_sha"): - new_manifest["_sha"] = previous_manifest["_sha"] - - write_baseline_manifest(repo, pr_number, new_manifest, token=token, client=client) - - return { - "comment_id": comment["id"], - "comment_url": comment["html_url"], - "upload_count": len(uploads), - "comparison_count": len(comparisons), - } From f31e738e2b47d6e32868b5f377ffc69d3a598fcd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 30 Jun 2026 20:19:07 +0000 Subject: [PATCH 09/10] Add agent feedback loop via commit comment build reports Post terminal success/failed/no_screenshots commit comments from ci_post even when xcodebuild fails. Add hidden build-status metadata, comment-build and wait-for-commit-report CLI commands, and poll instructions in trigger-screenshots.sh. Co-authored-by: Greg --- Bedtime/ci_scripts/ci_post_xcodebuild.sh | 142 ++++++++---- scripts/README.md | 69 ++++-- scripts/fetch_xcode_cloud_screenshots.py | 200 ++++++++++++++--- scripts/trigger-screenshots.sh | 16 ++ .../{commit_report.py => build_report.py} | 72 +++++- scripts/xcode_cloud/extract.py | 86 +++++++- scripts/xcode_cloud/github_comments.py | 208 ++++++++++++++++-- scripts/xcode_cloud/github_commit.py | 85 ++++++- 8 files changed, 773 insertions(+), 105 deletions(-) rename scripts/xcode_cloud/{commit_report.py => build_report.py} (59%) diff --git a/Bedtime/ci_scripts/ci_post_xcodebuild.sh b/Bedtime/ci_scripts/ci_post_xcodebuild.sh index d24f1a1..9ceafc8 100755 --- a/Bedtime/ci_scripts/ci_post_xcodebuild.sh +++ b/Bedtime/ci_scripts/ci_post_xcodebuild.sh @@ -2,39 +2,19 @@ set -eu # Runs on the Xcode Cloud Mac after xcodebuild finishes. -# Extracts screenshots, compares to the main-branch baseline commit comment, -# uploads to Imgur, and updates a sticky comment on this commit. +# Always posts a terminal sticky commit comment for agents to poll: +# success (screenshot diff), failed (build/test errors), or no_screenshots. OUTPUT_DIR="${CI_DERIVED_DATA_PATH:-/tmp}/xcode-cloud-screenshots" SCREENSHOTS_DIR="$OUTPUT_DIR/screenshots" ONLY_FAILURES="${XCODE_CLOUD_SCREENSHOT_ONLY_FAILURES:-false}" BUILD_ID="${CI_BUILD_ID:-unknown}" - -if [ -z "${CI_RESULT_BUNDLE_PATH:-}" ]; then - echo "ci_post_xcodebuild: CI_RESULT_BUNDLE_PATH is not set; skipping screenshot export." - exit 0 -fi - -if [ ! -e "$CI_RESULT_BUNDLE_PATH" ]; then - echo "ci_post_xcodebuild: result bundle not found at $CI_RESULT_BUNDLE_PATH" - exit 0 -fi - -mkdir -p "$OUTPUT_DIR" +XCODEBUILD_EXIT="${CI_XCODEBUILD_EXIT_CODE:-0}" REPO_ROOT="${CI_PRIMARY_REPOSITORY_PATH:-$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)}" PYTHON_BIN="${PYTHON_BIN:-python3}" FETCH_SCRIPT="$REPO_ROOT/scripts/fetch_xcode_cloud_screenshots.py" -EXTRACT_ARGS="extract-local --bundle-path \"$CI_RESULT_BUNDLE_PATH\" --output-dir \"$OUTPUT_DIR\"" -if [ "$ONLY_FAILURES" = "true" ]; then - EXTRACT_ARGS="$EXTRACT_ARGS --only-failures" -fi - -echo "ci_post_xcodebuild: exporting screenshots from $CI_RESULT_BUNDLE_PATH" -# shellcheck disable=SC2086 -"$PYTHON_BIN" "$FETCH_SCRIPT" $EXTRACT_ARGS - COMMIT_SHA="${CI_COMMIT:-}" REPO_SLUG="${GITHUB_REPOSITORY:-${CI_PULL_REQUEST_TARGET_REPO:-${CI_PULL_REQUEST_SOURCE_REPO:-}}}" BASELINE_COMMIT="${CI_PULL_REQUEST_TARGET_COMMIT:-}" @@ -47,31 +27,105 @@ if [ -d "$REPO_ROOT/.git" ]; then ' | sed '/^[[:space:]]*$/d' || true)" fi -if [ -n "$REPO_SLUG" ] && [ -n "$COMMIT_SHA" ] && [ -n "${GITHUB_TOKEN:-}" ] && [ -n "${IMGUR_CLIENT_ID:-}" ]; then - echo "ci_post_xcodebuild: publishing sticky screenshot comment on commit ${COMMIT_SHA}" - "$PYTHON_BIN" -m pip install --quiet -r "$REPO_ROOT/scripts/requirements.txt" - COMMENT_ARGS="comment-commit --repo \"$REPO_SLUG\" --commit-sha \"$COMMIT_SHA\" --run-id \"$BUILD_ID\" --screenshots-dir \"$SCREENSHOTS_DIR\"" - if [ -n "$BASELINE_COMMIT" ] && [ "$BASELINE_COMMIT" != "$COMMIT_SHA" ]; then - COMMENT_ARGS="$COMMENT_ARGS --baseline-commit \"$BASELINE_COMMIT\"" - echo "ci_post_xcodebuild: comparing against baseline commit ${BASELINE_COMMIT}" +mkdir -p "$OUTPUT_DIR" +ERRORS_FILE="$OUTPUT_DIR/errors.txt" +: > "$ERRORS_FILE" + +COMMENT_ARGS="--repo \"$REPO_SLUG\" --commit-sha \"$COMMIT_SHA\" --run-id \"$BUILD_ID\"" +if [ -n "$BASELINE_COMMIT" ] && [ "$BASELINE_COMMIT" != "$COMMIT_SHA" ]; then + COMMENT_ARGS="$COMMENT_ARGS --baseline-commit \"$BASELINE_COMMIT\"" +fi +if [ -n "$WHAT_TO_TEST" ]; then + WHAT_TO_TEST_FILE="$OUTPUT_DIR/what-to-test.txt" + printf '%s\n' "$WHAT_TO_TEST" > "$WHAT_TO_TEST_FILE" + COMMENT_ARGS="$COMMENT_ARGS --what-to-test-file \"$WHAT_TO_TEST_FILE\"" +fi + +append_failures_from_bundle() { + if [ -e "${CI_RESULT_BUNDLE_PATH:-}" ]; then + "$PYTHON_BIN" "$FETCH_SCRIPT" extract-failures \ + --bundle-path "$CI_RESULT_BUNDLE_PATH" >> "$ERRORS_FILE" 2>/dev/null || true fi - if [ -n "$WHAT_TO_TEST" ]; then - WHAT_TO_TEST_FILE="$OUTPUT_DIR/what-to-test.txt" - printf '%s\n' "$WHAT_TO_TEST" > "$WHAT_TO_TEST_FILE" - COMMENT_ARGS="$COMMENT_ARGS --what-to-test-file \"$WHAT_TO_TEST_FILE\"" +} + +publish_report() { + if [ -z "$REPO_SLUG" ] || [ -z "$COMMIT_SHA" ] || [ -z "${GITHUB_TOKEN:-}" ]; then + echo "ci_post_xcodebuild: skipping commit report (need repo, commit, GITHUB_TOKEN)" + return 0 fi + + echo "ci_post_xcodebuild: publishing build report on commit ${COMMIT_SHA}" + "$PYTHON_BIN" -m pip install --quiet -r "$REPO_ROOT/scripts/requirements.txt" # shellcheck disable=SC2086 - "$PYTHON_BIN" "$FETCH_SCRIPT" $COMMENT_ARGS -elif [ -n "${IMGUR_CLIENT_ID:-}" ] || [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then - echo "ci_post_xcodebuild: uploading screenshots without commit comment" - if [ -n "${SCREENSHOTS_S3_BUCKET:-}" ]; then - "$PYTHON_BIN" -m pip install --quiet boto3 + "$PYTHON_BIN" "$FETCH_SCRIPT" comment-build $COMMENT_ARGS +} + +if [ "$XCODEBUILD_EXIT" != "0" ]; then + echo "ci_post_xcodebuild: xcodebuild failed with exit code ${XCODEBUILD_EXIT}" + printf '%s\n' "xcodebuild failed with exit code ${XCODEBUILD_EXIT}" >> "$ERRORS_FILE" + if [ -n "${CI_XCODEBUILD_ACTION:-}" ]; then + printf '%s\n' "Action: ${CI_XCODEBUILD_ACTION}" >> "$ERRORS_FILE" fi - "$PYTHON_BIN" -m pip install --quiet -r "$REPO_ROOT/scripts/requirements.txt" - "$PYTHON_BIN" "$FETCH_SCRIPT" upload-screenshots \ - --screenshots-dir "$SCREENSHOTS_DIR" \ - --build-id "$BUILD_ID" \ - --manifest "$OUTPUT_DIR/screenshots-manifest.json" + if [ -n "${CI_BUILD_URL:-}" ]; then + printf '%s\n' "Build logs: ${CI_BUILD_URL}" >> "$ERRORS_FILE" + fi + append_failures_from_bundle + COMMENT_ARGS="$COMMENT_ARGS --status failed --exit-code \"$XCODEBUILD_EXIT\" --errors-file \"$ERRORS_FILE\"" + publish_report || true + exit 0 +fi + +if [ -z "${CI_RESULT_BUNDLE_PATH:-}" ] || [ ! -e "$CI_RESULT_BUNDLE_PATH" ]; then + echo "ci_post_xcodebuild: no test result bundle available" + printf '%s\n' "No test result bundle was produced." >> "$ERRORS_FILE" + COMMENT_ARGS="$COMMENT_ARGS --status no_screenshots --errors-file \"$ERRORS_FILE\"" + publish_report || true + exit 0 +fi + +EXTRACT_ARGS="extract-local --bundle-path \"$CI_RESULT_BUNDLE_PATH\" --output-dir \"$OUTPUT_DIR\"" +if [ "$ONLY_FAILURES" = "true" ]; then + EXTRACT_ARGS="$EXTRACT_ARGS --only-failures" fi +echo "ci_post_xcodebuild: exporting screenshots from $CI_RESULT_BUNDLE_PATH" +set +e +# shellcheck disable=SC2086 +"$PYTHON_BIN" "$FETCH_SCRIPT" $EXTRACT_ARGS +EXTRACT_EXIT=$? +set -eu + +if [ "$EXTRACT_EXIT" -ne 0 ]; then + printf '%s\n' "Screenshot extraction failed with exit code ${EXTRACT_EXIT}" >> "$ERRORS_FILE" + append_failures_from_bundle + COMMENT_ARGS="$COMMENT_ARGS --status failed --exit-code \"$EXTRACT_EXIT\" --errors-file \"$ERRORS_FILE\"" + publish_report || true + exit 0 +fi + +SCREENSHOT_COUNT=0 +if [ -d "$SCREENSHOTS_DIR" ]; then + SCREENSHOT_COUNT="$(find "$SCREENSHOTS_DIR" -name '*.png' -type f | wc -l | tr -d ' ')" +fi + +if [ "$SCREENSHOT_COUNT" = "0" ]; then + echo "ci_post_xcodebuild: no screenshot PNGs extracted" + printf '%s\n' "No screenshot PNG attachments were found in the test result bundle." >> "$ERRORS_FILE" + append_failures_from_bundle + COMMENT_ARGS="$COMMENT_ARGS --status no_screenshots --errors-file \"$ERRORS_FILE\"" + publish_report || true + exit 0 +fi + +if [ -z "${IMGUR_CLIENT_ID:-}" ]; then + echo "ci_post_xcodebuild: IMGUR_CLIENT_ID is not set" + printf '%s\n' "IMGUR_CLIENT_ID is not configured for screenshot upload." >> "$ERRORS_FILE" + COMMENT_ARGS="$COMMENT_ARGS --status failed --errors-file \"$ERRORS_FILE\"" + publish_report || true + exit 0 +fi + +COMMENT_ARGS="$COMMENT_ARGS --status success --screenshots-dir \"$SCREENSHOTS_DIR\"" +publish_report || true + echo "ci_post_xcodebuild: screenshots available in $SCREENSHOTS_DIR" diff --git a/scripts/README.md b/scripts/README.md index 9a45790..bb4940f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -10,6 +10,24 @@ PR push → compare vs main baseline commit comment, sticky git push screenshots/pr-42 → on-demand screenshot run (branch start condition) ``` +### Agent feedback loop + +Cloud agents can trigger, poll, and iterate without watching Xcode Cloud directly: + +```text +1. ./scripts/trigger-screenshots.sh --pr 42 --notes "Check home screen" +2. Poll until a terminal commit comment appears: + python3 scripts/fetch_xcode_cloud_screenshots.py wait-for-commit-report \ + --repo owner/repo --commit-sha +3. Read status from the hidden JSON block: + success → screenshot diff table + Imgur URLs + failed → xcodebuild/test errors to fix + no_screenshots → tests ran but no PNG attachments +4. Fix code, push again, repeat +``` + +`ci_post_xcodebuild.sh` **always** posts a terminal commit comment when `GITHUB_TOKEN` and `CI_COMMIT` are set — including build failures — so agents do not poll forever. + ### 1. Create a dedicated Xcode Cloud workflow In Xcode or App Store Connect, add a **Screenshots** workflow with: @@ -27,32 +45,42 @@ Keep this workflow separate from your main CI so screenshot runs do not block me ### 2. Xcode Cloud secrets ```bash -IMGUR_CLIENT_ID=... # public image URLs -GITHUB_TOKEN=... # sticky commit comments +IMGUR_CLIENT_ID=... # public image URLs (success path only) +GITHUB_TOKEN=... # sticky commit comments (success + failure) ``` -`GITHUB_TOKEN` needs permission to create and update **commit comments** (`repo` scope or fine-grained equivalent). +`GITHUB_TOKEN` needs permission to create and update **commit comments**. No `APP_STORE_CONNECT_*` keys needed for the git-trigger path. No S3, no separate baseline branch. -### Sticky commit comments with before/after diffs +### Sticky commit comments -Screenshots are reported as a **commit comment** on the built commit (`github.com/{owner}/{repo}/commit/{sha}`). No PR required. +Screenshots and build outcomes are reported as a **commit comment** on the built commit (`github.com/{owner}/{repo}/commit/{sha}`). No PR required. **Main builds** (`CI_COMMIT` on `main`): 1. Upload screenshots to Imgur 2. Post/update one sticky comment on that commit -3. Embed Imgur URLs in a hidden metadata block inside the comment (for baseline lookup) +3. Embed Imgur URLs in hidden metadata for baseline lookup **PR builds**: 1. Read Imgur URLs from the commit comment on `CI_PULL_REQUEST_TARGET_COMMIT` (main HEAD) -2. Download those images as the "before" baseline -3. Compare pixel-by-pixel against new screenshots -4. Post/update sticky comment on `CI_COMMIT` (PR head) with before/after table for changed/new only +2. Compare pixel-by-pixel against new screenshots +3. Post/update sticky comment on `CI_COMMIT` with before/after table for changed/new only + +**Failed builds** (`CI_XCODEBUILD_EXIT_CODE != 0`): -First PR run after a fresh main baseline shows all screenshots as **new**. Unchanged PR re-runs show "no changes compared to `{main_sha}`". +1. Skip screenshot upload +2. Post/update sticky comment with exit code, test failures, and build log URL + +Hidden status block (for agents): + +```html + +``` ### 3. Trigger from your machine or a Cursor agent @@ -60,8 +88,9 @@ First PR run after a fresh main baseline shows all screenshots as **new**. Uncha # On-demand screenshots for PR 42 with notes ./scripts/trigger-screenshots.sh --pr 42 --notes "Verify home + settings after sleep bank changes" -# Or push a tag (if the workflow uses tag start conditions) -./scripts/trigger-screenshots.sh --tag release-1.0 --notes "Release candidate UI pass" +# Poll for outcome (commit SHA printed by trigger script) +python3 scripts/fetch_xcode_cloud_screenshots.py wait-for-commit-report \ + --repo owner/repo --commit-sha abc123def456 --output-json /tmp/report.json ``` Commit/tag message format (parsed automatically): @@ -77,12 +106,22 @@ What to test: ### 4. Manual CLI ```bash -python3 scripts/fetch_xcode_cloud_screenshots.py comment-commit \ +# Publish outcomes directly +python3 scripts/fetch_xcode_cloud_screenshots.py comment-build \ --repo owner/repo \ - --commit-sha abc123def456 \ + --commit-sha abc123 \ --baseline-commit mainsha789 \ --run-id build-1 \ + --status success \ --screenshots-dir ./screenshots + +python3 scripts/fetch_xcode_cloud_screenshots.py comment-build \ + --repo owner/repo \ + --commit-sha abc123 \ + --run-id build-1 \ + --status failed \ + --exit-code 65 \ + --errors-file ./errors.txt ``` ## What generates the screenshots @@ -106,4 +145,4 @@ python3 scripts/fetch_xcode_cloud_screenshots.py trigger-and-fetch \ --workflow-id WORKFLOW_ID --branch main ``` -This still polls until the build completes. Prefer git push when possible. +This still polls App Store Connect until the build completes. Prefer git push + `wait-for-commit-report` when possible. diff --git a/scripts/fetch_xcode_cloud_screenshots.py b/scripts/fetch_xcode_cloud_screenshots.py index ff26b2c..54def14 100644 --- a/scripts/fetch_xcode_cloud_screenshots.py +++ b/scripts/fetch_xcode_cloud_screenshots.py @@ -14,8 +14,13 @@ from scripts.xcode_cloud.asc_auth import create_asc_token, credentials_from_env from scripts.xcode_cloud.client import XcodeCloudClient from scripts.xcode_cloud.extract import XcresultToolNotFoundError -from scripts.xcode_cloud.commit_report import publish_screenshot_commit_report -from scripts.xcode_cloud.github_comments import build_screenshot_comment +from scripts.xcode_cloud.build_report import ( + publish_build_failure_report, + publish_no_screenshots_report, + publish_screenshot_commit_report, +) +from scripts.xcode_cloud.github_comments import BuildStatus, build_screenshot_comment +from scripts.xcode_cloud.github_commit import upsert_commit_comment, wait_for_commit_report from scripts.xcode_cloud.screenshots import ( extract_screenshots_from_local_bundle, fetch_screenshots_from_build_run, @@ -106,24 +111,75 @@ def build_parser() -> argparse.ArgumentParser: local_parser.add_argument("--only-failures", action="store_true") comment_parser = subparsers.add_parser( - "comment-commit", - help="Post or update a sticky screenshot comment on a Git commit.", + "comment-build", + help="Post or update a sticky build report comment on a Git commit.", ) comment_parser.add_argument("--repo", required=True, help="owner/repo") comment_parser.add_argument("--commit-sha", required=True, help="Commit to comment on") + comment_parser.add_argument("--run-id", required=True) + comment_parser.add_argument( + "--status", + choices=[status.value for status in BuildStatus], + help="Terminal build status to publish", + ) comment_parser.add_argument( "--baseline-commit", - help="Baseline commit to compare against (e.g. main HEAD for PR builds)", + help="Baseline commit to compare against (success path only)", ) - comment_parser.add_argument("--run-id", required=True) comment_parser.add_argument( "--screenshots-dir", - help="Directory containing extracted .png files (used when no manifest is provided)", + help="Directory containing extracted .png files (success path)", + ) + comment_parser.add_argument("--exit-code", type=int, help="xcodebuild exit code (failed path)") + comment_parser.add_argument( + "--errors-file", + help="Text file with one error per line (failed/no-screenshots paths)", ) comment_parser.add_argument( + "--log-file", + help="Log excerpt to include in a failed build comment", + ) + comment_parser.add_argument( + "--what-to-test-file", + help="Text file with What to test notes for the commit comment", + ) + + comment_commit_parser = subparsers.add_parser( + "comment-commit", + help="Alias for comment-build --status success.", + ) + comment_commit_parser.add_argument("--repo", required=True, help="owner/repo") + comment_commit_parser.add_argument("--commit-sha", required=True, help="Commit to comment on") + comment_commit_parser.add_argument( + "--baseline-commit", + help="Baseline commit to compare against (e.g. main HEAD for PR builds)", + ) + comment_commit_parser.add_argument("--run-id", required=True) + comment_commit_parser.add_argument( + "--screenshots-dir", + help="Directory containing extracted .png files (used when no manifest is provided)", + ) + comment_commit_parser.add_argument( "--manifest", help="JSON manifest written by upload-screenshots (legacy, no diff)", ) + comment_commit_parser.add_argument( + "--what-to-test-file", + help="Text file with What to test notes for the commit comment", + ) + + wait_report_parser = subparsers.add_parser( + "wait-for-commit-report", + help="Poll until a terminal build report comment appears on a commit.", + ) + wait_report_parser.add_argument("--repo", required=True, help="owner/repo") + wait_report_parser.add_argument("--commit-sha", required=True) + wait_report_parser.add_argument("--timeout-seconds", type=int, default=3600) + wait_report_parser.add_argument("--poll-interval-seconds", type=int, default=30) + wait_report_parser.add_argument( + "--output-json", + help="Write the parsed build-status payload to this JSON file", + ) upload_parser = subparsers.add_parser( "upload-screenshots", @@ -142,13 +198,79 @@ def build_parser() -> argparse.ArgumentParser: default="./xcode-cloud-output/screenshots-manifest.json", help="Where to write the public URL manifest", ) - comment_parser.add_argument( - "--what-to-test-file", - help="Text file with What to test notes for the commit comment", + + failures_parser = subparsers.add_parser( + "extract-failures", + help="Print failing test summaries from a local .xcresult bundle.", ) + failures_parser.add_argument("--bundle-path", required=True) return parser +def _read_error_lines(path: str | None) -> list[str]: + if not path: + return [] + return [line.strip() for line in Path(path).read_text().splitlines() if line.strip()] + + +def _publish_comment_build(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: + import os + + token = os.environ.get("GITHUB_TOKEN") + if not token: + parser.error("GITHUB_TOKEN is required") + + what_to_test = _read_what_to_test(args) + errors = _read_error_lines(getattr(args, "errors_file", None)) + log_excerpt = None + log_file = getattr(args, "log_file", None) + if log_file: + log_excerpt = Path(log_file).read_text().strip() or None + + status = getattr(args, "status", None) + if status is None and args.screenshots_dir: + status = BuildStatus.SUCCESS.value + if status is None: + parser.error("comment-build requires --status or --screenshots-dir") + + if status == BuildStatus.SUCCESS.value: + if not args.screenshots_dir: + parser.error("comment-build --status success requires --screenshots-dir") + result = publish_screenshot_commit_report( + args.repo, + args.commit_sha, + args.run_id, + Path(args.screenshots_dir), + token=token, + baseline_commit_sha=getattr(args, "baseline_commit", None), + what_to_test=what_to_test, + ) + elif status == BuildStatus.FAILED.value: + result = publish_build_failure_report( + args.repo, + args.commit_sha, + args.run_id, + token=token, + exit_code=getattr(args, "exit_code", None), + errors=errors or None, + log_excerpt=log_excerpt, + ) + else: + result = publish_no_screenshots_report( + args.repo, + args.commit_sha, + args.run_id, + token=token, + errors=errors or None, + ) + + print( + f"Published {result['status']} report on {args.repo}@{args.commit_sha[:7]}: " + f"{result['comment_url']}" + ) + return 0 + + def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) @@ -162,6 +284,16 @@ def main(argv: list[str] | None = None) -> int: _print_screenshots(screenshots) return 0 + if args.command == "extract-failures": + from scripts.xcode_cloud.extract import extract_test_failure_summaries + + for line in extract_test_failure_summaries(Path(args.bundle_path)): + print(line) + return 0 + + if args.command == "comment-build": + return _publish_comment_build(args, parser) + if args.command == "comment-commit": import json import os @@ -173,20 +305,8 @@ def main(argv: list[str] | None = None) -> int: what_to_test = _read_what_to_test(args) if args.screenshots_dir: - result = publish_screenshot_commit_report( - args.repo, - args.commit_sha, - args.run_id, - Path(args.screenshots_dir), - token=token, - baseline_commit_sha=args.baseline_commit, - what_to_test=what_to_test, - ) - print( - f"Updated commit comment on {args.repo}@{args.commit_sha[:7]}: " - f"{result['comment_url']}" - ) - return 0 + args.status = BuildStatus.SUCCESS.value + return _publish_comment_build(args, parser) if args.manifest: manifest = json.loads(Path(args.manifest).read_text()) @@ -203,8 +323,6 @@ def main(argv: list[str] | None = None) -> int: what_to_test=what_to_test, screenshot_urls=screenshot_urls, ) - from scripts.xcode_cloud.github_commit import upsert_commit_comment - upsert_commit_comment( args.repo, args.commit_sha, @@ -216,6 +334,36 @@ def main(argv: list[str] | None = None) -> int: parser.error("comment-commit requires --screenshots-dir or --manifest") + if args.command == "wait-for-commit-report": + import json + import os + + token = os.environ.get("GITHUB_TOKEN") + if not token: + parser.error("GITHUB_TOKEN is required for wait-for-commit-report") + + try: + report = wait_for_commit_report( + args.repo, + args.commit_sha, + token=token, + timeout_seconds=args.timeout_seconds, + poll_interval_seconds=args.poll_interval_seconds, + ) + except TimeoutError as error: + print(str(error), file=sys.stderr) + return 1 + + print(f"Build report status: {report.status.value}") + print(f"Comment: {report.comment_url}") + if args.output_json: + Path(args.output_json).write_text(json.dumps(report.payload, indent=2)) + print(f"Wrote status payload to {args.output_json}") + return 0 if report.status == BuildStatus.SUCCESS else 1 + + if args.command == "comment-pr": + parser.error("comment-pr was removed; use comment-build or wait-for-commit-report") + if args.command == "upload-screenshots": try: uploads = upload_screenshots( diff --git a/scripts/trigger-screenshots.sh b/scripts/trigger-screenshots.sh index 2b7189d..9b4b953 100755 --- a/scripts/trigger-screenshots.sh +++ b/scripts/trigger-screenshots.sh @@ -108,11 +108,13 @@ run() { if [[ -n "$TAG" ]]; then run git tag -fa "$REF" -m "$MESSAGE" run git push origin "$REF" --force + POLL_SHA="$(git rev-parse "$REF^{commit}" 2>/dev/null || git rev-parse "$REF")" echo "Pushed tag $REF" else CURRENT_BRANCH="$(git branch --show-current)" run git checkout -B "$REF" run git commit --allow-empty -m "$MESSAGE" + POLL_SHA="$(git rev-parse HEAD)" run git push -u origin "$REF" if [[ -n "$CURRENT_BRANCH" ]]; then run git checkout "$CURRENT_BRANCH" @@ -120,4 +122,18 @@ else echo "Pushed branch $REF" fi +REMOTE_URL="$(git remote get-url origin 2>/dev/null || true)" +REPO_SLUG="" +if [[ "$REMOTE_URL" =~ github\.com[:/]([^/]+/[^/.]+) ]]; then + REPO_SLUG="${BASH_REMATCH[1]%.git}" +fi + echo "Xcode Cloud should start the screenshots workflow shortly." +if [[ -n "$POLL_SHA" && -n "$REPO_SLUG" ]]; then + cat < dict[str, Any]: + body = build_failure_comment( + build_run_id=build_run_id, + commit_sha=commit_sha, + exit_code=exit_code, + errors=errors, + log_excerpt=log_excerpt, + ) + with httpx.Client(timeout=60.0) as client: + comment = upsert_commit_comment( + repo, + commit_sha, + body, + token=token, + client=client, + ) + return { + "status": BuildStatus.FAILED.value, + "comment_id": comment["id"], + "comment_url": comment["html_url"], + } + + +def publish_no_screenshots_report( + repo: str, + commit_sha: str, + build_run_id: str, + *, + token: str, + errors: list[str] | None = None, +) -> dict[str, Any]: + body = build_no_screenshots_comment( + build_run_id=build_run_id, + commit_sha=commit_sha, + errors=errors, + ) + with httpx.Client(timeout=60.0) as client: + comment = upsert_commit_comment( + repo, + commit_sha, + body, + token=token, + client=client, + ) + return { + "status": BuildStatus.NO_SCREENSHOTS.value, + "comment_id": comment["id"], + "comment_url": comment["html_url"], + } + + def publish_screenshot_commit_report( repo: str, commit_sha: str, @@ -81,9 +147,11 @@ def publish_screenshot_commit_report( ) return { + "status": BuildStatus.SUCCESS.value, "comment_id": comment["id"], "comment_url": comment["html_url"], "upload_count": len(uploads), "comparison_count": len(comparisons), "baseline_commit_sha": baseline_commit_sha, + "screenshot_urls": screenshot_urls, } diff --git a/scripts/xcode_cloud/extract.py b/scripts/xcode_cloud/extract.py index 0bb8c51..8a7ab9b 100644 --- a/scripts/xcode_cloud/extract.py +++ b/scripts/xcode_cloud/extract.py @@ -1,7 +1,8 @@ -"""Extract screenshot attachments from an .xcresult bundle.""" +"""Extract screenshot attachments and failure summaries from an .xcresult bundle.""" from __future__ import annotations +import json import shutil import subprocess from pathlib import Path @@ -47,3 +48,86 @@ def extract_attachments( subprocess.run(command, check=True, capture_output=True, text=True) return sorted(path for path in output_dir.rglob("*.png") if path.is_file()) + + +def extract_test_failure_summaries( + bundle_path: Path, + *, + limit: int = 20, +) -> list[str]: + """Return human-readable failing test summaries when xcresulttool is available.""" + if not bundle_path.exists(): + return [] + + tool = xcresulttool_path() + if tool is None: + return [] + + command = [ + tool, + "get", + "test-results", + "tests", + "--path", + str(bundle_path), + "--format", + "json", + ] + result = subprocess.run(command, capture_output=True, text=True, check=False) + if result.returncode != 0 or not result.stdout.strip(): + return _summarize_test_failures_legacy(tool, bundle_path, limit=limit) + + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError: + return _summarize_test_failures_legacy(tool, bundle_path, limit=limit) + + failures: list[str] = [] + tests = payload.get("tests", payload) + if isinstance(tests, list): + for item in tests: + if not isinstance(item, dict): + continue + status = str(item.get("testStatus", item.get("status", ""))).upper() + if status not in {"FAILURE", "FAILED"}: + continue + name = item.get("name") or item.get("identifier", "unknown test") + message = item.get("failureMessage") or item.get("message") + if message: + failures.append(f"{name}: {message}") + else: + failures.append(str(name)) + if len(failures) >= limit: + break + return failures + + +def _summarize_test_failures_legacy( + tool: str, + bundle_path: Path, + *, + limit: int, +) -> list[str]: + command = [ + tool, + "get", + "test-results", + "summary", + "--path", + str(bundle_path), + "--format", + "json", + ] + result = subprocess.run(command, capture_output=True, text=True, check=False) + if result.returncode != 0 or not result.stdout.strip(): + return [] + + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError: + return [] + + failed_count = payload.get("failedTests", payload.get("totalFailureCount")) + if failed_count: + return [f"{failed_count} test(s) failed"] + return [] diff --git a/scripts/xcode_cloud/github_comments.py b/scripts/xcode_cloud/github_comments.py index 3705074..16b48ef 100644 --- a/scripts/xcode_cloud/github_comments.py +++ b/scripts/xcode_cloud/github_comments.py @@ -4,14 +4,25 @@ import json import re +from enum import Enum from pathlib import Path +from typing import Any from scripts.xcode_cloud.compare import ScreenshotChange, ScreenshotComparison from scripts.xcode_cloud.upload import UploadedScreenshot COMMENT_MARKER = "" URLS_MARKER_START = "" +STATUS_MARKER_START = "" + +TERMINAL_BUILD_STATUSES = frozenset({"success", "failed", "no_screenshots"}) + + +class BuildStatus(str, Enum): + SUCCESS = "success" + FAILED = "failed" + NO_SCREENSHOTS = "no_screenshots" def github_headers(token: str) -> dict[str, str]: @@ -26,24 +37,17 @@ def short_sha(commit_sha: str) -> str: return commit_sha[:7] -def embed_screenshot_urls(body: str, urls: dict[str, str]) -> str: - stripped = strip_screenshot_urls_block(body).rstrip() - payload = json.dumps(urls, separators=(",", ":"), sort_keys=True) - block = f"{URLS_MARKER_START}\n{payload}\n{URLS_MARKER_END}" - return f"{stripped}\n\n{block}" - - -def strip_screenshot_urls_block(body: str) -> str: +def _strip_marker_block(body: str, marker_start: str) -> str: pattern = re.compile( - rf"\n*{re.escape(URLS_MARKER_START)}.*?{re.escape(URLS_MARKER_END)}", + rf"\n*{re.escape(marker_start)}.*?{re.escape(MARKER_END)}", flags=re.DOTALL, ) return pattern.sub("", body) -def parse_screenshot_urls(body: str) -> dict[str, str]: +def _parse_marker_json(body: str, marker_start: str) -> dict[str, Any]: match = re.search( - rf"{re.escape(URLS_MARKER_START)}\s*(\{{.*?\}})\s*{re.escape(URLS_MARKER_END)}", + rf"{re.escape(marker_start)}\s*(\{{.*?\}})\s*{re.escape(MARKER_END)}", body, flags=re.DOTALL, ) @@ -53,9 +57,173 @@ def parse_screenshot_urls(body: str) -> dict[str, str]: payload = json.loads(match.group(1)) except json.JSONDecodeError: return {} - if not isinstance(payload, dict): - return {} - return {str(name): str(url) for name, url in payload.items()} + return payload if isinstance(payload, dict) else {} + + +def _embed_marker_block(body: str, marker_start: str, payload: dict[str, Any]) -> str: + stripped = _strip_marker_block(body, marker_start).rstrip() + encoded = json.dumps(payload, separators=(",", ":"), sort_keys=True) + block = f"{marker_start}\n{encoded}\n{MARKER_END}" + return f"{stripped}\n\n{block}" + + +def strip_metadata_blocks(body: str) -> str: + return _strip_marker_block(_strip_marker_block(body, STATUS_MARKER_START), URLS_MARKER_START) + + +def embed_screenshot_urls(body: str, urls: dict[str, str]) -> str: + return _embed_marker_block(body, URLS_MARKER_START, urls) + + +def strip_screenshot_urls_block(body: str) -> str: + return _strip_marker_block(body, URLS_MARKER_START) + + +def parse_screenshot_urls(body: str) -> dict[str, str]: + urls_payload = _parse_marker_json(body, URLS_MARKER_START) + if urls_payload and "status" not in urls_payload: + return {str(name): str(url) for name, url in urls_payload.items()} + + status_payload = parse_build_status_payload(body) + screenshot_urls = status_payload.get("screenshot_urls", {}) + if isinstance(screenshot_urls, dict): + return {str(name): str(url) for name, url in screenshot_urls.items()} + return {} + + +def embed_build_status(body: str, payload: dict[str, Any]) -> str: + return _embed_marker_block(body, STATUS_MARKER_START, payload) + + +def parse_build_status_payload(body: str) -> dict[str, Any]: + return _parse_marker_json(body, STATUS_MARKER_START) + + +def parse_build_status(body: str) -> BuildStatus | None: + payload = parse_build_status_payload(body) + status = payload.get("status") + if status in TERMINAL_BUILD_STATUSES: + return BuildStatus(status) + return None + + +def build_status_payload( + *, + status: BuildStatus, + build_run_id: str, + commit_sha: str | None = None, + exit_code: int | None = None, + errors: list[str] | None = None, + screenshot_urls: dict[str, str] | None = None, + baseline_commit_sha: str | None = None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "status": status.value, + "build_id": build_run_id, + } + if commit_sha: + payload["commit_sha"] = commit_sha + if exit_code is not None: + payload["exit_code"] = exit_code + if errors: + payload["errors"] = errors + if screenshot_urls: + payload["screenshot_urls"] = screenshot_urls + if baseline_commit_sha: + payload["baseline_commit_sha"] = baseline_commit_sha + return payload + + +def build_failure_comment( + *, + build_run_id: str, + commit_sha: str, + exit_code: int | None = None, + errors: list[str] | None = None, + log_excerpt: str | None = None, +) -> str: + lines = [ + COMMENT_MARKER, + "### Xcode Cloud: failed", + "", + f"Commit `{short_sha(commit_sha)}`", + f"Build `{build_run_id}`", + "", + ] + if exit_code is not None: + lines.append(f"`xcodebuild` exit code: `{exit_code}`") + lines.append("") + if errors: + lines.extend(["**Errors**", ""]) + lines.extend(f"- {error}" for error in errors) + lines.append("") + if log_excerpt: + lines.extend(["**Log excerpt**", "", "```", log_excerpt.rstrip(), "```", ""]) + lines.append("Fix the issues above and push again to re-run screenshots.") + body = "\n".join(lines).rstrip() + return embed_build_status( + body, + build_status_payload( + status=BuildStatus.FAILED, + build_run_id=build_run_id, + commit_sha=commit_sha, + exit_code=exit_code, + errors=errors or [], + ), + ) + + +def build_no_screenshots_comment( + *, + build_run_id: str, + commit_sha: str, + errors: list[str] | None = None, +) -> str: + lines = [ + COMMENT_MARKER, + "### Xcode Cloud: no screenshots", + "", + f"Commit `{short_sha(commit_sha)}`", + f"Build `{build_run_id}`", + "", + "The build finished but no screenshot PNGs were extracted from the test result bundle.", + "", + ] + if errors: + lines.extend(["**Details**", ""]) + lines.extend(f"- {error}" for error in errors) + lines.append("") + body = "\n".join(lines).rstrip() + return embed_build_status( + body, + build_status_payload( + status=BuildStatus.NO_SCREENSHOTS, + build_run_id=build_run_id, + commit_sha=commit_sha, + errors=errors or [], + ), + ) + + +def finalize_success_comment( + body: str, + *, + build_run_id: str, + commit_sha: str, + screenshot_urls: dict[str, str], + baseline_commit_sha: str | None = None, +) -> str: + body = embed_screenshot_urls(body, screenshot_urls) + return embed_build_status( + body, + build_status_payload( + status=BuildStatus.SUCCESS, + build_run_id=build_run_id, + commit_sha=commit_sha, + screenshot_urls=screenshot_urls, + baseline_commit_sha=baseline_commit_sha, + ), + ) def build_screenshot_comment( @@ -117,8 +285,16 @@ def build_screenshot_comment( lines.append(f"| `{path.name}` |") body = "\n".join(lines) + if screenshot_urls and commit_sha: + return finalize_success_comment( + body, + build_run_id=build_run_id, + commit_sha=commit_sha, + screenshot_urls=screenshot_urls, + baseline_commit_sha=baseline_commit_sha, + ) if screenshot_urls: - body = embed_screenshot_urls(body, screenshot_urls) + return embed_screenshot_urls(body, screenshot_urls) return body diff --git a/scripts/xcode_cloud/github_commit.py b/scripts/xcode_cloud/github_commit.py index 3f7bd79..6765770 100644 --- a/scripts/xcode_cloud/github_commit.py +++ b/scripts/xcode_cloud/github_commit.py @@ -2,11 +2,30 @@ from __future__ import annotations +import time +from dataclasses import dataclass from pathlib import Path import httpx -from scripts.xcode_cloud.github_comments import COMMENT_MARKER, github_headers, parse_screenshot_urls +from scripts.xcode_cloud.github_comments import ( + COMMENT_MARKER, + TERMINAL_BUILD_STATUSES, + BuildStatus, + github_headers, + parse_build_status, + parse_build_status_payload, + parse_screenshot_urls, +) + + +@dataclass(frozen=True) +class CommitReportResult: + status: BuildStatus + comment_id: int + comment_url: str + body: str + payload: dict def find_commit_comment_id( @@ -72,6 +91,70 @@ def upsert_commit_comment( http.close() +def fetch_commit_report( + repo: str, + commit_sha: str, + *, + token: str, + client: httpx.Client | None = None, +) -> CommitReportResult | None: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + response = http.get( + f"https://api.github.com/repos/{repo}/commits/{commit_sha}/comments", + headers=github_headers(token), + ) + if response.status_code == 404: + return None + response.raise_for_status() + for comment in response.json(): + body = comment.get("body", "") + if COMMENT_MARKER not in body: + continue + status = parse_build_status(body) + if status is None: + continue + return CommitReportResult( + status=status, + comment_id=comment["id"], + comment_url=comment["html_url"], + body=body, + payload=parse_build_status_payload(body), + ) + return None + finally: + if close_client: + http.close() + + +def wait_for_commit_report( + repo: str, + commit_sha: str, + *, + token: str, + timeout_seconds: int = 3600, + poll_interval_seconds: int = 30, + client: httpx.Client | None = None, +) -> CommitReportResult: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + deadline = time.time() + timeout_seconds + try: + while time.time() < deadline: + report = fetch_commit_report(repo, commit_sha, token=token, client=http) + if report is not None and report.status.value in TERMINAL_BUILD_STATUSES: + return report + time.sleep(poll_interval_seconds) + finally: + if close_client: + http.close() + raise TimeoutError( + f"Timed out after {timeout_seconds}s waiting for a terminal build report " + f"on {repo}@{commit_sha[:7]}" + ) + + def fetch_screenshot_urls_from_commit( repo: str, commit_sha: str, From ef8684b244d9eece3e4e4868f97d42a03a38f97f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 7 Aug 2026 22:29:33 +0000 Subject: [PATCH 10/10] Remove trigger script, polling, and App Store Connect API path Xcode Cloud runs on every push; Cursor can wait for CI instead of custom polling. Drop trigger-screenshots.sh, wait-for-commit-report, and ASC fetch/trigger commands. Keep ci_post, Imgur upload, and commit comments as the only integration surface. Co-authored-by: Greg --- Bedtime/ci_scripts/ci_post_xcodebuild.sh | 4 +- scripts/README.md | 118 +++--------- scripts/fetch_xcode_cloud_screenshots.py | 233 +++-------------------- scripts/trigger-screenshots.sh | 139 -------------- scripts/xcode_cloud/__init__.py | 2 +- scripts/xcode_cloud/asc_auth.py | 59 ------ scripts/xcode_cloud/client.py | 134 ------------- scripts/xcode_cloud/extract.py | 17 ++ scripts/xcode_cloud/github_commit.py | 85 +-------- scripts/xcode_cloud/screenshots.py | 157 --------------- scripts/xcode_cloud/trigger.py | 92 --------- 11 files changed, 76 insertions(+), 964 deletions(-) delete mode 100755 scripts/trigger-screenshots.sh delete mode 100644 scripts/xcode_cloud/asc_auth.py delete mode 100644 scripts/xcode_cloud/client.py delete mode 100644 scripts/xcode_cloud/screenshots.py delete mode 100644 scripts/xcode_cloud/trigger.py diff --git a/Bedtime/ci_scripts/ci_post_xcodebuild.sh b/Bedtime/ci_scripts/ci_post_xcodebuild.sh index 9ceafc8..e8165ab 100755 --- a/Bedtime/ci_scripts/ci_post_xcodebuild.sh +++ b/Bedtime/ci_scripts/ci_post_xcodebuild.sh @@ -2,8 +2,8 @@ set -eu # Runs on the Xcode Cloud Mac after xcodebuild finishes. -# Always posts a terminal sticky commit comment for agents to poll: -# success (screenshot diff), failed (build/test errors), or no_screenshots. +# Extracts screenshots, uploads to Imgur, and posts a sticky commit comment. +# Cursor/agents can wait for CI, then read that comment for results. OUTPUT_DIR="${CI_DERIVED_DATA_PATH:-/tmp}/xcode-cloud-screenshots" SCREENSHOTS_DIR="$OUTPUT_DIR/screenshots" diff --git a/scripts/README.md b/scripts/README.md index bb4940f..e34cbfc 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,112 +1,67 @@ # Xcode Cloud screenshots -## Recommended: git-triggered workflow (no API polling) +Every push runs Xcode Cloud. `ci_post_xcodebuild.sh` extracts UI test screenshots, uploads them to Imgur, and posts a sticky **commit comment** with results. -The cleanest setup avoids App Store Connect API credentials entirely: +## Flow ```text -main push → screenshots on commit, Imgur URLs in commit comment -PR push → compare vs main baseline commit comment, sticky comment on PR commit -git push screenshots/pr-42 → on-demand screenshot run (branch start condition) -``` - -### Agent feedback loop +push to main → tests capture screenshots → commit comment (full set, becomes baseline) -Cloud agents can trigger, poll, and iterate without watching Xcode Cloud directly: - -```text -1. ./scripts/trigger-screenshots.sh --pr 42 --notes "Check home screen" -2. Poll until a terminal commit comment appears: - python3 scripts/fetch_xcode_cloud_screenshots.py wait-for-commit-report \ - --repo owner/repo --commit-sha -3. Read status from the hidden JSON block: - success → screenshot diff table + Imgur URLs - failed → xcodebuild/test errors to fix - no_screenshots → tests ran but no PNG attachments -4. Fix code, push again, repeat +push to PR → tests capture screenshots → commit comment (diff vs main baseline) ``` -`ci_post_xcodebuild.sh` **always** posts a terminal commit comment when `GITHUB_TOKEN` and `CI_COMMIT` are set — including build failures — so agents do not poll forever. +Cursor (or any agent) can **wait for CI to finish**, then read the commit comment on that push's SHA for screenshots, diffs, or failure details. -### 1. Create a dedicated Xcode Cloud workflow +## Xcode Cloud setup -In Xcode or App Store Connect, add a **Screenshots** workflow with: +Add a workflow (or use your existing per-push workflow) with: | Setting | Value | |---------|--------| -| Start condition | **Branch changes** → `main` (keeps baseline fresh) | -| | and/or branches beginning with `screenshots/` | -| | and/or **Pull request changes** | +| Start condition | **Every push** (branch + pull request) | | Test action | Scheme `Bedtime`, include `BedtimeUITests` | | Test plan | Screenshots: **On, and keep all** | -Keep this workflow separate from your main CI so screenshot runs do not block merges. - -### 2. Xcode Cloud secrets +### Secrets ```bash -IMGUR_CLIENT_ID=... # public image URLs (success path only) -GITHUB_TOKEN=... # sticky commit comments (success + failure) +IMGUR_CLIENT_ID=... # public image URLs (success path) +GITHUB_TOKEN=... # commit comments (success + failure) ``` `GITHUB_TOKEN` needs permission to create and update **commit comments**. -No `APP_STORE_CONNECT_*` keys needed for the git-trigger path. No S3, no separate baseline branch. - -### Sticky commit comments - -Screenshots and build outcomes are reported as a **commit comment** on the built commit (`github.com/{owner}/{repo}/commit/{sha}`). No PR required. +## Commit comments -**Main builds** (`CI_COMMIT` on `main`): +Reports appear on `github.com/{owner}/{repo}/commit/{sha}` (Commits tab on PRs). -1. Upload screenshots to Imgur -2. Post/update one sticky comment on that commit -3. Embed Imgur URLs in hidden metadata for baseline lookup +**Main builds** — upload all screenshots to Imgur; embed URLs in the comment for baseline lookup. -**PR builds**: +**PR builds** — download main's baseline from `CI_PULL_REQUEST_TARGET_COMMIT`, pixel-compare, show before/after for changed and new screenshots only. -1. Read Imgur URLs from the commit comment on `CI_PULL_REQUEST_TARGET_COMMIT` (main HEAD) -2. Compare pixel-by-pixel against new screenshots -3. Post/update sticky comment on `CI_COMMIT` with before/after table for changed/new only +**Failed builds** — post exit code, test failures, and build log URL (no Imgur upload). -**Failed builds** (`CI_XCODEBUILD_EXIT_CODE != 0`): - -1. Skip screenshot upload -2. Post/update sticky comment with exit code, test failures, and build log URL - -Hidden status block (for agents): +Hidden status block for programmatic reads: ```html ``` -### 3. Trigger from your machine or a Cursor agent - -```bash -# On-demand screenshots for PR 42 with notes -./scripts/trigger-screenshots.sh --pr 42 --notes "Verify home + settings after sleep bank changes" - -# Poll for outcome (commit SHA printed by trigger script) -python3 scripts/fetch_xcode_cloud_screenshots.py wait-for-commit-report \ - --repo owner/repo --commit-sha abc123def456 --output-json /tmp/report.json -``` - -Commit/tag message format (parsed automatically): +## New screenshot scenarios -```text -screenshots: trigger +Use scenario-specific attachment names (e.g. `01-home-empty-bank`). When adding a new test case, **merge the test to main early** so main captures the baseline before the feature branch gets far ahead. -What to test: -- Home screen with mock sleep data -- Settings sliders and wake time picker -``` +## Local / manual CLI -### 4. Manual CLI +Used by `ci_post_xcodebuild.sh`; also runnable locally on a Mac with a `.xcresult` bundle: ```bash -# Publish outcomes directly +python3 scripts/fetch_xcode_cloud_screenshots.py extract-local \ + --bundle-path /path/to/Result.xcresult \ + --output-dir ./screenshots + python3 scripts/fetch_xcode_cloud_screenshots.py comment-build \ --repo owner/repo \ --commit-sha abc123 \ @@ -114,19 +69,11 @@ python3 scripts/fetch_xcode_cloud_screenshots.py comment-build \ --run-id build-1 \ --status success \ --screenshots-dir ./screenshots - -python3 scripts/fetch_xcode_cloud_screenshots.py comment-build \ - --repo owner/repo \ - --commit-sha abc123 \ - --run-id build-1 \ - --status failed \ - --exit-code 65 \ - --errors-file ./errors.txt ``` ## What generates the screenshots -**`BedtimeUITests/ScreenshotTests.swift`** — XCUITest that saves `XCTAttachment` PNGs with `.keepAlways`. +**`BedtimeUITests/ScreenshotTests.swift`** — XCUITest attachments with `.keepAlways`. ```bash xcodebuild test \ @@ -135,14 +82,3 @@ xcodebuild test \ -destination 'platform=iOS Simulator,name=iPhone 16' \ -only-testing:BedtimeUITests/ScreenshotTests ``` - -## Optional: App Store Connect API path - -Only needed if you cannot push git refs (e.g. trigger from a system with no git write access): - -```bash -python3 scripts/fetch_xcode_cloud_screenshots.py trigger-and-fetch \ - --workflow-id WORKFLOW_ID --branch main -``` - -This still polls App Store Connect until the build completes. Prefer git push + `wait-for-commit-report` when possible. diff --git a/scripts/fetch_xcode_cloud_screenshots.py b/scripts/fetch_xcode_cloud_screenshots.py index 54def14..c33be6c 100644 --- a/scripts/fetch_xcode_cloud_screenshots.py +++ b/scripts/fetch_xcode_cloud_screenshots.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -"""Fetch Xcode Cloud screenshots after a build run completes.""" +"""Xcode Cloud screenshot extraction and GitHub commit reporting.""" from __future__ import annotations import argparse +import json +import os import sys from pathlib import Path @@ -11,22 +13,14 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from scripts.xcode_cloud.asc_auth import create_asc_token, credentials_from_env -from scripts.xcode_cloud.client import XcodeCloudClient -from scripts.xcode_cloud.extract import XcresultToolNotFoundError from scripts.xcode_cloud.build_report import ( publish_build_failure_report, publish_no_screenshots_report, publish_screenshot_commit_report, ) +from scripts.xcode_cloud.extract import extract_screenshots_from_local_bundle from scripts.xcode_cloud.github_comments import BuildStatus, build_screenshot_comment -from scripts.xcode_cloud.github_commit import upsert_commit_comment, wait_for_commit_report -from scripts.xcode_cloud.screenshots import ( - extract_screenshots_from_local_bundle, - fetch_screenshots_from_build_run, - fetch_test_result_bundle, -) -from scripts.xcode_cloud.trigger import trigger_and_wait +from scripts.xcode_cloud.github_commit import upsert_commit_comment from scripts.xcode_cloud.upload import ( UploadConfigError, UploadedScreenshot, @@ -37,67 +31,10 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description="Fetch Xcode Cloud test screenshots after a build completes." + description="Extract Xcode Cloud UI test screenshots and publish GitHub commit reports." ) subparsers = parser.add_subparsers(dest="command", required=True) - fetch_parser = subparsers.add_parser( - "fetch", - help="Download the test result bundle for a completed build run.", - ) - fetch_parser.add_argument("--run-id", required=True, help="ciBuildRuns ID") - fetch_parser.add_argument( - "--output-dir", - default="./xcode-cloud-output", - help="Directory for downloaded bundles and screenshots", - ) - fetch_parser.add_argument( - "--only-failures", - action="store_true", - help="Export only attachments associated with failing tests", - ) - fetch_parser.add_argument( - "--skip-extract", - action="store_true", - help="Download the bundle but do not run xcresulttool extraction", - ) - - wait_parser = subparsers.add_parser( - "wait-and-fetch", - help="Poll until the build run completes, then fetch screenshots.", - ) - wait_parser.add_argument("--run-id", required=True, help="ciBuildRuns ID") - wait_parser.add_argument( - "--output-dir", - default="./xcode-cloud-output", - help="Directory for downloaded bundles and screenshots", - ) - wait_parser.add_argument("--timeout-seconds", type=int, default=3600) - wait_parser.add_argument("--poll-interval-seconds", type=int, default=30) - wait_parser.add_argument("--only-failures", action="store_true") - - trigger_parser = subparsers.add_parser( - "trigger-and-fetch", - help="Start an Xcode Cloud build, wait until it finishes, then fetch screenshots.", - ) - trigger_parser.add_argument("--workflow-id", required=True, help="ciWorkflows ID") - branch_group = trigger_parser.add_mutually_exclusive_group(required=True) - branch_group.add_argument("--branch", help="Branch name to build") - branch_group.add_argument("--git-reference-id", help="scmGitReferences ID") - trigger_parser.add_argument( - "--output-dir", - default="./xcode-cloud-output", - help="Directory for downloaded bundles and screenshots", - ) - trigger_parser.add_argument("--timeout-seconds", type=int, default=3600) - trigger_parser.add_argument("--poll-interval-seconds", type=int, default=30) - trigger_parser.add_argument("--only-failures", action="store_true") - trigger_parser.add_argument( - "--skip-extract", - action="store_true", - help="Download the bundle but do not run xcresulttool extraction", - ) - local_parser = subparsers.add_parser( "extract-local", help="Extract screenshots from a local .xcresult bundle.", @@ -110,6 +47,12 @@ def build_parser() -> argparse.ArgumentParser: ) local_parser.add_argument("--only-failures", action="store_true") + failures_parser = subparsers.add_parser( + "extract-failures", + help="Print failing test summaries from a local .xcresult bundle.", + ) + failures_parser.add_argument("--bundle-path", required=True) + comment_parser = subparsers.add_parser( "comment-build", help="Post or update a sticky build report comment on a Git commit.", @@ -168,22 +111,9 @@ def build_parser() -> argparse.ArgumentParser: help="Text file with What to test notes for the commit comment", ) - wait_report_parser = subparsers.add_parser( - "wait-for-commit-report", - help="Poll until a terminal build report comment appears on a commit.", - ) - wait_report_parser.add_argument("--repo", required=True, help="owner/repo") - wait_report_parser.add_argument("--commit-sha", required=True) - wait_report_parser.add_argument("--timeout-seconds", type=int, default=3600) - wait_report_parser.add_argument("--poll-interval-seconds", type=int, default=30) - wait_report_parser.add_argument( - "--output-json", - help="Write the parsed build-status payload to this JSON file", - ) - upload_parser = subparsers.add_parser( "upload-screenshots", - help="Upload extracted screenshots to a public S3 bucket.", + help="Upload extracted screenshots to Imgur or S3.", ) upload_parser.add_argument("--screenshots-dir", required=True) upload_parser.add_argument("--build-id", required=True) @@ -198,12 +128,6 @@ def build_parser() -> argparse.ArgumentParser: default="./xcode-cloud-output/screenshots-manifest.json", help="Where to write the public URL manifest", ) - - failures_parser = subparsers.add_parser( - "extract-failures", - help="Print failing test summaries from a local .xcresult bundle.", - ) - failures_parser.add_argument("--bundle-path", required=True) return parser @@ -213,9 +137,15 @@ def _read_error_lines(path: str | None) -> list[str]: return [line.strip() for line in Path(path).read_text().splitlines() if line.strip()] -def _publish_comment_build(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: - import os +def _read_what_to_test(args: argparse.Namespace) -> str | None: + path = getattr(args, "what_to_test_file", None) + if not path: + return None + text = Path(path).read_text().strip() + return text or None + +def _publish_comment_build(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: token = os.environ.get("GITHUB_TOKEN") if not token: parser.error("GITHUB_TOKEN is required") @@ -281,7 +211,12 @@ def main(argv: list[str] | None = None) -> int: Path(args.output_dir), only_failures=args.only_failures, ) - _print_screenshots(screenshots) + if not screenshots: + print("No screenshots found.") + else: + print(f"Extracted {len(screenshots)} screenshot(s):") + for path in screenshots: + print(path) return 0 if args.command == "extract-failures": @@ -295,9 +230,6 @@ def main(argv: list[str] | None = None) -> int: return _publish_comment_build(args, parser) if args.command == "comment-commit": - import json - import os - token = os.environ.get("GITHUB_TOKEN") if not token: parser.error("GITHUB_TOKEN is required for comment-commit") @@ -334,36 +266,6 @@ def main(argv: list[str] | None = None) -> int: parser.error("comment-commit requires --screenshots-dir or --manifest") - if args.command == "wait-for-commit-report": - import json - import os - - token = os.environ.get("GITHUB_TOKEN") - if not token: - parser.error("GITHUB_TOKEN is required for wait-for-commit-report") - - try: - report = wait_for_commit_report( - args.repo, - args.commit_sha, - token=token, - timeout_seconds=args.timeout_seconds, - poll_interval_seconds=args.poll_interval_seconds, - ) - except TimeoutError as error: - print(str(error), file=sys.stderr) - return 1 - - print(f"Build report status: {report.status.value}") - print(f"Comment: {report.comment_url}") - if args.output_json: - Path(args.output_json).write_text(json.dumps(report.payload, indent=2)) - print(f"Wrote status payload to {args.output_json}") - return 0 if report.status == BuildStatus.SUCCESS else 1 - - if args.command == "comment-pr": - parser.error("comment-pr was removed; use comment-build or wait-for-commit-report") - if args.command == "upload-screenshots": try: uploads = upload_screenshots( @@ -382,86 +284,7 @@ def main(argv: list[str] | None = None) -> int: print(f"Manifest: {manifest_path}") return 0 - credentials = credentials_from_env() - output_dir = Path(args.output_dir) - build_succeeded = True - - with XcodeCloudClient(lambda: create_asc_token(credentials)) as client: - run_id = getattr(args, "run_id", None) - - if args.command == "trigger-and-fetch": - trigger_result = trigger_and_wait( - client, - args.workflow_id, - git_reference_id=getattr(args, "git_reference_id", None), - branch=getattr(args, "branch", None), - timeout_seconds=args.timeout_seconds, - poll_interval_seconds=args.poll_interval_seconds, - ) - run_id = trigger_result.build_run_id - build_succeeded = trigger_result.status.completion_status == "SUCCEEDED" - print( - f"Triggered build run {run_id}; completed with status " - f"{trigger_result.status.completion_status or 'UNKNOWN'}" - ) - - elif args.command == "wait-and-fetch": - status = client.wait_for_build_run( - args.run_id, - timeout_seconds=args.timeout_seconds, - poll_interval_seconds=args.poll_interval_seconds, - ) - run_id = status.run_id - build_succeeded = status.completion_status == "SUCCEEDED" - print( - f"Build run {status.run_id} completed with status " - f"{status.completion_status or 'UNKNOWN'}" - ) - - assert run_id is not None - - if getattr(args, "skip_extract", False): - _, artifact_id, bundle_path = fetch_test_result_bundle( - client, - run_id, - output_dir, - ) - print(f"Downloaded artifact {artifact_id} to {bundle_path}") - return 0 if build_succeeded else 1 - - try: - result = fetch_screenshots_from_build_run( - client, - run_id, - output_dir, - only_failures=getattr(args, "only_failures", False), - ) - except XcresultToolNotFoundError as error: - print(str(error), file=sys.stderr) - return 2 - - print(f"Test action: {result.test_action_id}") - print(f"Artifact: {result.artifact_id}") - print(f"Bundle: {result.bundle_path}") - _print_screenshots(result.screenshot_paths) - return 0 if build_succeeded else 1 - - -def _print_screenshots(screenshots: list[Path] | tuple[Path, ...]) -> None: - if not screenshots: - print("No screenshots found.") - return - print(f"Extracted {len(screenshots)} screenshot(s):") - for path in screenshots: - print(path) - - -def _read_what_to_test(args: argparse.Namespace) -> str | None: - path = getattr(args, "what_to_test_file", None) - if not path: - return None - text = Path(path).read_text().strip() - return text or None + parser.error(f"Unknown command: {args.command}") if __name__ == "__main__": diff --git a/scripts/trigger-screenshots.sh b/scripts/trigger-screenshots.sh deleted file mode 100755 index 9b4b953..0000000 --- a/scripts/trigger-screenshots.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - cat <<'EOF' -Trigger an Xcode Cloud screenshot workflow via git push (no App Store Connect API). - -Create a dedicated workflow in Xcode Cloud with a start condition for either: - - branches beginning with screenshots/ - - tags beginning with screenshots/ - -Usage: - scripts/trigger-screenshots.sh --pr 42 --notes "Check home and settings" - scripts/trigger-screenshots.sh --branch feature/foo --notes "Dark mode pass" - scripts/trigger-screenshots.sh --tag release-1.2.0 --notes "Release screenshots" - -Options: - --pr NUMBER Target PR number (creates branch screenshots/pr-NUMBER) - --branch NAME Push to screenshots/NAME instead - --tag NAME Create annotated tag screenshots/NAME (use tag start condition) - --notes TEXT "What to test" notes included in the commit/tag message - --dry-run Print the git commands without running them - -h, --help Show this help - -The commit/tag message uses this format: - - screenshots: trigger - - What to test: - -EOF -} - -PR="" -BRANCH="" -TAG="" -NOTES="" -DRY_RUN=false - -while [[ $# -gt 0 ]]; do - case "$1" in - --pr) - PR="$2" - shift 2 - ;; - --branch) - BRANCH="$2" - shift 2 - ;; - --tag) - TAG="$2" - shift 2 - ;; - --notes) - NOTES="$2" - shift 2 - ;; - --dry-run) - DRY_RUN=true - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "Unknown option: $1" >&2 - usage >&2 - exit 1 - ;; - esac -done - -if [[ -n "$PR" && ( -n "$BRANCH" || -n "$TAG" ) ]]; then - echo "Use only one of --pr, --branch, or --tag" >&2 - exit 1 -fi - -if [[ -z "$PR" && -z "$BRANCH" && -z "$TAG" ]]; then - echo "Provide --pr, --branch, or --tag" >&2 - usage >&2 - exit 1 -fi - -if [[ -n "$PR" ]]; then - REF="screenshots/pr-${PR}" -elif [[ -n "$BRANCH" ]]; then - REF="screenshots/${BRANCH}" -else - REF="screenshots/${TAG}" -fi - -MESSAGE=$'screenshots: trigger\n' -if [[ -n "$NOTES" ]]; then - MESSAGE+=$'\nWhat to test:\n'"${NOTES}"$'\n' -fi - -run() { - if [[ "$DRY_RUN" == true ]]; then - printf '+' - printf ' %q' "$@" - printf '\n' - else - "$@" - fi -} - -if [[ -n "$TAG" ]]; then - run git tag -fa "$REF" -m "$MESSAGE" - run git push origin "$REF" --force - POLL_SHA="$(git rev-parse "$REF^{commit}" 2>/dev/null || git rev-parse "$REF")" - echo "Pushed tag $REF" -else - CURRENT_BRANCH="$(git branch --show-current)" - run git checkout -B "$REF" - run git commit --allow-empty -m "$MESSAGE" - POLL_SHA="$(git rev-parse HEAD)" - run git push -u origin "$REF" - if [[ -n "$CURRENT_BRANCH" ]]; then - run git checkout "$CURRENT_BRANCH" - fi - echo "Pushed branch $REF" -fi - -REMOTE_URL="$(git remote get-url origin 2>/dev/null || true)" -REPO_SLUG="" -if [[ "$REMOTE_URL" =~ github\.com[:/]([^/]+/[^/.]+) ]]; then - REPO_SLUG="${BASH_REMATCH[1]%.git}" -fi - -echo "Xcode Cloud should start the screenshots workflow shortly." -if [[ -n "$POLL_SHA" && -n "$REPO_SLUG" ]]; then - cat < str: - """Create a short-lived JWT for App Store Connect API requests.""" - now = int(time.time()) - headers = {"alg": "ES256", "kid": credentials.key_id, "typ": "JWT"} - payload = { - "iss": credentials.issuer_id, - "iat": now, - "exp": now + expiration_seconds, - "aud": "appstoreconnect-v1", - } - return jwt.encode(payload, credentials.private_key, algorithm="ES256", headers=headers) - - -def credentials_from_env() -> AscCredentials: - """Load App Store Connect credentials from standard environment variables.""" - import os - - key_id = os.environ.get("APP_STORE_CONNECT_KEY_ID") - issuer_id = os.environ.get("APP_STORE_CONNECT_ISSUER_ID") - private_key = os.environ.get("APP_STORE_CONNECT_PRIVATE_KEY") - - missing = [ - name - for name, value in ( - ("APP_STORE_CONNECT_KEY_ID", key_id), - ("APP_STORE_CONNECT_ISSUER_ID", issuer_id), - ("APP_STORE_CONNECT_PRIVATE_KEY", private_key), - ) - if not value - ] - if missing: - raise ValueError(f"Missing required environment variables: {', '.join(missing)}") - - return AscCredentials( - key_id=key_id, - issuer_id=issuer_id, - private_key=private_key.replace("\\n", "\n"), - ) diff --git a/scripts/xcode_cloud/client.py b/scripts/xcode_cloud/client.py deleted file mode 100644 index efbdaf0..0000000 --- a/scripts/xcode_cloud/client.py +++ /dev/null @@ -1,134 +0,0 @@ -"""App Store Connect API client for Xcode Cloud resources.""" - -from __future__ import annotations - -import time -from dataclasses import dataclass -from typing import Any, Callable -from urllib.parse import urljoin - -import httpx - - -class XcodeCloudError(RuntimeError): - """Raised when an Xcode Cloud API request fails.""" - - -@dataclass(frozen=True) -class BuildRunStatus: - run_id: str - execution_progress: str - completion_status: str | None - - -class XcodeCloudClient: - BASE_URL = "https://api.appstoreconnect.apple.com/v1/" - - def __init__( - self, - token_provider: Callable[[], str], - *, - client: httpx.Client | None = None, - ) -> None: - self._token_provider = token_provider - self._client = client or httpx.Client(timeout=60.0) - - def close(self) -> None: - self._client.close() - - def __enter__(self) -> XcodeCloudClient: - return self - - def __exit__(self, *args: object) -> None: - self.close() - - def _request( - self, - method: str, - path: str, - *, - allowed_statuses: set[int] | None = None, - **kwargs: Any, - ) -> dict[str, Any]: - url = path if path.startswith("http") else urljoin(self.BASE_URL, path.lstrip("/")) - headers = kwargs.pop("headers", {}) - headers["Authorization"] = f"Bearer {self._token_provider()}" - response = self._client.request(method, url, headers=headers, **kwargs) - allowed = allowed_statuses or {200} - if response.status_code not in allowed: - raise XcodeCloudError( - f"{method} {url} failed with {response.status_code}: {response.text}" - ) - if not response.content: - return {} - return response.json() - - def get_build_run(self, run_id: str) -> dict[str, Any]: - return self._request("GET", f"ciBuildRuns/{run_id}") - - def create_build_run(self, workflow_id: str, git_reference_id: str) -> dict[str, Any]: - body = { - "data": { - "type": "ciBuildRuns", - "relationships": { - "workflow": { - "data": {"type": "ciWorkflows", "id": workflow_id}, - }, - "sourceBranchOrTag": { - "data": {"type": "scmGitReferences", "id": git_reference_id}, - }, - }, - } - } - return self._request( - "POST", - "ciBuildRuns", - json=body, - allowed_statuses={201}, - ) - - def get_workflow(self, workflow_id: str, *, include_repository: bool = False) -> dict[str, Any]: - query = "include=repository" if include_repository else None - path = f"ciWorkflows/{workflow_id}" - if query: - path = f"{path}?{query}" - return self._request("GET", path) - - def list_git_references(self, repository_id: str) -> list[dict[str, Any]]: - payload = self._request("GET", f"scmRepositories/{repository_id}/gitReferences") - return payload.get("data", []) - - def list_build_actions(self, run_id: str) -> list[dict[str, Any]]: - payload = self._request("GET", f"ciBuildRuns/{run_id}/actions") - return payload.get("data", []) - - def list_artifacts(self, action_id: str) -> list[dict[str, Any]]: - payload = self._request("GET", f"ciBuildActions/{action_id}/artifacts") - return payload.get("data", []) - - def get_artifact(self, artifact_id: str) -> dict[str, Any]: - return self._request("GET", f"ciArtifacts/{artifact_id}") - - def get_build_run_status(self, run_id: str) -> BuildRunStatus: - payload = self.get_build_run(run_id) - attributes = payload["data"]["attributes"] - return BuildRunStatus( - run_id=run_id, - execution_progress=attributes.get("executionProgress", ""), - completion_status=attributes.get("completionStatus"), - ) - - def wait_for_build_run( - self, - run_id: str, - *, - timeout_seconds: int = 3600, - poll_interval_seconds: int = 30, - ) -> BuildRunStatus: - deadline = time.time() + timeout_seconds - while time.time() < deadline: - status = self.get_build_run_status(run_id) - if status.execution_progress == "COMPLETE": - return status - time.sleep(poll_interval_seconds) - raise TimeoutError(f"Timed out waiting for build run {run_id} to complete") diff --git a/scripts/xcode_cloud/extract.py b/scripts/xcode_cloud/extract.py index 8a7ab9b..e6d3cc5 100644 --- a/scripts/xcode_cloud/extract.py +++ b/scripts/xcode_cloud/extract.py @@ -131,3 +131,20 @@ def _summarize_test_failures_legacy( if failed_count: return [f"{failed_count} test(s) failed"] return [] + + +def extract_screenshots_from_local_bundle( + bundle_path: Path, + output_dir: Path, + *, + only_failures: bool = False, +) -> tuple[Path, ...]: + """Extract screenshots from a local .xcresult bundle (Xcode Cloud Mac path).""" + screenshots_dir = output_dir / "screenshots" + return tuple( + extract_attachments( + bundle_path, + screenshots_dir, + only_failures=only_failures, + ) + ) diff --git a/scripts/xcode_cloud/github_commit.py b/scripts/xcode_cloud/github_commit.py index 6765770..3f7bd79 100644 --- a/scripts/xcode_cloud/github_commit.py +++ b/scripts/xcode_cloud/github_commit.py @@ -2,30 +2,11 @@ from __future__ import annotations -import time -from dataclasses import dataclass from pathlib import Path import httpx -from scripts.xcode_cloud.github_comments import ( - COMMENT_MARKER, - TERMINAL_BUILD_STATUSES, - BuildStatus, - github_headers, - parse_build_status, - parse_build_status_payload, - parse_screenshot_urls, -) - - -@dataclass(frozen=True) -class CommitReportResult: - status: BuildStatus - comment_id: int - comment_url: str - body: str - payload: dict +from scripts.xcode_cloud.github_comments import COMMENT_MARKER, github_headers, parse_screenshot_urls def find_commit_comment_id( @@ -91,70 +72,6 @@ def upsert_commit_comment( http.close() -def fetch_commit_report( - repo: str, - commit_sha: str, - *, - token: str, - client: httpx.Client | None = None, -) -> CommitReportResult | None: - http = client or httpx.Client(timeout=30.0) - close_client = client is None - try: - response = http.get( - f"https://api.github.com/repos/{repo}/commits/{commit_sha}/comments", - headers=github_headers(token), - ) - if response.status_code == 404: - return None - response.raise_for_status() - for comment in response.json(): - body = comment.get("body", "") - if COMMENT_MARKER not in body: - continue - status = parse_build_status(body) - if status is None: - continue - return CommitReportResult( - status=status, - comment_id=comment["id"], - comment_url=comment["html_url"], - body=body, - payload=parse_build_status_payload(body), - ) - return None - finally: - if close_client: - http.close() - - -def wait_for_commit_report( - repo: str, - commit_sha: str, - *, - token: str, - timeout_seconds: int = 3600, - poll_interval_seconds: int = 30, - client: httpx.Client | None = None, -) -> CommitReportResult: - http = client or httpx.Client(timeout=30.0) - close_client = client is None - deadline = time.time() + timeout_seconds - try: - while time.time() < deadline: - report = fetch_commit_report(repo, commit_sha, token=token, client=http) - if report is not None and report.status.value in TERMINAL_BUILD_STATUSES: - return report - time.sleep(poll_interval_seconds) - finally: - if close_client: - http.close() - raise TimeoutError( - f"Timed out after {timeout_seconds}s waiting for a terminal build report " - f"on {repo}@{commit_sha[:7]}" - ) - - def fetch_screenshot_urls_from_commit( repo: str, commit_sha: str, diff --git a/scripts/xcode_cloud/screenshots.py b/scripts/xcode_cloud/screenshots.py deleted file mode 100644 index 4b9229c..0000000 --- a/scripts/xcode_cloud/screenshots.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Download Xcode Cloud test result bundles and extract screenshots.""" - -from __future__ import annotations - -import shutil -import zipfile -from dataclasses import dataclass -from pathlib import Path -from typing import Iterable - -import httpx - -from scripts.xcode_cloud.client import XcodeCloudClient, XcodeCloudError -from scripts.xcode_cloud.extract import extract_attachments - - -TEST_ACTION_TYPES = {"TEST"} -TEST_RESULT_FILE_TYPES = {"TEST_RESULT_BUNDLE", "XCRESULT"} - - -@dataclass(frozen=True) -class ScreenshotFetchResult: - build_run_id: str - test_action_id: str - artifact_id: str - bundle_path: Path - screenshot_paths: tuple[Path, ...] - - -def find_test_action(actions: Iterable[dict]) -> dict: - for action in actions: - action_type = action.get("attributes", {}).get("actionType") - if action_type in TEST_ACTION_TYPES: - return action - raise XcodeCloudError("No TEST build action found for this build run") - - -def find_test_result_artifact(artifacts: Iterable[dict]) -> dict: - for artifact in artifacts: - file_type = artifact.get("attributes", {}).get("fileType") - if file_type in TEST_RESULT_FILE_TYPES: - return artifact - raise XcodeCloudError("No test result bundle artifact found for the TEST action") - - -def _download_file(url: str, destination: Path, *, client: httpx.Client | None = None) -> None: - http = client or httpx.Client(timeout=120.0, follow_redirects=True) - close_client = client is None - try: - with http.stream("GET", url) as response: - response.raise_for_status() - destination.parent.mkdir(parents=True, exist_ok=True) - with destination.open("wb") as handle: - for chunk in response.iter_bytes(): - handle.write(chunk) - finally: - if close_client: - http.close() - - -def _prepare_xcresult_bundle(download_path: Path, output_dir: Path) -> Path: - if download_path.suffix == ".xcresult" and download_path.is_dir(): - return download_path - - if download_path.suffix == ".zip" or zipfile.is_zipfile(download_path): - extract_dir = output_dir / "extracted" - extract_dir.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(download_path) as archive: - archive.extractall(extract_dir) - candidates = list(extract_dir.rglob("*.xcresult")) - if not candidates: - raise XcodeCloudError(f"No .xcresult bundle found inside {download_path}") - return candidates[0] - - if download_path.name.endswith(".xcresult"): - if download_path.is_dir(): - return download_path - bundle_dir = output_dir / download_path.name - if bundle_dir.exists(): - shutil.rmtree(bundle_dir) - shutil.move(str(download_path), str(bundle_dir)) - return bundle_dir - - raise XcodeCloudError(f"Unsupported test result artifact format: {download_path}") - - -def fetch_test_result_bundle( - client: XcodeCloudClient, - build_run_id: str, - output_dir: Path, - *, - download_client: httpx.Client | None = None, -) -> tuple[str, str, Path]: - """Download the Xcode Cloud test result bundle for a completed build run.""" - actions = client.list_build_actions(build_run_id) - test_action = find_test_action(actions) - test_action_id = test_action["id"] - - artifacts = client.list_artifacts(test_action_id) - artifact = find_test_result_artifact(artifacts) - artifact_id = artifact["id"] - - artifact_payload = client.get_artifact(artifact_id) - download_url = artifact_payload["data"]["attributes"]["downloadUrl"] - file_name = artifact_payload["data"]["attributes"].get("fileName", f"{artifact_id}.zip") - - download_path = output_dir / file_name - _download_file(download_url, download_path, client=download_client) - bundle_path = _prepare_xcresult_bundle(download_path, output_dir) - return test_action_id, artifact_id, bundle_path - - -def fetch_screenshots_from_build_run( - client: XcodeCloudClient, - build_run_id: str, - output_dir: Path, - *, - only_failures: bool = False, - download_client: httpx.Client | None = None, -) -> ScreenshotFetchResult: - """Fetch a build run's test bundle and extract screenshot attachments.""" - screenshots_dir = output_dir / "screenshots" - test_action_id, artifact_id, bundle_path = fetch_test_result_bundle( - client, - build_run_id, - output_dir, - download_client=download_client, - ) - screenshot_paths = extract_attachments( - bundle_path, - screenshots_dir, - only_failures=only_failures, - ) - return ScreenshotFetchResult( - build_run_id=build_run_id, - test_action_id=test_action_id, - artifact_id=artifact_id, - bundle_path=bundle_path, - screenshot_paths=tuple(screenshot_paths), - ) - - -def extract_screenshots_from_local_bundle( - bundle_path: Path, - output_dir: Path, - *, - only_failures: bool = False, -) -> tuple[Path, ...]: - """Extract screenshots from a local .xcresult bundle (Xcode Cloud Mac path).""" - screenshots_dir = output_dir / "screenshots" - return tuple( - extract_attachments( - bundle_path, - screenshots_dir, - only_failures=only_failures, - ) - ) diff --git a/scripts/xcode_cloud/trigger.py b/scripts/xcode_cloud/trigger.py deleted file mode 100644 index 0140632..0000000 --- a/scripts/xcode_cloud/trigger.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Trigger Xcode Cloud builds and wait for completion.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from scripts.xcode_cloud.client import BuildRunStatus, XcodeCloudClient, XcodeCloudError - - -@dataclass(frozen=True) -class TriggerResult: - build_run_id: str - status: BuildRunStatus - - -def repository_id_for_workflow(client: XcodeCloudClient, workflow_id: str) -> str: - payload = client.get_workflow(workflow_id, include_repository=True) - included = payload.get("included", []) - for resource in included: - if resource.get("type") == "scmRepositories": - return resource["id"] - - relationships = payload.get("data", {}).get("relationships", {}) - repository = relationships.get("repository", {}).get("data") - if repository and repository.get("id"): - return repository["id"] - - raise XcodeCloudError(f"Could not resolve repository for workflow {workflow_id}") - - -def git_reference_id_for_branch( - client: XcodeCloudClient, - repository_id: str, - branch: str, -) -> str: - references = client.list_git_references(repository_id) - normalized = branch.removeprefix("refs/heads/") - for reference in references: - attributes = reference.get("attributes", {}) - if attributes.get("kind") != "BRANCH": - continue - name = attributes.get("name", "") - canonical = attributes.get("canonicalName", "") - if name == normalized or canonical == f"refs/heads/{normalized}": - return reference["id"] - - raise XcodeCloudError( - f"Could not find git reference for branch '{branch}' in repository {repository_id}" - ) - - -def trigger_build_run( - client: XcodeCloudClient, - workflow_id: str, - *, - git_reference_id: str | None = None, - branch: str | None = None, -) -> str: - if bool(git_reference_id) == bool(branch): - raise ValueError("Provide exactly one of git_reference_id or branch") - - resolved_reference_id = git_reference_id - if branch is not None: - repository_id = repository_id_for_workflow(client, workflow_id) - resolved_reference_id = git_reference_id_for_branch(client, repository_id, branch) - - assert resolved_reference_id is not None - payload = client.create_build_run(workflow_id, resolved_reference_id) - return payload["data"]["id"] - - -def trigger_and_wait( - client: XcodeCloudClient, - workflow_id: str, - *, - git_reference_id: str | None = None, - branch: str | None = None, - timeout_seconds: int = 3600, - poll_interval_seconds: int = 30, -) -> TriggerResult: - build_run_id = trigger_build_run( - client, - workflow_id, - git_reference_id=git_reference_id, - branch=branch, - ) - status = client.wait_for_build_run( - build_run_id, - timeout_seconds=timeout_seconds, - poll_interval_seconds=poll_interval_seconds, - ) - return TriggerResult(build_run_id=build_run_id, status=status)