Up

.github/scripts/fetch-trace-fixture-lfs.sh

@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=trace-fixture-lib.sh
+. "${script_dir}/trace-fixture-lib.sh"
+
 if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
   echo "usage: $0 <trace-case> [fixture-dir]" >&2
   exit 2
@@ -62,57 +66,6 @@ if [ "${case_name}" = "OpenRA" ]; then
   exit 0
 fi
 
-get_lfs_metadata() {
-  local file="$1"
-  local pointer
-  local expected_oid
-  local expected_size
-
-  if ! pointer="$(git show "HEAD:${file}" 2>/dev/null)"; then
-    echo "failed to read tracked fixture metadata: ${file}" >&2
-    return 1
-  fi
-  if ! grep -q '^version https://git-lfs.github.com/spec/v1$' <<< "${pointer}"; then
-    echo "tracked fixture is not a Git LFS pointer: ${file}" >&2
-    return 1
-  fi
-
-  expected_oid="$(awk '$1 == "oid" && $2 ~ /^sha256:/ { sub(/^sha256:/, "", $2); print $2 }' <<< "${pointer}")"
-  expected_size="$(awk '$1 == "size" { print $2 }' <<< "${pointer}")"
-  if ! [[ "${expected_oid}" =~ ^[0-9a-f]{64}$ ]] || ! [[ "${expected_size}" =~ ^[0-9]+$ ]]; then
-    echo "invalid Git LFS pointer metadata: ${file}" >&2
-    return 1
-  fi
-
-  printf '%s %s\n' "${expected_oid}" "${expected_size}"
-}
-
-verify_fixture_file() {
-  local downloaded_file="$1"
-  local display_name="$2"
-  local expected_oid="$3"
-  local expected_size="$4"
-  local actual_oid
-  local actual_size
-
-  if [ ! -f "${downloaded_file}" ]; then
-    echo "fixture file is missing: ${display_name}" >&2
-    return 1
-  fi
-
-  actual_size="$(wc -c < "${downloaded_file}" | tr -d '[:space:]')"
-  if [ "${actual_size}" != "${expected_size}" ]; then
-    echo "fixture size mismatch for ${display_name}: expected ${expected_size}, got ${actual_size}" >&2
-    return 1
-  fi
-
-  actual_oid="$(sha256sum "${downloaded_file}" | awk '{ print $1 }')"
-  if [ "${actual_oid}" != "${expected_oid}" ]; then
-    echo "fixture SHA-256 mismatch for ${display_name}: expected ${expected_oid}, got ${actual_oid}" >&2
-    return 1
-  fi
-}
-
 fetch_file_from_mirror() {
   local file="$1"
   local url="$2"

.github/scripts/trace-fixture-cache.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# Cache-side helper for trace fixtures.
+#
+#   key <case> [fixture-dir]     derive the actions/cache key and path list
+#   verify <case> [fixture-dir]  check restored fixtures against their pointers
+#   reset <case> [fixture-dir]   drop restored fixtures, leaving the pointers
+#
+# The cache key is content-addressed on the Git LFS pointer oids tracked at
+# HEAD, which are readable from a plain checkout without smudging. Fixture
+# content therefore maps 1:1 onto a key: unchanged content hits, changed
+# content is a new key and thus a miss, and the download path handles it. The
+# key deliberately carries no restore-keys prefix in the workflow - a fixture
+# that does not match the pointer exactly must never be restored.
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=trace-fixture-lib.sh
+. "${script_dir}/trace-fixture-lib.sh"
+
+# Bump when the key derivation changes in a way that must invalidate old
+# entries; the content digest alone would not notice a format change.
+key_schema="v1"
+
+if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
+  echo "usage: $0 <key|verify|reset> <trace-case> [fixture-dir]" >&2
+  exit 2
+fi
+
+command_name="$1"
+case_name="$2"
+fixture_dir="${3:-tools/trace_replay/fixtures}"
+python_bin="${PYTHON:-python3}"
+
+if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then
+  python_bin=python
+fi
+
+mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}")
+if [ "${#files[@]}" -eq 0 ]; then
+  echo "no fixture files declared for trace case: ${case_name}" >&2
+  exit 1
+fi
+
+# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to
+# stdout otherwise so the script stays runnable (and testable) off-CI.
+emit_output() {
+  local name="$1"
+  local value="$2"
+  if [ -n "${GITHUB_OUTPUT:-}" ]; then
+    if [[ "${value}" == *$'\n'* ]]; then
+      local delimiter="ghadelim_$(date +%s%N)_$$"
+      {
+        printf '%s<<%s\n' "${name}" "${delimiter}"
+        printf '%s\n' "${value}"
+        printf '%s\n' "${delimiter}"
+      } >> "${GITHUB_OUTPUT}"
+    else
+      printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}"
+    fi
+  fi
+  printf '%s=%s\n' "${name}" "${value}"
+}
+
+sanitize_case() {
+  printf '%s' "$1" | sed 's/[^A-Za-z0-9._-]/_/g'
+}
+
+case "${command_name}" in
+  key)
+    manifest=""
+    for file in "${files[@]}"; do
+      # A case whose fixtures are committed directly rather than through Git LFS
+      # (OpenRA) has no pointer oid to key on, and nothing to download either.
+      # Report it as uncacheable so the workflow skips the cache entirely.
+      if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then
+        echo "trace case ${case_name} is not stored in Git LFS; skipping fixture cache" >&2
+        emit_output "cacheable" "false"
+        emit_output "key" ""
+        exit 0
+      fi
+      read -r expected_oid expected_size <<< "${metadata}"
+      manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n'
+    done
+
+    digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')"
+    safe_case="$(sanitize_case "${case_name}")"
+
+    emit_output "cacheable" "true"
+    emit_output "key" "trace-fixture-${key_schema}-${safe_case}-${digest}"
+    emit_output "paths" "$(printf '%s\n' "${files[@]}")"
+    ;;
+
+  verify)
+    for file in "${files[@]}"; do
+      metadata="$(get_lfs_metadata "${file}")"
+      read -r expected_oid expected_size <<< "${metadata}"
+      verify_fixture_file "${file}" "${file}" "${expected_oid}" "${expected_size}"
+    done
+    echo "Verified ${#files[@]} fixture file(s) for ${case_name} against the tracked Git LFS pointers."
+    ;;
+
+  reset)
+    # Put the working tree back to the pointer files a fresh checkout would
+    # have, so that a rejected cache entry falls through to exactly the same
+    # download path a cache miss takes.
+    for file in "${files[@]}"; do
+      rm -f "${file}" "${file}.tmp"
+    done
+    git checkout -- "${files[@]}"
+    echo "Reset ${#files[@]} fixture file(s) for ${case_name} to their tracked Git LFS pointers."
+    ;;
+
+  *)
+    echo "unknown command: ${command_name}" >&2
+    exit 2
+    ;;
+esac

.github/scripts/trace-fixture-lib.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# Shared helpers for trace-fixture handling: reading the in-tree Git LFS pointer
+# metadata and verifying a fixture file against it. Sourced by
+# fetch-trace-fixture-lfs.sh (verify after download) and by
+# trace-fixture-cache.sh (cache key derivation and verify after cache restore),
+# so both paths agree on what a valid fixture is.
+
+# Reads the Git LFS pointer tracked at HEAD for a fixture path and prints
+# "<oid> <size>". Fails if the tracked blob is not a well-formed LFS pointer.
+get_lfs_metadata() {
+  local file="$1"
+  local pointer
+  local expected_oid
+  local expected_size
+
+  if ! pointer="$(git show "HEAD:${file}" 2>/dev/null)"; then
+    echo "failed to read tracked fixture metadata: ${file}" >&2
+    return 1
+  fi
+  if ! grep -q '^version https://git-lfs.github.com/spec/v1$' <<< "${pointer}"; then
+    echo "tracked fixture is not a Git LFS pointer: ${file}" >&2
+    return 1
+  fi
+
+  expected_oid="$(awk '$1 == "oid" && $2 ~ /^sha256:/ { sub(/^sha256:/, "", $2); print $2 }' <<< "${pointer}")"
+  expected_size="$(awk '$1 == "size" { print $2 }' <<< "${pointer}")"
+  if ! [[ "${expected_oid}" =~ ^[0-9a-f]{64}$ ]] || ! [[ "${expected_size}" =~ ^[0-9]+$ ]]; then
+    echo "invalid Git LFS pointer metadata: ${file}" >&2
+    return 1
+  fi
+
+  printf '%s %s\n' "${expected_oid}" "${expected_size}"
+}
+
+# Checks an on-disk fixture against the size and SHA-256 from its LFS pointer.
+verify_fixture_file() {
+  local downloaded_file="$1"
+  local display_name="$2"
+  local expected_oid="$3"
+  local expected_size="$4"
+  local actual_oid
+  local actual_size
+
+  if [ ! -f "${downloaded_file}" ]; then
+    echo "fixture file is missing: ${display_name}" >&2
+    return 1
+  fi
+
+  actual_size="$(wc -c < "${downloaded_file}" | tr -d '[:space:]')"
+  if [ "${actual_size}" != "${expected_size}" ]; then
+    echo "fixture size mismatch for ${display_name}: expected ${expected_size}, got ${actual_size}" >&2
+    return 1
+  fi
+
+  actual_oid="$(sha256sum "${downloaded_file}" | awk '{ print $1 }')"
+  if [ "${actual_oid}" != "${expected_oid}" ]; then
+    echo "fixture SHA-256 mismatch for ${display_name}: expected ${expected_oid}, got ${actual_oid}" >&2
+    return 1
+  fi
+}
+
+# Prints the fixture file paths of a trace case, one per line. Strips CR so the
+# result is usable when python emits CRLF (Git Bash on Windows).
+trace_fixture_files() {
+  local case_name="$1"
+  local fixture_dir="$2"
+  local python_bin="${3:-python3}"
+
+  "${python_bin}" tools/trace_replay/trace_cases.py \
+    --format fixture-files \
+    --case "${case_name}" \
+    --fixture-root "${fixture_dir}" | tr -d '\r'
+}

.github/workflows/apk.yml

@@ -201,9 +201,41 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v6
 
+      - name: Derive trace fixture cache key
+        id: fixture-key
+        run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}'
+
+      - name: Restore trace fixture cache
+        id: fixture-cache
+        if: steps.fixture-key.outputs.cacheable == 'true'
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ steps.fixture-key.outputs.paths }}
+          key: ${{ steps.fixture-key.outputs.key }}
+
+      - name: Verify restored trace fixture
+        id: fixture-verify
+        if: steps.fixture-cache.outputs.cache-hit == 'true'
+        run: |
+          if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then
+            echo "ok=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "ok=false" >> "$GITHUB_OUTPUT"
+            echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path"
+            bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}'
+          fi
+
       - name: Fetch trace fixture
+        if: steps.fixture-verify.outputs.ok != 'true'
         run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}'
 
+      - name: Save trace fixture cache
+        if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v5
+        with:
+          path: ${{ steps.fixture-key.outputs.paths }}
+          key: ${{ steps.fixture-key.outputs.key }}
+
       - name: Stage trace fixture
         run: |
           safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
@@ -411,6 +443,24 @@ jobs:
           run_retrace || retrace_status=$?
           if [ "${retrace_status}" -eq 75 ]; then
             echo "::warning::Android emulator infrastructure failed; restarting it and retrying this retrace once."
+            # Surface-lost is retried rather than failed, so it would otherwise
+            # be invisible. Report it per job - a healthy run prints nothing and
+            # a rate spike shows up as a row per affected case.
+            reason_file="android-retrace-result/infrastructure-failure-reason.txt"
+            surface_lost_retries=0
+            if [ -f "${reason_file}" ]; then
+              surface_lost_retries="$(grep -c 'angle-surface-lost' "${reason_file}" || true)"
+            fi
+            if [ "${surface_lost_retries}" -gt 0 ]; then
+              echo "surface-lost retries: ${surface_lost_retries} (${{ matrix.backend.name }}, ${{ matrix.case.name }})" \
+                >> "${GITHUB_STEP_SUMMARY}"
+            fi
+            # The restart truncates EMULATOR_LOG, and the attempt that lost the
+            # emulator is the one worth reading - the retry usually only shows
+            # the wreckage. Keep the first attempt's log before it is clobbered.
+            if [ -f "${EMULATOR_LOG}" ]; then
+              cp "${EMULATOR_LOG}" "${EMULATOR_LOG}.first-attempt" || true
+            fi
             sh android-plugin/run-avd-ci.sh stop \
               --avd-name "${AVD_NAME}" \
               --emulator-log "${EMULATOR_LOG}" \
@@ -450,6 +500,13 @@ jobs:
           if [ -f "${EMULATOR_LOG}" ]; then
             cp "${EMULATOR_LOG}" android-retrace-result/diagnostics/emulator.log
           fi
+          if [ -f "${EMULATOR_LOG}.first-attempt" ]; then
+            cp "${EMULATOR_LOG}.first-attempt" android-retrace-result/diagnostics/emulator-first-attempt.log
+          fi
+          # A vanished emulator looks identical whether the host OOM killer took
+          # qemu or the renderer faulted. These two say which.
+          free -h > android-retrace-result/diagnostics/host-memory.txt 2>&1 || true
+          sudo dmesg -T 2>/dev/null | tail -300 > android-retrace-result/diagnostics/host-dmesg.txt || true
 
       - name: Stop Emulator
         if: always()
@@ -531,22 +588,41 @@ jobs:
           )
 
           if ((${#failed_cases[@]})); then
-            echo "Retaining fixtures for failed retrace case(s):"
+            echo "Retaining fixtures and results for failed retrace case(s):"
             printf '  %s\n' "${!failed_cases[@]}"
           else
-            echo "All retrace jobs succeeded; no fixtures need to be retained."
+            echo "All retrace jobs succeeded; nothing needs to be retained."
           fi
 
           deleted=0
           retained=0
           while IFS=$'\t' read -r artifact_id artifact_name; do
+            keep=0
             if [[ "${artifact_name}" == MobileGL-trace-fixture-* ]]; then
               case_name="${artifact_name#MobileGL-trace-fixture-}"
               if [[ -v "failed_cases[${case_name}]" ]]; then
-                echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace."
-                ((retained += 1))
-                continue
+                keep=1
               fi
+            elif [[ "${artifact_name}" == MobileGL-android-retrace-result-* ]]; then
+              # The result artifact carries mobilegl.log, retrace.log, logcat,
+              # the emulator log and the actual/diff images - the only record of
+              # why a retrace failed. Its name ends in -<backend>-<case>, so a
+              # suffix match on the case name keeps both backends' results for a
+              # case that failed on either of them, which is what a comparison
+              # needs. The match is anchored at the end, so a case name that is a
+              # prefix of a longer one does not retain the longer one's results.
+              for case_name in "${!failed_cases[@]}"; do
+                if [[ "${artifact_name}" == *-"${case_name}" ]]; then
+                  keep=1
+                  break
+                fi
+              done
+            fi
+
+            if ((keep)); then
+              echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace."
+              ((retained += 1))
+              continue
             fi
 
             echo "Deleting ${artifact_name} (${artifact_id})"