diff --git a/.github/workflows/llup60-automated-replay.yml b/.github/workflows/llup60-automated-replay.yml new file mode 100644 index 000000000..691e160a4 --- /dev/null +++ b/.github/workflows/llup60-automated-replay.yml @@ -0,0 +1,363 @@ +name: LLUP-60 automated evidence replay + +on: + workflow_dispatch: + push: + branches: + - agent/llup60-automated-replay + paths: + - ".github/workflows/llup60-automated-replay.yml" + - "docs/workstreams/README.md" + - "docs/workstreams/llup-v0-3-automated-replay.md" + +permissions: + contents: read + +concurrency: + group: llup60-automated-replay-${{ github.ref }} + cancel-in-progress: true + +env: + RUNTIME_SOURCE_COMMIT: 59af48313b450d9cff13c7f43458c2e5e6560374 + EXPECTED_LLAMA_CPP_TAG: v0.3.0 + EXPECTED_LLAMA_CPP_REVISION: c1d0e7a004015f23bc0233470b747b596f29b264 + QWEN35_08B_SHA256: bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517 + QWEN35_08B_BYTES: "532517120" + QWEN35_2B_SHA256: aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223 + QWEN35_2B_BYTES: "1280835840" + +jobs: + provenance: + name: Exact-source and identity provenance + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + submodules: recursive + + - name: Verify stacked evidence scope + shell: bash + run: | + set -euo pipefail + git merge-base --is-ancestor "$RUNTIME_SOURCE_COMMIT" HEAD + changed="$(git diff --name-only "$RUNTIME_SOURCE_COMMIT"...HEAD)" + unexpected="$(printf '%s\n' "$changed" | grep -Ev '^(\.github/workflows/llup60-automated-replay\.yml|docs/workstreams/(README\.md|llup-v0-3-automated-replay\.md))$' || true)" + if [[ -n "$unexpected" ]]; then + echo "LLUP-60 evidence branch changes runtime/product files:" >&2 + printf '%s\n' "$unexpected" >&2 + exit 1 + fi + + - name: Verify authoritative backend identity + shell: bash + run: | + set -euo pipefail + ./scripts/verify-llama-cpp-pin.sh + python3 - <<'PY' + import json + import os + import subprocess + from pathlib import Path + + pin = json.loads(Path("backends/llama-cpp/llama-cpp-pin.json").read_text()) + expected_tag = os.environ["EXPECTED_LLAMA_CPP_TAG"] + expected_revision = os.environ["EXPECTED_LLAMA_CPP_REVISION"] + actual_submodule = subprocess.check_output( + ["git", "-C", "third_party/llama.cpp", "rev-parse", "HEAD"], text=True + ).strip() + if pin.get("tag") != expected_tag: + raise SystemExit(f"pin tag mismatch: {pin.get('tag')} != {expected_tag}") + if pin.get("commit") != expected_revision: + raise SystemExit(f"pin revision mismatch: {pin.get('commit')} != {expected_revision}") + if actual_submodule != expected_revision: + raise SystemExit(f"submodule mismatch: {actual_submodule} != {expected_revision}") + PY + bash scripts/test-verify-llama-cpp-pin.sh + + - name: Record provenance evidence + shell: bash + run: | + mkdir -p build/llup60 + python3 - <<'PY' + import json + import os + from pathlib import Path + + payload = { + "schemaVersion": 1, + "workstream": "LLUP-60A", + "runtimeSourceCommit": os.environ["RUNTIME_SOURCE_COMMIT"], + "evidenceHarnessCommit": os.environ["GITHUB_SHA"], + "llamaCppTag": os.environ["EXPECTED_LLAMA_CPP_TAG"], + "llamaCppRevision": os.environ["EXPECTED_LLAMA_CPP_REVISION"], + "identityGuard": "pass", + "realEnvironmentDeferred": True, + } + Path("build/llup60/provenance.json").write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + PY + + - uses: actions/upload-artifact@v7 + with: + name: llup60-provenance + path: build/llup60/provenance.json + if-no-files-found: error + retention-days: 14 + + native-host: + name: Native API and ownership replay + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Verify candidate pin + run: ./scripts/verify-llama-cpp-pin.sh + + - name: Build and run host-native suite + shell: bash + run: | + set -euo pipefail + cmake -S backends/llama-cpp/src/test-native -B build/llup60-native -DCMAKE_BUILD_TYPE=Release + cmake --build build/llup60-native --parallel 2 + ctest --test-dir build/llup60-native --output-on-failure + + jvm-contracts: + name: Q35 and runtime contract replay + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "17" + check-latest: false + + - uses: android-actions/setup-android@v3 + with: + packages: platform-tools + + - name: Install Android SDK components + shell: bash + run: | + sdkmanager "platforms;android-36" "build-tools;36.0.0" + + - uses: gradle/actions/setup-gradle@v6 + with: + cache-provider: basic + + - name: Replay Q35 capability and runtime contracts + shell: bash + run: | + chmod +x gradlew + ./gradlew --stacktrace :models:model-profile:test :core:runtime-core:test + + evaluation-evidence: + name: Evaluation and benchmark contract replay + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "17" + check-latest: false + + - uses: android-actions/setup-android@v3 + with: + packages: platform-tools + + - name: Install Android SDK components + shell: bash + run: | + sdkmanager "platforms;android-36" "build-tools;36.0.0" + + - uses: gradle/actions/setup-gradle@v6 + with: + cache-provider: basic + + - name: Replay evaluation and benchmark evidence contracts + shell: bash + run: | + chmod +x gradlew + ./gradlew --no-configuration-cache --stacktrace \ + :evaluation:contracts:testDebugUnitTest \ + :evaluation:datasets:testDebugUnitTest \ + :evaluation:evaluators:testDebugUnitTest \ + :evaluation:engine:testDebugUnitTest \ + :evaluation:runtime-adapter:testDebugUnitTest \ + :evaluation:comparison:testDebugUnitTest \ + :evaluation:persistence:testDebugUnitTest \ + :observability:contracts:testDebugUnitTest \ + :observability:benchmark-engine:testDebugUnitTest + + qwen35-reference: + name: Exact Qwen3.5 host compatibility replay + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Verify candidate pin + run: ./scripts/verify-llama-cpp-pin.sh + + - name: Build candidate llama.cpp smoke runner + shell: bash + run: | + set -euo pipefail + cmake -S third_party/llama.cpp -B build/llup60-qwen35 \ + -DLLAMA_CURL=OFF \ + -DGGML_NATIVE=OFF \ + -DGGML_OPENMP=OFF \ + -DLLAMA_BUILD_TOOLS=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DLLAMA_BUILD_EXAMPLES=ON + cmake --build build/llup60-qwen35 --target llama-simple -j2 + + - name: Download exact Qwen3.5 reference artifacts + shell: bash + run: | + set -euo pipefail + mkdir -p build/llup60-models + curl --fail --location --retry 3 --output build/llup60-models/qwen35-08b-q4-k-m.gguf \ + "https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/5aea8824cba95d22990acc6ea66c2c1909530650/Qwen3.5-0.8B-Q4_K_M.gguf?download=true" + curl --fail --location --retry 3 --output build/llup60-models/qwen35-2b-q4-k-m.gguf \ + "https://huggingface.co/unsloth/Qwen3.5-2B-GGUF/resolve/802854bfd388ed92748de119df31327962811548/Qwen3.5-2B-Q4_K_M.gguf?download=true" + + - name: Verify exact Qwen3.5 artifact identity + shell: bash + run: | + set -euo pipefail + echo "$QWEN35_08B_SHA256 build/llup60-models/qwen35-08b-q4-k-m.gguf" | sha256sum --check + echo "$QWEN35_2B_SHA256 build/llup60-models/qwen35-2b-q4-k-m.gguf" | sha256sum --check + test "$(stat -c%s build/llup60-models/qwen35-08b-q4-k-m.gguf)" = "$QWEN35_08B_BYTES" + test "$(stat -c%s build/llup60-models/qwen35-2b-q4-k-m.gguf)" = "$QWEN35_2B_BYTES" + + - name: Smoke load tokenize generate + shell: bash + run: | + set -euo pipefail + cli=build/llup60-qwen35/bin/llama-simple + for model in build/llup60-models/qwen35-08b-q4-k-m.gguf build/llup60-models/qwen35-2b-q4-k-m.gguf; do + timeout 120 "$cli" -m "$model" -ngl 0 -n 1 "Reply OK." >/dev/null + done + + - name: Record Qwen3.5 compatibility evidence + shell: bash + run: | + mkdir -p build/llup60 + python3 - <<'PY' + import json + import os + from pathlib import Path + + payload = { + "schemaVersion": 1, + "workstream": "LLUP-60A", + "runtimeSourceCommit": os.environ["RUNTIME_SOURCE_COMMIT"], + "evidenceHarnessCommit": os.environ["GITHUB_SHA"], + "llamaCppRevision": os.environ["EXPECTED_LLAMA_CPP_REVISION"], + "artifacts": { + "qwen35-08b-q4-k-m": { + "sha256": os.environ["QWEN35_08B_SHA256"], + "bytes": int(os.environ["QWEN35_08B_BYTES"]), + "hostSmoke": "pass", + }, + "qwen35-2b-q4-k-m": { + "sha256": os.environ["QWEN35_2B_SHA256"], + "bytes": int(os.environ["QWEN35_2B_BYTES"]), + "hostSmoke": "pass", + }, + }, + } + Path("build/llup60/qwen35-host-compatibility.json").write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + PY + + - uses: actions/upload-artifact@v7 + with: + name: llup60-qwen35-host-compatibility + path: build/llup60/qwen35-host-compatibility.json + if-no-files-found: error + retention-days: 14 + + summary: + name: Automated replay evidence summary + runs-on: ubuntu-latest + if: always() + needs: + - provenance + - native-host + - jvm-contracts + - evaluation-evidence + - qwen35-reference + env: + PROVENANCE_RESULT: ${{ needs.provenance.result }} + NATIVE_HOST_RESULT: ${{ needs.native-host.result }} + JVM_CONTRACTS_RESULT: ${{ needs.jvm-contracts.result }} + EVALUATION_EVIDENCE_RESULT: ${{ needs.evaluation-evidence.result }} + QWEN35_RESULT: ${{ needs.qwen35-reference.result }} + steps: + - name: Write replay manifest + shell: bash + run: | + set -euo pipefail + mkdir -p build/llup60 + python3 - <<'PY' + import json + import os + from pathlib import Path + + checks = { + "provenance": os.environ["PROVENANCE_RESULT"], + "nativeHost": os.environ["NATIVE_HOST_RESULT"], + "jvmContracts": os.environ["JVM_CONTRACTS_RESULT"], + "evaluationAndBenchmarkContracts": os.environ["EVALUATION_EVIDENCE_RESULT"], + "qwen35ExactHostCompatibility": os.environ["QWEN35_RESULT"], + } + payload = { + "schemaVersion": 1, + "workstream": "LLUP-60A", + "runtimeSourceCommit": os.environ["RUNTIME_SOURCE_COMMIT"], + "evidenceHarnessCommit": os.environ["GITHUB_SHA"], + "llamaCppTag": os.environ["EXPECTED_LLAMA_CPP_TAG"], + "llamaCppRevision": os.environ["EXPECTED_LLAMA_CPP_REVISION"], + "checks": checks, + "automatedReplayComplete": all(value == "success" for value in checks.values()), + "realEnvironmentDeferred": True, + "deferred": [ + "LLUP-50 same-device physical A/B", + "LLRT KV-cache physical evidence", + "LLRT evaluation-batch physical evidence", + "device-only Q35 performance/memory/lifecycle evidence", + "output-quality replay whose canonical owner requires representative-device execution", + ], + "promotionDecisionAllowed": False, + } + Path("build/llup60/automated-replay-manifest.json").write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + if not payload["automatedReplayComplete"]: + raise SystemExit("one or more automated replay lanes failed") + PY + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: llup60-automated-replay-manifest + path: build/llup60/automated-replay-manifest.json + if-no-files-found: error + retention-days: 14 diff --git a/docs/workstreams/README.md b/docs/workstreams/README.md index 1d336eaf0..f736e6c98 100644 --- a/docs/workstreams/README.md +++ b/docs/workstreams/README.md @@ -5,7 +5,7 @@ Document type: documentation-governance Owner: repository Canonical scope: documentation.workstreams Read when: creating, locating, updating or finalizing a bounded implementation workstream -Last reviewed: 2026-08-26 +Last reviewed: 2026-08-30 This directory is the canonical home for repository-level **temporary implementation workstreams** that must preserve dependency, sequencing or handoff state across pull requests or coding agents. @@ -16,6 +16,7 @@ A workstream belongs here only while it is active. It is not a second project-st - [`q35-runtime-qualification-wave.md`](q35-runtime-qualification-wave.md) — coordinates the temporary Qwen3.5 CPU runtime qualification wave, including measured-profile acceptance, lifecycle/memory evidence, representative-device gates and final review dependencies. - [`application-control-plane-ux.md`](application-control-plane-ux.md) — coordinates the Applications -> assigned use case -> preset-control UX implementation, including parallel UI/control-plane slices, revision-safe mutations, adaptive/accessibility convergence and effective two-APK evidence. - [`control-plane-state-reconciliation.md`](control-plane-state-reconciliation.md) — coordinates startup reconciliation of persisted mandatory built-in control-plane state, conservative upgrade repair, cross-surface consistency and the dependent physical upgrade/two-APK gates. +- [`llup-v0-3-automated-replay.md`](llup-v0-3-automated-replay.md) — tracks the temporary LLUP v0.3.0 exact-head automated replay, LLUP-50 package/device qualification boundary and LLUP-70 promotion readiness. ## Lifecycle diff --git a/docs/workstreams/llup-v0-3-automated-replay.md b/docs/workstreams/llup-v0-3-automated-replay.md new file mode 100644 index 000000000..7e59b5139 --- /dev/null +++ b/docs/workstreams/llup-v0-3-automated-replay.md @@ -0,0 +1,76 @@ +# LLUP v0.3.0 automated replay checkpoint + +Status: active +Document type: workstream-state +Owner: llama-cpp-runtime / runtime-memory +Canonical scope: workstream.llup-v0-3-automated-replay +Read when: refreshing LLUP-60 evidence, preparing LLUP-50 device qualification, or evaluating LLUP-70 promotion readiness +Last reviewed: 2026-08-30 + +Repository integrated state and blockers remain owned by [`../current-state.md`](../current-state.md); this file owns only the bounded LLUP qualification/replay sequence. + +## Frozen qualification identities + +- Control: `dev@80164329bbc41a00b75721e3d0524294c03fdb56`, llama.cpp `b9637` / `aedb2a5e9ca3d4064148bbb919e0ddc0c1b70ab3`. +- Candidate: `59af48313b450d9cff13c7f43458c2e5e6560374`, llama.cpp `v0.3.0` / `c1d0e7a004015f23bc0233470b747b596f29b264`. +- Control LLUP-50 evidence ref: `evidence/llup50-control@fcbefc7cd9af84de570da96d039582175dd1700b`. +- Candidate LLUP-50 evidence ref: `evidence/llup50-candidate@a2a050d9551db541bb4c6b152cba8623c782164d`. +- LLUP-50 runner branch: `agent/llup50-physical-runner@cefb893ae95cfd95339de4a24a955a652f0011e6`. + +The previous candidate `f796235f47899314b65a5abc95c998b396883c4b` was refreshed only because `dev` gained the canonical Android AAB packaging command. The llama.cpp runtime behavior did not change, but packaging identity is part of qualification, so relevant exact-head evidence is refreshed rather than reused. The automated work is refreshed for this candidate/base pair while `REAL_ENVIRONMENT` evidence remains intentionally deferred. + +## Current automated state + +PR #490 has repository-owned FULL evidence on exact candidate head `59af48313b450d9cff13c7f43458c2e5e6560374` against `dev@80164329bbc41a00b75721e3d0524294c03fdb56`. Authoritative run: `33332939707`. + +- LLUP-20: `PASS` / `AUTOMATED_PREFLIGHT_CONFIRMED`. +- LLUP-30: `PASS` / `AUTOMATED_PREFLIGHT_CONFIRMED`. +- LLUP-40: `PASS` / `AUTOMATED_PREFLIGHT_CONFIRMED`. +- LLUP-50: `PENDING_REAL_ENVIRONMENT`. +- LLUP-60: split into automated LLUP-60A and representative-device LLUP-60B; globally incomplete until both required parts pass. +- LLUP-70: `BLOCKED` until the physical block completes. +- MRES-10+: `BLOCKED` by LLUP-70. + +Previous LLUP-60A run `33332064271` and STRONG preflight `33332084155` remain historical provenance but are stale for the refreshed candidate/base pair. + +## LLUP-60A acceptance + +The repository workflow `.github/workflows/llup60-automated-replay.yml` must prove on the refreshed exact candidate: + +1. evidence-only branch scope; +2. authoritative pin/submodule/backend/Q35 identity consistency; +3. native backend ownership/API contracts; +4. Q35 model-profile and runtime-core JVM contracts; +5. evaluation and observability/benchmark contracts; +6. exact Qwen3.5 0.8B and 2B GGUF identity; +7. host load/tokenize/generate compatibility for both curated GGUFs; +8. machine-readable provenance separating runtime source SHA from evidence-harness SHA; +9. promotion disabled while representative-device evidence is pending. + +Automated replay does not replace output-quality, latency, memory, thermal or lifecycle evidence that genuinely requires representative Android execution. + +## LLUP-50 boundary + +The exact-APK runner remains isolated in draft PR #491. It consumes CI-built APKs, verifies package source revision and SHA-256, captures model-load/tuning/memory/cancellation/LOW_MEMORY/switch evidence and compares paired control/candidate results without inventing promotion thresholds. + +Before using a phone, repository-owned package automation must build exact-ref APK/manifests for the refreshed control and candidate evidence refs. Packaging is `REMOTE_AUTOMATED`; executing those artifacts on the representative phone is `REAL_ENVIRONMENT`. + +## Deferred REAL_ENVIRONMENT block + +- LLUP-50 same-device `b9637` versus `v0.3.0` A/B; +- LLRT KV-cache and evaluation-batch physical evidence; +- device-only Q35 load/performance/PSS/memory/thermal/cancellation/recovery/lifecycle evidence; +- representative-device resident-count, warm-idle and prepare-after-release recovery; +- output-quality evidence whose canonical execution path requires representative-device runtime execution. + +## Promotion-readiness boundary + +Classify the workstream as `WAITING_REAL_ENVIRONMENT` only when: + +- PR #490 remains unmerged and exact-head FULL green; +- PR #491 tooling validation is green on its refreshed head; +- LLUP-60A replay and STRONG preflight are green on the same refreshed evidence head; +- exact control/candidate package identities are available from repository-owned automation; +- no deterministic automation gap remains. + +Then execute LLUP-50 and LLUP-60B, re-check promotion FULL requirements, make the explicit LLUP-70 decision, and only after promotion authorization merge PR #490 and unblock MRES-10.