From 31aeef387b3e28cd429a6a640e09fc18b5e944e5 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Thu, 17 Sep 2026 09:41:30 -0400 Subject: [PATCH 01/46] Add portable cross-platform validation harness and remote runners --- .github/workflows/ci.yml | 18 + .github/workflows/validation_bundles.yml | 77 ++ .gitignore | 4 + doc/cross_platform_validation.md | 255 ++++ doc/cross_platform_validation_plan.md | 960 ++++++++++++++ doc/testing_matrix.md | 11 + example/chat_app/android/app/build.gradle.kts | 7 + .../MainActivityTest.java | 14 + example/chat_app/android/gradle.properties | 3 + .../integration_test/validation_test.dart | 32 + .../ios/Runner.xcodeproj/project.pbxproj | 8 +- .../chat_app/ios/RunnerTests/RunnerTests.m | 35 + .../ios/RunnerTests/RunnerTests.swift | 12 - .../chat_app/lib/validation/controller.dart | 156 +++ example/chat_app/lib/validation/host.dart | 34 + .../chat_app/lib/validation/host_native.dart | 78 ++ example/chat_app/lib/validation/host_web.dart | 104 ++ example/chat_app/lib/validation_main.dart | 146 +++ example/chat_app/pubspec.lock | 9 +- example/chat_app/pubspec.yaml | 4 +- .../test/validation_controller_test.dart | 86 ++ packages/llamadart_validation/README.md | 29 + .../analysis_options.yaml | 1 + .../assets/profiles/chat-gguf-cpu.json | 28 + .../assets/profiles/chat-gguf-cuda.json | 28 + .../assets/profiles/chat-gguf-metal.json | 28 + .../assets/profiles/chat-gguf-vulkan.json | 28 + .../assets/profiles/chat-litert-cpu.json | 29 + .../assets/profiles/chat-litert-gpu.json | 29 + .../assets/profiles/tiny-gguf-cpu.json | 32 + .../assets/profiles/tiny-gguf-cuda.json | 32 + .../assets/profiles/tiny-gguf-metal.json | 32 + .../assets/profiles/tiny-gguf-vulkan.json | 32 + packages/llamadart_validation/bin/report.dart | 23 + packages/llamadart_validation/bin/run.dart | 128 ++ packages/llamadart_validation/lib/io.dart | 177 +++ .../lib/llamadart_validation.dart | 6 + .../lib/src/manifest.dart | 212 ++++ .../lib/src/placement.dart | 72 ++ .../llamadart_validation/lib/src/report.dart | 304 +++++ .../llamadart_validation/lib/src/runner.dart | 642 ++++++++++ packages/llamadart_validation/pubspec.lock | 460 +++++++ packages/llamadart_validation/pubspec.yaml | 25 + .../schemas/event.schema.json | 91 ++ .../schemas/profile.schema.json | 90 ++ .../test/model_io_test.dart | 108 ++ .../test/validation_test.dart | 415 ++++++ scripts/build_chat_app_web.sh | 13 +- test/unit/tooling/prepare_workspace_test.dart | 1 + test/unit/tooling/validation_remote_test.dart | 527 ++++++++ tool/prepare_workspace.dart | 5 + tool/testing/run_local_e2e.dart | 32 + tool/testing/test_matrix.dart | 22 + tool/testing/validation.dart | 236 ++++ tool/testing/validation/bundle.dart | 469 +++++++ tool/testing/validation/collect.dart | 153 +++ tool/testing/validation/firebase.example.json | 13 + tool/testing/validation/gce.example.json | 22 + tool/testing/validation/process.dart | 103 ++ tool/testing/validation/process_host.py | 107 ++ tool/testing/validation/remote.dart | 1113 +++++++++++++++++ tool/testing/validation/run-remote.ps1 | 38 + tool/testing/validation/run-remote.sh | 36 + 63 files changed, 8005 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/validation_bundles.yml create mode 100644 doc/cross_platform_validation.md create mode 100644 doc/cross_platform_validation_plan.md create mode 100644 example/chat_app/android/app/src/androidTest/java/com/example/llamadart_chat_example/MainActivityTest.java create mode 100644 example/chat_app/integration_test/validation_test.dart create mode 100644 example/chat_app/ios/RunnerTests/RunnerTests.m delete mode 100644 example/chat_app/ios/RunnerTests/RunnerTests.swift create mode 100644 example/chat_app/lib/validation/controller.dart create mode 100644 example/chat_app/lib/validation/host.dart create mode 100644 example/chat_app/lib/validation/host_native.dart create mode 100644 example/chat_app/lib/validation/host_web.dart create mode 100644 example/chat_app/lib/validation_main.dart create mode 100644 example/chat_app/test/validation_controller_test.dart create mode 100644 packages/llamadart_validation/README.md create mode 100644 packages/llamadart_validation/analysis_options.yaml create mode 100644 packages/llamadart_validation/assets/profiles/chat-gguf-cpu.json create mode 100644 packages/llamadart_validation/assets/profiles/chat-gguf-cuda.json create mode 100644 packages/llamadart_validation/assets/profiles/chat-gguf-metal.json create mode 100644 packages/llamadart_validation/assets/profiles/chat-gguf-vulkan.json create mode 100644 packages/llamadart_validation/assets/profiles/chat-litert-cpu.json create mode 100644 packages/llamadart_validation/assets/profiles/chat-litert-gpu.json create mode 100644 packages/llamadart_validation/assets/profiles/tiny-gguf-cpu.json create mode 100644 packages/llamadart_validation/assets/profiles/tiny-gguf-cuda.json create mode 100644 packages/llamadart_validation/assets/profiles/tiny-gguf-metal.json create mode 100644 packages/llamadart_validation/assets/profiles/tiny-gguf-vulkan.json create mode 100644 packages/llamadart_validation/bin/report.dart create mode 100644 packages/llamadart_validation/bin/run.dart create mode 100644 packages/llamadart_validation/lib/io.dart create mode 100644 packages/llamadart_validation/lib/llamadart_validation.dart create mode 100644 packages/llamadart_validation/lib/src/manifest.dart create mode 100644 packages/llamadart_validation/lib/src/placement.dart create mode 100644 packages/llamadart_validation/lib/src/report.dart create mode 100644 packages/llamadart_validation/lib/src/runner.dart create mode 100644 packages/llamadart_validation/pubspec.lock create mode 100644 packages/llamadart_validation/pubspec.yaml create mode 100644 packages/llamadart_validation/schemas/event.schema.json create mode 100644 packages/llamadart_validation/schemas/profile.schema.json create mode 100644 packages/llamadart_validation/test/model_io_test.dart create mode 100644 packages/llamadart_validation/test/validation_test.dart create mode 100644 test/unit/tooling/validation_remote_test.dart create mode 100644 tool/testing/validation.dart create mode 100644 tool/testing/validation/bundle.dart create mode 100644 tool/testing/validation/collect.dart create mode 100644 tool/testing/validation/firebase.example.json create mode 100644 tool/testing/validation/gce.example.json create mode 100644 tool/testing/validation/process.dart create mode 100644 tool/testing/validation/process_host.py create mode 100644 tool/testing/validation/remote.dart create mode 100644 tool/testing/validation/run-remote.ps1 create mode 100644 tool/testing/validation/run-remote.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d965bad7..122eb4ef1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,24 @@ jobs: - name: Check Pana Score run: flutter pub global run pana . --exit-code-threshold 0 + validation-harness: + name: Validation harness (model-free) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: subosito/flutter-action@v2 + with: + flutter-version: 3.47.1 + channel: stable + cache: true + - run: dart pub get + working-directory: packages/llamadart_validation + - run: dart analyze --fatal-infos + working-directory: packages/llamadart_validation + - run: dart test + working-directory: packages/llamadart_validation + companion-packages: name: Companion Package (${{ matrix.package }}) runs-on: macos-latest diff --git a/.github/workflows/validation_bundles.yml b/.github/workflows/validation_bundles.yml new file mode 100644 index 000000000..c2ea190fa --- /dev/null +++ b/.github/workflows/validation_bundles.yml @@ -0,0 +1,77 @@ +name: Validation bundles + +on: + workflow_dispatch: + inputs: + profile: + description: Locked quick profile compiled into mobile and Web apps + type: choice + options: [tiny-gguf-cpu, tiny-gguf-metal, tiny-gguf-vulkan, tiny-gguf-cuda, chat-gguf-cpu, chat-gguf-metal, chat-gguf-vulkan, chat-gguf-cuda, chat-litert-cpu, chat-litert-gpu] + default: tiny-gguf-cpu + +permissions: + contents: read + +concurrency: + group: validation-bundles-${{ github.ref }} + cancel-in-progress: false + +jobs: + desktop: + name: Desktop ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2022, macos-15] + runs-on: ${{ matrix.os }} + timeout-minutes: 35 + steps: + - uses: actions/checkout@v6 + - uses: subosito/flutter-action@v2 + with: + flutter-version: 3.47.1 + channel: stable + cache: true + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - run: dart pub get + - run: dart run tool/testing/validation.dart build --target desktop --profile ${{ inputs.profile }} --out .dart_tool/validation/bundle + - name: Preserve executable permissions in transport + run: tar -czf .dart_tool/validation/bundle.tar.gz -C .dart_tool/validation bundle + - uses: actions/upload-artifact@v6 + with: + name: validation-desktop-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + path: .dart_tool/validation/bundle.tar.gz + retention-days: 7 + if-no-files-found: error + + app: + name: App ${{ matrix.target }} + strategy: + fail-fast: false + matrix: + target: [android, web, ios-inputs] + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + - uses: subosito/flutter-action@v2 + with: + flutter-version: 3.47.1 + channel: stable + cache: true + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '17' + - run: dart run tool/prepare_workspace.dart + - run: dart run tool/testing/validation.dart build --target ${{ matrix.target }} --profile ${{ inputs.profile }} --out .dart_tool/validation/bundle + - name: Preserve executable permissions in transport + run: tar -czf .dart_tool/validation/bundle.tar.gz -C .dart_tool/validation bundle + - uses: actions/upload-artifact@v6 + with: + name: validation-${{ matrix.target }}-${{ github.sha }} + path: .dart_tool/validation/bundle.tar.gz + retention-days: 7 + if-no-files-found: error diff --git a/.gitignore b/.gitignore index f45570e43..112846dbf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ build/ pubspec.lock !example/chat_app/pubspec.lock +!packages/llamadart_validation/pubspec.lock + +# Native runtime profiling output from optional model validation. +*.profraw .flutter-plugins .flutter-plugins-dependencies .metadata diff --git a/doc/cross_platform_validation.md b/doc/cross_platform_validation.md new file mode 100644 index 000000000..87c47d495 --- /dev/null +++ b/doc/cross_platform_validation.md @@ -0,0 +1,255 @@ +# Cross-platform validation runbook + +The first implementation supplies the quick core, shared reports, portable +builds, and explicit Firebase/GCE orchestration. It uses the current checkout's +runtime pins. It does not replace the release matrix, qualify unrun platforms, +or schedule paid work. The broader [validation plan](cross_platform_validation_plan.md) +retains the later feature and device milestones. + +## Source and generated artifacts + +- `packages/llamadart_validation/`: private Dart suite, locked profiles, desktop + runner/reporter, JSONL validation and JSON/JUnit/CSV/HTML rendering. +- `example/chat_app/lib/validation_main.dart`: interactive QA app. Run with + `flutter run -t lib/validation_main.dart` from `example/chat_app`. +- `example/chat_app/integration_test/validation_test.dart`: unattended entrypoint; + Android instrumentation and iOS XCTest invoke the same controller. +- `tool/testing/validation.dart`: build, local, report, plan, run, status, collect, + reconcile, cleanup. Provider helpers live beside it under `tool/testing/validation/`. +- `.github/workflows/validation_bundles.yml`: manual build-only workflow. No cloud + credentials, model runs, VM creation or Firebase submission in CI. +- `.dart_tool/validation/`: ignored local models, bundles and journals. Do not + commit weights, signing files, account configs, results or credentials. + +Use Flutter **3.47.1**, its Dart executable, and Python 3.10+ (Windows: `python`, +other hosts: `python3`). Android uses the installed Android SDK/JDK; XCTest needs +Xcode, configured local signing and the owned Mac. gcloud is needed only for +provider operations. Normal chat-app behavior is unchanged. +The `local` command records source/runtime provenance automatically. Plain +`flutter run` is useful for diagnostics, but an app without the builder's identity +defines cannot qualify. Dirty builds retain all assertion results and metrics; +qualification requires a clean committed source and known runtime identities. + +## Quick model profiles and cases + +| Profiles | Locked fixture | Use | +| --- | --- | --- | +| `tiny-gguf-{cpu,metal,vulkan,cuda}` | stories15M, 98,357,920 bytes | Packaging, native loading, lifecycle; throughput is a tiny-model diagnostic | +| `chat-gguf-{cpu,metal,vulkan,cuda}` | Qwen3.5 0.8B Q4_0, 563,036,064 bytes | GGUF chat, history and instruction checks | +| `chat-litert-{cpu,gpu}` | Qwen3 0.6B LiteRT-LM, 614,236,160 bytes | Native LiteRT public path; explicit GPU proof remains incomplete | + +Full revisions and SHA256 values live in profile JSON. The instruction GGUF is +[ggml-org's Q4_0 artifact](https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/blob/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q4_0.gguf), +so its results are a distinct cohort from the Q4_K_M candidate in the original plan. +Native LiteRT files cannot qualify LiteRT Web; that requires a Web model bundle. +The app reports this explicitly before downloading a native LiteRT fixture in Web. + +The quick inventory is C01 load/diagnostics, C02 Unicode tokenize/detokenize, +C03 raw generation, C04 hello/arithmetic and C06 multi-turn history for chat +fixtures, C08 cancellation/control/recovery, C09 dispose/new engine/reload, +C10 one-token limit, C12 missing-model rejection/recovery, and B01 one warmup +plus three measured generations. Independent assertion failures do not suppress +later metrics; load failure or timeout prevents unsafe later inference. + +Sampling is temperature 0, seed 1, top-k 40, top-p .9, repeat penalty 1.1, +context 1024, four threads, 32 generated tokens, thinking disabled, prompt reuse +disabled. Case overrides (one-token limit and 256-token cancellation) are recorded. +C08 compares the same prompt against an uncancelled control; the pinned WASM +delegate's explicit cancellation AbortError is also recorded as interruption evidence. Merely calling cancel +is not a pass; missing interruption evidence is NOT_RUN. Case timeouts are 60s; +disposal has a 10s bound and unresolved work cannot report successful cleanup. + +Prompts, regex expectations, exact output/thinking, ordered terminal case IDs, +configuration hashes, model hashes, source/runtime pins and environment all appear +in the journal. The raw tiny fixture does not claim chat capability. + +Thinking, tool calls, stop-marker fixtures, batching, expanded unsupported guards, +multimodal/speech/embedding packs, NPU, and full browser/device rotation remain +subsequent qualification work. Selecting `release` today keeps those additional +obligations visible as NOT_RUN and cannot pass as a release qualification. + +## Build and run + +```sh +# Model-free safety and report regression tests. +dart run tool/testing/run_local_e2e.dart --scenario validation-harness + +# Uses a shared verified model cache; creates a fresh output directory. +dart run tool/testing/validation.dart local --profile chat-litert-cpu + +# Build on each native OS/architecture; weights are downloaded when running. +dart run tool/testing/validation.dart build --target desktop --profile tiny-gguf-cpu --out .dart_tool/validation/bundles/desktop +dart run tool/testing/validation.dart build --target android --profile tiny-gguf-vulkan --out .dart_tool/validation/bundles/android +dart run tool/testing/validation.dart build --target web --out .dart_tool/validation/bundles/web +dart run tool/testing/validation.dart build --target ios-inputs --out .dart_tool/validation/bundles/ios-inputs +# On the owned Mac with existing local Xcode signing configured: +dart run tool/testing/validation.dart build --target ios --profile tiny-gguf-cpu --out .dart_tool/validation/bundles/ios +``` + +Desktop bundles include `bin/llamadart-validate`, `bin/llamadart-report`, native +code assets, profiles and remote wrappers. Run outside the repository: + +```sh +bin/llamadart-validate --profile tiny-gguf-cpu --environment-file environment.json --out results +bin/llamadart-report results +# Explicit GGUF accelerator evidence can be derived from that run's native log: +bin/llamadart-report results --native-log results/stderr.log +``` + +Windows executables have `.exe`. Keep the complete bundle layout. GitHub artifacts +contain `bundle.tar.gz` so executable permissions survive download; extract it +before use. Bundle manifests enumerate every shipped byte and record build OS/ABI, +SDKs, source commit/dirty state, native pins and the compiled mobile profile. +Resolved suite/app dependency locks are included as provenance too. +Remote dispatch rejects changed, dirty, wrong-OS or wrong-profile bundles. + +Android produces `qa-app.apk` for interactive installation and a matched +`app.apk`/`test.apk` instrumentation pair. These are Debug builds, explicitly +labelled; do not compare their TPS to Release builds. Both native runtimes are +bundled, making APKs larger than a single-runtime deployment. + +`ios-inputs` contains committed source, no signing credentials. `ios` invokes +`build-for-testing` and packages `tests.zip`; signing must already be configured. The unique existing project team is applied +to the test target too; use `--team TEAMID` when selecting another configured team. +iOS execution and attachment retrieval still need a separately approved device +run. Web builds use the maintained bridge-staging script. Serve with isolation +headers, for example: + +```sh +python3 tool/testing/serve_static_with_headers.py --directory .dart_tool/validation/bundles/web --port 7367 +``` + +## Firebase setup, submission and collection + +Use an explicitly selected **unbilled Spark project** and account. Preflight checks +billing is disabled and the selected physical model/OS exists in the live catalog. +Pick one row per submission; never use an implicit device matrix. Check available +physical quota in Firebase before creating a receipt. Receipt evidence expires +after 15 minutes. The local journal permits at most four physical submissions in +a rolling 24 hours; other clients still share the provider's quota. + +Copy `tool/testing/validation/firebase.example.json` to an ignored local path and +fill account, project, exact device/OS and a freshly checked quota receipt. + +```sh +dart run tool/testing/validation.dart plan --target firebase-android --config .dart_tool/validation/firebase.json --bundle .dart_tool/validation/bundles/android --out .dart_tool/validation/plan.json +dart run tool/testing/validation.dart run --plan .dart_tool/validation/plan.json +``` + +Use `firebase-ios` with the signed `ios` bundle. Plan creation is local/read-only; +`run` rechecks identity, budget evidence and live provider state before submission. +The CLI uses explicit `--project` and `--account`, one 20-minute execution, no +flaky retries and no video. It records the matrix ID, polls terminal state, copies +the default Test Lab results, then verifies completion or cancellation. It does +not enable billing or create a custom result bucket. + +Android pulls external app result files with complete console JSONL as fallback. +iOS attaches bounded result files to XCTest; collection exports `.xcresult` +attachments on macOS. Missing, truncated or conflicting evidence remains incomplete. +Physical device export/crash behavior is an explicit live-qualification step; +a build and fake-provider tests alone do not prove it. + +## GCE setup and teardown + +Run the GCE controller from the owned Mac or Linux. Windows is supported as the +CUDA test guest; Windows-hosted GCE orchestration is not yet qualified. + +Copy `tool/testing/validation/gce.example.json` to an ignored path. Configure an +immutable **GPU-ready image**, exact expected driver, compatible machine/accelerator, +zone, an existing network and a targeted IAP TCP/22 firewall tag. Linux needs +Python, `timeout` and `sha256sum`; Windows needs Google-supported SSH and PowerShell. +This initial adapter consumes an already qualified image; it does not install GPU +drivers or change project networking. Check quotas and image/machine compatibility +before the first approved run. Instance readiness rechecks the actual driver. + +The GCE profile must explicitly select GGUF CUDA and the uploaded desktop bundle +must be built for the destination x64 OS. Each run requires fresh evidence that +promotional credit applies to **all** expected costs, has at least twice the +estimated run cost available, and expires at least two hours after dispatch. +The receipt is an operator-verified billing-console record, not an automatic +credit API lookup. A budget alert is not a guarantee against charges. + +```sh +dart run tool/testing/validation.dart plan --target gce-linux-cuda --profile tiny-gguf-cuda --config .dart_tool/validation/gce.json --bundle .dart_tool/validation/bundles/linux --out .dart_tool/validation/plan.json +dart run tool/testing/validation.dart run --plan .dart_tool/validation/plan.json +``` + +Use `gce-windows-cuda` with a Windows x64 bundle and Windows-ready image. The +adapter records intent before creating one VM, requires provider-confirmed deletion +after 60 minutes, persists instance/disk IDs before setup, verifies the driver, +uploads/checks the bundle over IAP, runs under a 20-minute watchdog, collects +results, deletes the owned VM/boot disk, and reads the inventories back. +It creates no reusable volumes, static IPs or service accounts. An ephemeral +public IP provides outbound model download access and belongs in the cost estimate. + +Deletion is the default cleanup. **Stopped VMs can retain charged disks**; stopping +also clears the runtime deadline. Provider deletion deadlines and local finally +cleanup are complementary, not a zero-cost guarantee. If identity is uncertain, +cleanup refuses to delete a potentially unrelated resource, records UNKNOWN and +blocks another run. No cloud run is permitted under the $0-out-of-pocket policy +when applicable credit cannot be verified. + +## Recovery and evidence + +```sh +dart run tool/testing/validation.dart status --run-id qa-EXACT-ID +dart run tool/testing/validation.dart collect --run-id qa-EXACT-ID +dart run tool/testing/validation.dart cleanup --run-id qa-EXACT-ID +# Only after finding the exact ID in the provider console: +dart run tool/testing/validation.dart reconcile --run-id qa-EXACT-ID --remote-id EXACT-PROVIDER-ID +``` + +Use the ID printed in the saved plan (the placeholder above is not a valid ID). +OS locks serialize provider operations and release if the controlling process dies. +SIGINT requests cancellation; cleanup still runs after the current bounded provider +operation. Reusing a run ID cannot submit twice. A missing creation/submission +response is UNKNOWN, never permission to retry. If no remote ID was received, +inspect the provider console using the exact project/run label, preserve the +journal, and use `reconcile` before recovery; do not delete the journal to bypass +the blocker. Reconciliation checks the Firebase project/matrix/run label, or the +GCE numeric instance ID, zone, label and owned automatic boot disk. It cannot +replace an established identity and does not erase the original failure. Then +run `status`, `collect` and `cleanup`. Recovery commands never submit a replacement. + +`orchestration.json` stores phase, exact IDs, provider state and collection state; +`cleanup.json` stores verified/unknown cleanup. `remote-summary.json` is the +combined verdict: provider success **and** validated Dart assertions/provenance +**and** complete retrieval **and** verified cleanup. Test-only report success +cannot override provider failure or unresolved infrastructure. + +Each run exports `events.jsonl`, `manifest.json`, `results.json`, `junit.xml`, +`samples.csv`, and `summary.html`. HTML has case status/output, native decode TPS, +estimated visible-output TPS, TTFA, and median/min/max for three measured samples. +Warmup is retained but excluded from these comparisons. Missing counters are null; +chunks are never called tokens. Backend-native decode timing and retokenized +wall-time estimates stay in separate series. Three samples are informational, +not a performance regression gate or a cross-device ranking. + +Explicit CPU rows reject contradictory GPU diagnostics. GGUF accelerator reports +require matching backend diagnostics plus positive native tensor offload and +compute allocations for all three successful loads. Device presence or requested +GPU layers alone does not qualify execution. LiteRT GPU/NPU and browser accelerator +proof still require their own qualified evidence adapters; they remain incomplete. + +Normal model runs are opt-in. Model-free suite/provider tests run in CI. Before +claiming another platform qualified, attach the exact commit, model/backend, +command, provider/device identity, combined verdict, cleanup and native evidence. + +## Initial owned-machine observations (2026-09-17) + +The portable macOS arm64 bundle ran outside the repository. Tiny GGUF CPU/Metal +completed inference/lifecycle/TPS, while native Unicode corruption remains tracked +in [#511](https://github.com/leehack/llamadart/issues/511). The tiny model's expected +SentencePiece leading space is an explicit fixture normalization; corrupted +Unicode is still a failure. A Web/WASM UI run preserved Unicode and proved +cancellation through the pinned delegate's explicit AbortError and successful +recovery. The CPU identity check recognizes the pinned WASM name only with zero +GPU layers and its CPU core metadata. +Web native-token counters were unavailable, so C10 stayed NOT_RUN. + +Qwen3.5 GGUF also exposed native Unicode corruption and returned `Cedar17` for the +case-sensitive `cedar17` history oracle. LiteRT CPU ran 13/14 cases successfully +but answered `2` for `2 + 2`; retain the semantic failure and relate it to +[#509](https://github.com/leehack/llamadart/issues/509), without inferring the same +cause before a native reference comparison. These runs are diagnostics, not +release qualification or stable performance baselines. diff --git a/doc/cross_platform_validation_plan.md b/doc/cross_platform_validation_plan.md new file mode 100644 index 000000000..d5589caa4 --- /dev/null +++ b/doc/cross_platform_validation_plan.md @@ -0,0 +1,960 @@ +# Lightweight cross-platform validation plan + +Status: **initial quick-core harness implemented; broader qualification remains planned** +(2026-09-17). The maintainer authorized implementation after the earlier deferral. +The implementation starts from current merged main and preserves its runtime pins; +the separate release task still owns pending runtime changes. See the +[implementation runbook](cross_platform_validation.md) for available commands, +actual bundles, verified behavior and remaining qualification work. Pilot and +catalog observations below are dated 2026-09-16; proposed later coverage is not +an assertion that those rows now pass. + +The objective is a small, repeatable test of the **public llamadart package**, +including native-library packaging, model routing and application lifecycle. +Use owned hardware and free CI for routine checks, then rotate selected Firebase +devices for hardware and OS gaps. Keep the operating budget at **$0 out of +pocket**, even after promotional GCP credit expires. + +## 1. Deliverables and boundaries + +One private shared Dart suite supplies cases, model manifests, assertions and +reporting. A small desktop CLI and thin Flutter test app invoke that suite. +The Flutter app supports local interactive runs and unattended integration tests; +Android instrumentation and iOS XCTest wrap the same cases for Firebase. +The app shows selected model/backend, progress, cancellation, result and export. + +CI produces versioned QA bundles with checksums and provenance: desktop CLI +bundles including native code assets, Android app/test APKs, iOS build inputs, +and a deployable Web bundle with its pinned runtime assets. Build each native +target on a supported host/toolchain; do not assume one host cross-compiles all +targets. Initially build/sign physical-iOS XCTest bundles on the owned Mac; +do not add signing credentials to CI as part of this plan. + +Register maintained scenarios in `tool/testing/run_local_e2e.dart` and +`tool/testing/test_matrix.dart`; classify any new example/companion with +`tool/prepare_workspace.dart`. Reuse the existing feature smokes and benchmark +helpers. Do not create a parallel collection of unregistered repro scripts. +Tests call public APIs; direct native/upstream calls are **diagnostic controls** +and cannot substitute for a passing public-package case. + +Use three selections from one case catalog; do not maintain three suites: + +| Selection | Contents | When | +| --- | --- | --- | +| Quick core | Packaging, Unicode, raw/chat stream, history, cancellation/recovery, one reload, token limit and one timing series per representative model/backend | First device qualification and relevant runtime changes; inference target under five minutes, to be measured | +| Change-focused | Quick core plus the thinking/tools/stop/batching or feature-pack cases affected by the change | Relevant PRs; run expensive positive cases on selected devices, cheap negative contracts in CI | +| Release selection | All applicable C01–C12 subcases, affected-family packs and required platform/packaging rows | Release/native-pin qualification; schedule across quota days | + +Keep the existing case IDs, expected results and release coverage when reducing +a quick run. Report omitted cases as NOT_RUN with the selection reason. Large +multimodal/speech downloads remain feature packs. The Firebase four-device core +rotation is a useful subset of release evidence, not the full release selection. + +The first implementation milestone is deliberately small: manifests, shared +cases, existing CLI/mobile/Web adapters, reliable result capture and a basic +offline report. Reuse the chat app for Flutter/Apple companion packaging checks; +do not require new desktop Flutter UIs in addition to CLI bundles. Advanced +trend charts, model conversion and NPU packaging are subsequent milestones and +must not block a usable CPU/GPU harness. + +### Proposed repository layout + +Keep the suite in the **llamadart repository**, next to the package it validates. +Use one private Dart package named `llamadart_validation`; the paths below are +planned, not implemented: + +```text +packages/llamadart_validation/ + pubspec.yaml # publish_to: none; depends on local llamadart + lib/llamadart_validation.dart # shared runner and result contract + lib/src/cases/ # C01-C12 and optional feature packs + lib/src/manifest/ # model/profile validation and selection + lib/src/results/ # events, assertions and metric definitions + assets/ # pinned manifests, prompts, small media fixtures + schemas/ # manifest, event and result JSON schemas + bin/run.dart # desktop CLI; native host adapter + bin/report.dart # single host-side JSON/JUnit/CSV/HTML exporter + test/ # fast tests of selection, assertions and reports + +example/chat_app/ + lib/validation_main.dart # dedicated interactive QA entry point + lib/validation/ # thin Flutter/device/browser adapters + integration_test/validation_test.dart # unattended entry point, same suite + test_driver/integration_test.dart # reuse existing integration-test driver + android/ # app/test APK instrumentation wiring + ios/ # physical-device XCTest wiring + +tool/testing/validation.dart # build/plan/run/status/collect/cleanup CLI +tool/testing/validation/ # GCE/Firebase adapters and VM bootstrap inputs +.github/workflows/validation_bundles.yml # host-specific QA builds and artifacts +doc/cross_platform_validation_plan.md # scope, matrix and reporting contract +``` + +The shared `lib/` stays usable from Dart and Flutter Web: platform adapters +provide model/fixture access, storage and device diagnostics. Keep filesystem, +process and Flutter imports in their host adapters. The desktop CLI and Flutter +entry points depend on the shared package, which calls the public `llamadart` +API; the core package has no dependency on the suite. Do not add QA exports to +`lib/llamadart.dart` or QA commands to the user-facing CLI example. + +Prompts, model hashes, inference profiles and small licensed fixtures have one +checked-in source under `assets/`. Bundle preparation stages the selected inputs +for desktop, Flutter and Web; adapters resolve them through the same manifest +IDs without assuming a checkout or working directory. Model weights remain +outside Git. Use `.dart_tool/validation/model-cache//` for the local +cache, `.dart_tool/validation/bundles//` for build outputs and +`.dart_tool/validation/runs//` for collected events, logs and reports. +Device adapters write to their sandbox and export into that run directory; +browser adapters provide download/export. These generated paths stay ignored. + +Register the private package in `tool/prepare_workspace.dart` and give it +explicit analyze/test coverage, since root analysis currently excludes +`packages/**`. Register orchestration selections in the existing E2E runner and +testing matrix. Keep existing core unit/integration/E2E tests in place; migrate +overlapping smoke logic incrementally once the shared cases preserve its checks. +Extend the current Web build script to accept the QA entry point while retaining +its runtime-asset staging and validation. The bundle workflow builds artifacts; +Firebase submission remains an explicit invocation with the quota preflight. + +Native runtime/build fixes remain in their owning sibling repositories; this +package owns public-Dart validation and its adapters. The earlier Firebase +pilot under `.dart_tool/firebase_pilot/20260916/` remains historical evidence, +not the maintained suite location. + +### Where to get the built apps + +The planned download location is **llamadart → GitHub Actions → Validation +Bundles → selected run → Artifacts**, produced by +`.github/workflows/validation_bundles.yml`. Name each artifact +`llamadart-validation---` so downloads identify their +platform, backend profile and source revision. This workflow and its artifacts +do not exist yet; this is the distribution contract for implementation. + +| Target | Download / runnable output | +| --- | --- | +| Android arm64 | Installable QA APK and its matching instrumentation test APK for unattended/Firebase runs | +| Windows x64/arm64, when buildable | CLI bundle containing `llamadart-validate.exe`, required native libraries and run instructions | +| Linux x64/arm64, when buildable | CLI bundle containing `llamadart-validate`, required native libraries and run instructions | +| macOS arm64/x64, when buildable | CLI bundle containing `llamadart-validate`, required native libraries and run instructions | +| Web | Deployable QA site bundle with pinned runtime assets; serve it with the required headers | +| iOS/iPadOS | CI build inputs initially; build/sign the physical-device app and XCTest bundle on the owned Mac | + +Local builds stage the same outputs under +`.dart_tool/validation/bundles////`. Each bundle +includes its manifest, checksums, selected small fixtures and model-fetch/run +instructions; model weights are separate. Keep CI artifacts on bounded retention +and regenerate from the recorded revision and pinned inputs when needed. No +separate download server or automatic GitHub Release publication is required. + +The interactive app source stays in `example/chat_app/`, using the dedicated +`lib/validation_main.dart` build target. Desktop v1 is the CLI from +`packages/llamadart_validation/bin/run.dart`. Produce target-specific builds on +supported toolchains; a successful build remains separate from running and +qualifying it on actual hardware. + +## 2. Pilot findings and tracked work + +Pilot source was main `a5df1c4fcbb1766d26efb5b1d9becda191df89a3`, Flutter +3.47.1 / Dart 3.13.1, llama.cpp artifact `v0.4.0`, LiteRT-LM `v0.17.0-3`. +It ran actual Flutter/llamadart code on physical devices, not a model-only service. + +| Path | Galaxy S24 SC-51E, API 36, Debug | iPhone 16 Pro, actual iOS 18.3.2, Release | +| --- | --- | --- | +| GGUF CPU | App records passed before a later GPU crash; overall execution failed | Passed | +| GGUF GPU | Vulkan SIGSEGV during compute-pipeline compilation | Metal passed | +| LiteRT CPU | Answer `2` instead of `4` | Answer `2` instead of `4` | +| LiteRT GPU | Incoherent mixed-language output; native WebGPU/Vulkan adapter identified | Answer `2` instead of `4`; Metal identified | + +Track the distinct findings: + +- [llamadart-native #79](https://github.com/leehack/llamadart-native/issues/79): + Galaxy S24 GGUF Vulkan crash. The vendor compiler appears in the stack; + originating driver/runtime/artifact/integration ownership remains unproven. +- [llamadart #509](https://github.com/leehack/llamadart/issues/509): shared + LiteRT Qwen3 arithmetic failure. A wrong small-model answer alone does not + establish a Dart regression; compare identical native and model-reference runs. +- [litert-lm-native #51](https://github.com/leehack/litert-lm-native/issues/51): + Android GPU output divergence. OpenCL was unavailable, then WebGPU selected an + Adreno 750 Vulkan adapter. The fallback is evidence, not an established cause. + +Existing [native tokenizer #48](https://github.com/leehack/litert-lm-native/issues/48), +[desktop empty-chat #505](https://github.com/leehack/llamadart/issues/505), +[desktop LiteRT GPU #506](https://github.com/leehack/llamadart/issues/506), +[Windows CUDA helper #504](https://github.com/leehack/llamadart/issues/504), and +[older Android CPU qualification #476](https://github.com/leehack/llamadart/issues/476) +remain separate unless investigation establishes a common cause. + +An initial pilot-wrapper mistake set LiteRT `numberOfThreadsBatch=4`; the corrected +runs used zero. Those initial typed rejections are **not runtime bugs**. +The pilot collected no LiteRT TPS because its arithmetic assertion ran before +timing. Android Debug and iOS Release results are not a performance comparison. +These are historical observations on the named commit/artifacts. The three pilot +issues were still open when checked on 2026-09-17; neither an issue's state nor +a different commit's passing result establishes the current candidate's outcome. + +## 3. Platform and backend coverage + +The [support matrix](../website/docs/platforms/support-matrix.md) and capability +probes remain authoritative. The table below is a test selection plan, not a +promise that every backend/model combination works. Reconcile it with the +active merged pins before qualification. A published library, successful build, +or selectable backend is not inference qualification. + +| Target | GGUF profiles | LiteRT profiles | Where / coverage limit | +| --- | --- | --- | --- | +| Android arm64 physical | CPU, Vulkan; OpenCL in targeted pack | CPU, GPU; NPU only with matched model/dispatch libraries | Owned Pixel 9 Pro plus Firebase rotation | +| Android arm64 virtual | CPU, install/load, 4K/16K page-size packaging | CPU where artifact supports it; explicit unsupported cases | Free Firebase virtual quota; no physical GPU/ISA qualification | +| Android x64 emulator | CPU; Vulkan and OpenCL targeted if actually exposed | CPU, GPU only with backend proof | Local/free CI emulator; unavailable GPU profiles remain NOT_RUN; current Firebase virtual catalog is arm64 | +| iOS arm64 physical | CPU, Metal | CPU, GPU | Owned iPad and Firebase iPhones | +| iOS arm64 simulator | CPU, available Metal path separately labelled simulator | CPU, available GPU path separately labelled simulator | Owned Mac; does not qualify physical-device drivers or memory | +| iOS x86_64 simulator | CPU / available Metal | Negative packaging contract: no LiteRT artifact | Intel host if available, otherwise NOT_RUN | +| macOS arm64 | CPU, Metal | CPU, GPU | Owned Mac if matching architecture; record actual hardware | +| macOS x64 | CPU, available Metal | CPU; GPU unsupported by documented x64 bundle | Available free runner or Intel host; GPU gaps explicit | +| Linux x64 | CPU; CUDA, Vulkan, HIP/BLAS targeted | CPU; GPU unsupported until qualified/enabled | Free CI CPU; GPU only on available hardware or separately verified credit-covered VM | +| Linux arm64 | CPU; Vulkan/BLAS targeted | CPU | Available arm64 runner/hardware; no x64 emulation as arm64 proof | +| Windows x64 | CPU; CUDA, Vulkan/BLAS targeted | CPU; GPU unsupported until qualified/enabled | Free CI CPU, accessible Windows hardware; CUDA requires actual NVIDIA GPU | +| Windows arm64 | CPU, Vulkan/BLAS where available | Negative artifact contract unless support is added | Hook/build coverage plus explicit runtime gap without hardware | +| Web, Chrome | WASM CPU and WebGPU separately | Browser CPU/GPU with Web-compatible model | CI WASM; owned Mac/Pixel real browser GPU | +| Web, Safari/iPadOS | WASM and WebGPU when exposed | Browser runtime capability-dependent | Owned Apple devices; verify browser/version, secure context and memory limits | +| Web, Firefox | WASM plus explicit capability checks | Only what active browser runtime exposes | Compatibility lane; unavailable WebGPU is not a GPU pass | + +Quick core applies to each selected supported runtime row. Release selection +accounts for the full supported matrix; unavailable hardware remains an explicit +gap. Feature positives require a compatible fixture and runtime. Exhaustive +option/unsupported guards run in cheap unit/integration lanes; device profiles +retain backend probes and one error/recovery check, expanding when those paths +change. Unexpected unsupported behavior in a promised supported row is a failure. + +Backend evidence must include requested selector, effective backend name, +loaded native modules, actual adapter/driver and offload/delegate diagnostics. +An echoed `gpuLayers=999`, `availableBackends` list, or GPU preference is +insufficient. An automatic CPU fallback can pass an **auto** policy case but +cannot pass the explicit GPU row. Missing proof leaves GPU qualification +incomplete. + +## 4. Models and immutable manifests + +Do not multiply every model by every device. Run representative core models on +the device rotation; exercise affected model families and feature packs on +selected capable owned devices. A custom manifest can add a user's model without +silently replacing a failing standard fixture. + +| Model ID | Proposed artifact | Purpose / scheduling | +| --- | --- | --- | +| `tiny-gguf` | stories15M.gguf, 98.36 MB | Packaging, raw stream, backend/lifecycle; not an instruction-quality oracle | +| `chat-gguf` | Qwen3.5-0.8B Q4_K_M GGUF | Representative chat/thinking/tools; qualify template and semantic fixtures first | +| `dense-state` | Qwen2.5-0.5B Q4_K_M GGUF | Dense context/state/prompt-reuse checks, targeted pack | +| `chat-litert` | Qwen3-0.6B.litertlm, 614.24 MB | Native LiteRT core plus retained arithmetic/tokenizer diagnostics | +| `npu-qualcomm-sm8650` | Gemma 3 1B IT, Qualcomm SM8650-specific LiteRT bundle, about 658 MB | Optional S24 NPU qualification; separate artifact from CPU/GPU models | +| `npu-tensor-g5` | Gemma 3 1B IT, Tensor G5-specific LiteRT bundle, about 1.7 GB | Optional Pixel 10 NPU qualification; separate vendor runtime and artifact | +| `tools-litert` | FunctionGemma 270M compatible LiteRT bundle | Optional native tool fixtures; verify actual template/tool support first | +| `gemma-gguf` | Gemma 4 E2B GGUF + matching projector, about 4 GB total | Vision/audio/thinking/tools and large-memory coverage | +| `gemma-litert-native` | Native Gemma 4 E2B LiteRT bundle, about 2.6 GB | Native multimodal pack | +| `gemma-litert-web` | Web-compatible Gemma 4 bundle, about 2 GB | LiteRT Web single-turn lane; never substitute the native bundle | +| `embedding` | EmbeddingGemma 300M Q8 GGUF, about 334 MB | Embeddings and batch consistency | +| `asr-gguf` | Qwen3-ASR GGUF + matching projector, about 1 GB | File transcription; Web WAV-only where supported | +| `asr-litert` | Moonshine tiny INT8 + tokenizer, about 54 MB | Native CPU streaming ASR | +| `tts-gguf` | Qwen3-TTS GGUF + matching projector, about 1.5 GB | Experimental typed synthesis and playback/export | + +Except the two pilot fixtures below, artifacts/sizes are **selection candidates**, +not locked or reference-qualified test inputs. Before a row is executable, resolve +the exact repository revision, filename, byte size, SHA256, format, quantization, +tokenizer/template identity, license/access requirements, companion hashes, +supported feature flags and measured memory envelope. Reject a manifest with +missing hashes or floating revisions. Do not download gated assets without access. + +Pinned pilot fixtures: + +| ID | Repository / revision / filename | SHA256 | +| --- | --- | --- | +| tiny-gguf | `ggml-org/tiny-llamas` / `99dd1a73db5a37100bd4ae633f4cfce6560e1567` / `stories15M.gguf` | `61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a` | +| chat-litert | `litert-community/Qwen3-0.6B` / `8414150f2e9dcc82449bcc9c5abc404b399a4d06` / `Qwen3-0.6B.litertlm` | `555579ff2f4fd13379abe69c1c3ab5200f7338bc92471557f1d6614a6e5ab0b4` | + +Hash downloaded files before load; record cache hit/miss and bytes. Download and +checksum time are separate from inference. Do not assume Firebase preserves app +model caches between executions. Large packs must pass disk/memory preflight and +are excluded from older/constrained-device core unless explicitly qualified. +Within one execution, prepare/hash each immutable model once, then reuse that +verified file for its cases. Load one model at a time; retain it across independent +requests, reset conversation/cache state, and reload only for explicit lifecycle +cases. Hash checks and downloads never run inside timing windows. Start with +tiny-gguf plus chat-gguf for GGUF profiles, and chat-litert for LiteRT profiles; +the full model inventory is not an automatic download list. + +## 5. Case catalog and expected results + +All cases record expectation, actual output, assertion result and capability +decision before reporting aggregate status. Evaluate independent assertions +without throwing away earlier evidence. Cases below are shared specifications; +the model manifest determines supported positive rows and typed negative rows. + +| ID | Input / action | Expected result and scope | +| --- | --- | --- | +| C01 packaging, routing and load | Verify binary/model hashes; load by `.gguf` or `.litertlm`; probe capabilities | Correct runtime family, ready state and effective backend evidence; missing library/corrupt model produces actionable error. Native/Web assets distinguished. | +| C02 tokenizer and Unicode | Encode/decode `Hello, Montréal! 안녕하세요 👋\n two spaces`; plain-text mode with explicit BOS/special-token policy | Exact round trip where advertised, stable IDs for the pinned tokenizer; capture literal byte-level spellings. LiteRT Web asserts typed unsupported tokenizer path. | +| C03 raw stream | Tiny GGUF: `Once upon a time`, max 32; native chat-capable models use their locked raw fixture | One nonempty valid, finite ordered stream with no crash/hang. Benchmark repetitions are scheduled once separately, not repeated for every case. A stories model need not answer instructions. | +| C04 visible chat and thinking control | `Reply with one short sentence saying hello.` and `What is 2+2? Answer only with the number.`; thinking disabled | Greeting contains `hello` case-insensitively; arithmetic trimmed matches `^4[.!]?$`; no hidden-thinking leakage. Existing LiteRT failures remain visible until reference triage. | +| C05 thinking and budget | Same arithmetic prompt with thinking on, max 256; native GGUF budget 0 and 32 on a qualified reasoning fixture | Final visible answer; correctly separated thinking when emitted; no leaked delimiters. Budget behavior checked against native counters/reference semantics, not character count. LiteRT/Web reject native GGUF-only budget controls. | +| C06 conversation and system messages | System `Answer briefly.`; user `Remember this code: cedar-17.`; assistant acknowledgement; user `What code did I ask you to remember? Reply only with the code.` | Native/GGUF history retains `cedar-17`; captured serialized messages preserve system role. LiteRT Web single-turn limitation is explicitly reported; no claimed history pass. | +| C07 tools | Tool `get_weather(city: string)`; user `Call get_weather for Montréal.`; modes auto, required, none; fixed tool response | Required emits the named tool and parsed `{"city":"Montréal"}` where supported; none emits no tool; auto accepts permitted text or valid call. Tool response continuation tested where supported. Unsupported constrained modes fail explicitly. | +| C08 cancellation and reuse | Start long generation; cancel after first nonempty delta, await completion, then issue hello on a new/reset conversation | Stream terminates within 5 seconds, engine remains usable, no late deltas after completion; regeneration passes. If generation already finished, mark cancel subcase NOT_RUN and use the locked longer fixture. | +| C09 dispose and reload | Load/generate/dispose, then new engine/load/generate once in quick core; twice for lifecycle-focused changes or release selection | Clean shutdown; no live worker/stream handles; repeat completion. Record memory after each cycle without demanding identical RSS. | +| C10 limits and stop markers | Locked fixture reliably emitting a chosen marker, plus max-token limit case | Advertised stop/limit semantics, correct finish classification, marker suppression according to API contract; no uncontrolled continuation. Unsupported stop options rejected. | +| C11 native stream batching | Compare supported defaults with one explicitly supported batching configuration at deterministic settings | Same reconstructed content/tool/thinking, ordered completion; chunk count may differ. LiteRT Web must reject nondefault native batching controls. | +| C12 failure and recovery | Missing file, bad model hash, unsupported option, unsupported feature, cancelled download; reload valid fixture | Typed actionable errors, no false success or permanently stuck worker; valid recovery completes. Package/model format misrouting and missing companion are covered. | + +For model-dependent expectations (C04–C07/C10), first obtain reference behavior +from the exact fixture and sampler. Keep semantic failures distinct from transport, +parser and runtime failures. Do not demand exact free-form prose or universal +CPU/GPU token identity. Determinism checks are within a pinned cohort; parity +assertions use the specific contract/reference under test. + +LiteRT Web runs C01, applicable single-turn C03/C04, lifecycle C08/C09 and C12, +plus explicit unsupported tokenizer/history/tool/native-control cases. It does +not inherit native conversation capabilities just because both use `.litertlm`. + +The catalog enumerates C01–C12; a run manifest expands only its selection into +mandatory device/model/backend/subcase rows before execution: + +- Quick core: C01–C04, C06, C08, C09's single reload, C10's token-limit subcase, + and C12's invalid-input/recovery subcase, plus one benchmark series. +- Change-focused: add C05 for reasoning changes, C07 for tool/template changes, + C10's stop-marker case and C11 for streaming/stop/batching changes, and the + applicable feature packs. Load/dispose/worker/lifecycle changes require C09's + second reload and the relevant full C12 failure/recovery subcases. + Parser/template changes require affected-family + fixtures and positive/negative modes, even if that expands the quick selection. +- Release: all applicable C01–C12 subcases and existing required release rows. + +Web applies the same selection rules to its documented positive and negative +contracts. A model lacking a runtime-supported feature leaves that positive +subcase NOT_RUN with a fixture reason; it is not runtime UNSUPPORTED. Selecting +only tiny-gguf cannot qualify chat. Missing a mandatory row makes that selection +incomplete; accounting for a missing test never turns it into a pass. + +## 6. Feature packs + +| Pack | Fixtures / assertions | Selection | +| --- | --- | --- | +| Structured output | Prompt `Return an object with count 3 and ok true.`; schema with required integer `count`, boolean `ok`, no additional properties. Parse exactly; test invalid types/unknown keys, partial-stream suppression, malformed-final rollback and tool modes with thinking prefixes. | Supported GGUF grammar path; compiled grammar acceptance/rejection and affected-family upstream parity remain mandatory existing checks. LiteRT grammar negative case. | +| State and prompt reuse | Dense model: full recomputation vs reused prefix; save/restore same continuation; malformed/version-mismatched state | Native and capable GGUF Web bridge. Web virtual-file persistence is not durable reload storage; test explicit export/import separately. LiteRT unsupported. | +| Embeddings | `A cat sits on a mat.`, `A feline rests on a rug.`, `The engine uses diesel.`; single vs batch | Correct dimension, finite/nonzero vectors, single/batch tolerance; reference-qualified similarity ordering. GGUF supported paths; LiteRT negative. | +| Vision | Bundled synthetic red square on white; `What color is the square? Answer with one word.` | `red`, media reaches model, missing/wrong projector rejected. Native/GGUF Web and native LiteRT compatible bundles; browser LiteRT unsupported as documented. | +| Audio understanding | Fixed WAV saying `The meeting is on Tuesday.`; ask day of meeting | `Tuesday` with model-qualified tolerance; file/blob paths and invalid media errors. No live microphone needed for deterministic lab case. | +| ASR | Fixed licensed PCM WAV `The quick brown fox jumps over the lazy dog.`; batch and chunked streaming where supported | Normalized transcript matches reference, correct sample-rate validation, finalization/cancel/restart; log WER and real-time factor. Moonshine native CPU; Qwen3 native and qualified Web WAV route. | +| TTS | Text `Hello from llamadart.` and a supported speaker configuration | Finite nonempty PCM, valid rate/channels/duration, playable/exportable WAV and cancellation; listening quality remains separate local QA. | +| LoRA | Locked matching base/adapter; baseline, adapter active, reset | Supported load/runtime semantics, reference-qualified output effect; reject incompatible/aLoRA/version-skew cases. Native LiteRT only one default-scale load-time adapter; reject runtime updates/stacking/scaling. | +| Speculative | Same target/prompt/sampler with drafting off/on; locked compatible draft if needed | Output parity against appropriate upstream reference, accepted/drafted counts and speed; distinguish MTP, n-gram, external draft and native LiteRT controls. No unsupported Web substitution. | +| Runtime controls | Threads 1/4, relevant activation type/prefill setting; full/compact CPU packaging; explicit CPU/GPU/fallback | Effective values/backend proof, supported behavior and typed invalid-combination failures. Unsupported/NPU deployments do not silently pass on CPU. | +| App/device/browser | Download cancel/resume/cache, background/foreground, rotation, memory warning, Web worker restart, denied microphone, audio playback/export | App stays responsive, progress/cancel honest, recoverable errors and safe cleanup. Owned hardware covers human audio and sustained thermals; cloud media fixtures cover deterministic paths. | + +Run affected-family fixtures when templates/parsers change; a convenient tiny +model is only pipeline evidence. Full and compact Android CPU packages both +need physical lower-ISA and modern-device coverage for the relevant release. +Neither a modern phone running compact nor an emulator closes issue #476. + +## 7. Prompt and inference configuration contract + +Store case prompts as versioned UTF-8 data. Reports include literal test prompts, +message roles, tool schemas, template identity/options, media fixture hashes, +expected predicates and case version. Use synthetic fixtures only; never collect +personal chat history or microphone content for unattended runs. + +Original profile design sketch (the executable JSON contract is now in +`packages/llamadart_validation/assets/profiles/`; this sketch is not CLI input): + +```yaml +schema_version: 1 +profile: native-quick +model_id: chat-litert +case_ids: [C01, C02, C03, C04, C06, C08, C09, C10.limit, C12.recovery] +load: + context_size: 1024 + threads: 4 + batch_threads: 0 # LiteRT; GGUF profile uses 4 + backend: cpu # separate execution for explicit gpu +generation: + max_tokens: 32 + temperature: 0 + seed: 1 + enable_thinking: false +benchmark: + id: short-generation + prompt: "List the numbers from one to twenty in English." + warmups: 1 + measured_runs: 3 +timeouts: + generation_seconds: 60 + model_prepare_seconds: 300 + cloud_execution_minutes: 20 +``` + +GGUF raw timing uses `Once upon a time`; instruction-model timing uses the list +prompt above. C05/C08 and feature packs override token budgets explicitly. +By default benchmark only chat-gguf or chat-litert once per backend, with one +warm-up and three measured samples total; do not multiply repetitions by case +count. Tiny-gguf timings are packaging diagnostics unless explicitly selected as +a separate benchmark cohort. If both models are benchmarked, report two series. +Default context is 1024; tools/feature packs can request 2048 with a distinct +configuration hash. Standard backend comparisons use the same settings and model. +Do not compare this 32-token core smoke to long-context throughput. + +At build/run time expand **all** remaining package/runtime defaults into the +effective record (top-k, top-p, min-p, penalties, stop strings, batch thresholds, +cache/prompt-reuse policy, GPU hint, activation type, prefill and speculative +settings). Include unsupported/null provenance rather than inventing values. +The frozen runtime revision plus effective configuration hash identifies the +experiment. Capability validation happens locally before a cloud submission. + +For warm measurements, retain the loaded model but start a fresh conversation +and reset request/KV state unless testing reuse. Record actual cache/reuse +behavior; if it cannot be disabled or verified, label the cohort as warm-cache +and do not compare it to uncached prefill. Native initialization may be lazy: +`loadModel` return time and first inference readiness are separate measurements. + +## 8. Firebase device selection and free rotation + +The live `gcloud firebase test ... models list` catalog on **2026-09-16** contained +205 Android entries (196 physical, nine virtual) and six iOS models. Counts +include device/form-factor variants, not 196 distinct useful inference targets; +capacity is not a reservation. The iOS catalog contained iPad 10, iPhone 8, +11 Pro, 14 Pro, 16 Pro and SE 3. Refresh supported OS/version and capacity before +every submission using the [Android](https://firebase.google.com/docs/test-lab/android/available-testing-devices) +and [iOS](https://firebase.google.com/docs/test-lab/ios/available-testing-devices) +catalog guidance. + +Choose for a new driver/SoC generation, CPU instruction baseline, OS boundary or +form factor. Avoid spending routine quota duplicating the owned Pixel 9 Pro. + +| Priority | Device ID / catalog OS | Added coverage | Initial cases | +| --- | --- | --- | --- | +| A | Galaxy S24 `SC-51E` / API 36; low capacity | Pilot's verified Adreno 750 crash/output-divergence reproducer | Four isolated GGUF CPU/Vulkan and LiteRT CPU/GPU profiles; #79/#51 diagnostic controls | +| A | Lenovo Tab P12 `TB370FU` / API 35; medium capacity | MediaTek Dimensity 7050 / Mali-G68, non-Pixel/non-Qualcomm driver stack, Android tablet | Same four core profiles; tablet app flow when relevant | +| A | iPhone 16 Pro `iphone16pro` / 18.3; medium capacity | Repeatable pilot Apple reference; actual OS was 18.3.2 | Four CPU/Metal and LiteRT CPU/GPU core profiles | +| A | iPhone SE 3 `iphonese3` / 26.3; medium capacity | Newer catalog OS on a different, older Apple device generation | Four core profiles, lifecycle and small-screen flow; retain 18.4 as optional same-model OS control | +| B | Galaxy A05s `a05s` / API 35; low capacity | Snapdragon 680 / Adreno 610, older/budget CPU and GPU; candidate lower-ISA qualification device | GGUF CPU full + compact first; probe CPU features and selected module before claiming older-ISA coverage; GPU and LiteRT next | +| B | iPhone 8 `iphone8` / 16.6; medium capacity | Older Apple hardware and nearest available OS to current iOS minimum 16.4 | Tiny GGUF CPU/Metal first; representative model only after memory preflight; LiteRT and resource failures explicitly recorded | +| C | iPad 10 `ipad10` / 16.6; medium capacity | Tablet/older-OS gap if owned iPad does not cover it | Layout/lifecycle and affected native backend only | +| C | Pixel 5 `redfin` / API 30; high capacity | Older Android OS when installation/runtime compatibility changes | Packaging/CPU core, then affected backend; different purpose from A05s ISA checks | + +Hardware family sources: [Lenovo Tab P12 specifications](https://psref.lenovo.com/syspool/Sys/PDF/Lenovo_Tablets/Tab_P12/Tab_P12_Spec.pdf), +[Samsung A05s specifications](https://www.samsung.com/africa_en/smartphones/galaxy-a/galaxy-a05s-silver-128gb-sm-a057fzsgafc/), +and [Qualcomm Snapdragon 680 brief](https://www.qualcomm.com/content/dam/qcomm-martech/dm-assets/documents/product_brief_-_snapdragon_680_4g_mobile_platform.pdf). +Probe actual RAM, CPU features, GPU/driver and OS build in each execution; do not +infer a lab SKU's RAM or instruction support from its marketing name. S24 regional +variants can differ; retain `SC-51E`, not just `Galaxy S24`. + +Virtual candidates in the current catalog: + +- `MediumPhone.arm`, API 29 and 36: older/current Android CPU and installation + checks, subject to the final test app's minimum SDK. +- `MediumPhone_ps16k.arm`, API 36: real 16K-page emulator packaging/load checks. +- `MediumPhone_ps16k_backcompat.arm`, API 36: separate compatibility-mode case + when changing native alignment/loading; never substitute it for native 16K. + +These are arm64 virtual entries. Keep Android x64 coverage in a local/CI emulator. +GPU-capable virtual infrastructure does not prove a physical Adreno/Mali driver, +device NPU, lower-ISA CPU selection or real-device memory behavior. + +### Optional LiteRT-LM NPU test pack + +Add **Galaxy S24 first, then Pixel 10** after the deferred implementation and +preflight prerequisites are complete. This is a qualification proposal, not a +claim that Firebase or the current llamadart bundle has passed NPU inference. +Google's [LiteRT-LM NPU guide](https://developers.google.com/edge/litert/next/litert_lm_npu) +documents SoC-specific Gemma 3 1B models for Qualcomm SM8650 and Google Tensor G5. +The [Google Tensor SDK](https://developers.google.com/edge/tensor-sdk) remains +labelled beta; verify required SDK/model access before selecting that row. + +| Order | Device / catalog OS | Required NPU target | Purpose | +| --- | --- | --- | --- | +| 1 | Galaxy S24 `SC-51E` / API 36 | Qualcomm SM8650, Snapdragon 8 Gen 3; matching Qualcomm dispatch and QAIRT/HTP libraries | Reuse the pilot device and add its first actual NPU qualification | +| 2 | Pixel 10 `frankel` / API 36 | Google Tensor G5; matching Google Tensor dispatch/runtime | Independent vendor path; catalog snapshot reports high capacity | +| Alternate | Pixel 10 Pro `blazer` / API 36 | Tensor G5, separately recorded device/OS/driver cohort | Use only if the base Pixel 10 is unavailable; snapshot reports low capacity | +| Later | Galaxy S25 Ultra `pa3q` / API 35 or 36; OnePlus 11 `CPH2449` / API 34 | Corresponding SM8750 or SM8550 model/runtime | Optional Qualcomm-generation expansion after the first two paths work | + +Device/OS entries are from the 2026-09-16 Firebase catalog snapshot. Confirm the +actual `ro.soc.model`, ABI, OS build and driver on device, then require an exact +manifest match before loading. S24 variants with other SoCs are not substitutes. +See [SC-51E hardware specifications](https://www.docomo.ne.jp/support/product/sc51e/spec.html) +and [Pixel 10 Tensor G5 specifications](https://blog.google/products-and-platforms/devices/pixel/tensor-g5-pixel-10/). +Catalog presence establishes access to a device, not its NPU permissions or +working dispatch libraries inside the Test Lab app sandbox. + +The owned Pixel 9 Pro's Tensor G4 and the Tab P12's Dimensity 7050 are absent from +the documented LLM NPU model table, so they are not positive NPU targets in this +pack. Current llamadart Apple backends expose CPU/GPU only. Existing GPU tests +remain useful and do not qualify any of these devices' NPUs. + +**Preflight before spending a device execution:** + +1. Resolve model access and freeze the exact `.litertlm` revision, SHA256, + quantization, compiled SoC, context limit and tokenizer/template metadata. + Use the vendor-specific Gemma fixture, not the pilot's generic Qwen3 bundle. + No gated download or SDK terms acceptance is implied by adding this plan. +2. Resolve compatible native runtime, vendor dispatch library and all dependent + libraries, including Qualcomm QAIRT/HTP host and DSP libraries where required. + Pin their versions/checksums and verify permitted packaging. Inspect final + APK contents and native dependencies; the inspected pilot APK contained no + Qualcomm NPU/dispatch libraries. A library search path alone cannot supply them. +3. Arrange model/library transfer into the application sandbox through a supported + test mechanism. The upstream `/data/local/tmp` CLI recipe is a diagnostic + reference, not proof an installed Flutter app can access the same files or + DSP libraries. Keep credentials out of APKs, logs and exported artifacts. +4. Use explicit `LiteRtLmBackendPreference.npu` and a validated + `liteRtLmDispatchLibDir`. Keep llama.cpp-only options at supported defaults + (`numberOfThreadsBatch=0`). Lock context 1280 for the documented Gemma NPU + fixtures, max output 32, temperature 0, seed 1, and one warm-up plus three + measured runs. Record all effective settings; do not apply Qwen-specific + thinking/template options to Gemma. Native automatic controls stay at their + documented defaults unless the exact NPU artifact supports an override. +5. Build a minimal direct-native reference test and the public Dart test with + the same model and native/runtime/library inputs. Both must run as installed + app tests without root, protected vendor-directory writes or special device + modifications. Missing access/library/model prerequisites leave the row + NOT_RUN with a concrete reason; do not burn cloud quota on an incomplete APK. + +| Case | Action | Required evidence / expected output | +| --- | --- | --- | +| N01 identity and load | Load the exact SoC-matched Gemma artifact through explicit NPU selection | Correct artifact/library hashes; native dispatch initialization and actual NPU graph/partition execution evidence. A selector or `availableBackends` string alone is insufficient. | +| N02 semantic reference | Native reference and public Dart each run C04 hello/arithmetic, the C02 tokenizer encode/decode fixture, and a separate generation prompt `Reply with exactly: Montréal 👋` | Apply C04 semantic oracles and C02 exact tokenizer round-trip rules separately. The generation fixture expects trimmed `Montréal 👋` after reference qualification; it cannot substitute for tokenizer coverage. Preserve raw native text and Dart content/finish handling; no corruption or silent empty response. | +| N03 bounded throughput | `List the numbers from one to twenty in English.`; one warm-up, three measured generations | Coherent output and clean finish; cold readiness, TTFA, actual token counts, native prefill/decode TPS when available, estimated wall TPS and memory. Missing counters remain null. | +| N04 lifecycle | Cancel a qualified long fixture after first output; reload and generate again | Bounded stream termination and usable engine, no stale callbacks or native crash; use the C08/C09 rules. | +| N05 failure contracts | Unit/local fixture checks for missing dispatch dependency and mismatched SoC/model manifest | Preflight rejects incompatible manifests; missing native dependency produces actionable diagnostics. Do not deliberately load an incompatible compiled model on the lab device. | +| N06 explicit backend integrity | Inspect native execution/delegate evidence during each successful generation | Record partition placement and any CPU fallback. A CPU-only result fails the requested NPU row; mixed execution is labelled hybrid, never reported as all-NPU. Missing proof leaves qualification incomplete. | + +The direct-native control must be compared on the same target device/OS and +exact compiled artifact. A separate CPU Gemma model can help diagnose prompt or +model quality, but its weights/quantization/conversion may differ: log those +differences and do not present its TPS ratio as a pure NPU speedup. Similarly, +do not run an NPU-compiled file on CPU unless the artifact explicitly supports it. + +Initially schedule three separate executions per device: **direct-native NPU +reference, public llamadart NPU, and a compatible CPU semantic control**. Each +test adapter can contain multiple cases and timing repetitions within that one +execution. Keep the native reference and Dart path in separate app processes so +a crash or initialization state cannot conceal the difference. In a first smoke, +stop before the Dart cloud submission if the native control cannot initialize; +retain the unspent quota and record Dart qualification as NOT_RUN. + +Append S24 on day 5 and Pixel 10 on day 6 of the proposed rotation: **six optional +executions over two additional quota days**, bringing core plus this initial NPU +pack to 22 executions across at least six days. This stays below four planned +physical executions/day and preserves the fifth daily slot for an explicit +diagnostic rerun. Do not add NPU to an already-full four-profile core day. +Further repetitions, alternate devices, or artifact experiments require extra +quota days; no automatic retries or billing upgrade. + +Three controls per device are the initial qualification budget, not an obligation +for every later Dart-only edit. After qualification, rerun current public-Dart +NPU cases on relevant changes; rerun the native control when native/model/vendor +inputs, template/sampling settings, SoC, OS or driver change, or results diverge. +Reuse a CPU semantic reference only when its model/configuration/template and +device/OS identities still match. Show reused +controls as dated reference links, never current-head passes. An explicit release +or issue-verification requirement to rerun a control overrides this optimization. + +Reports add NPU/hybrid rows to the existing heatmap and TPS panels, with vendor, +SoC, compiled-model hash, dispatch/QAIRT version, partition placement and proof +links. A usable NPU result requires correct public-package output, lifecycle +completion and verified NPU execution; missing evidence or native failure never +becomes a green backend badge. No NPU run is scheduled by this document update. + +### Budget and scheduling + +Use existing Spark project `llamadart-device-qa-20260916`, **without linking +billing**. Firebase currently allows five physical executions and ten virtual +executions per project/day on Spark. Each device configuration, retry and shard +can consume another execution; five devices times four profiles is twenty +executions, not five. See [official quotas](https://firebase.google.com/docs/test-lab/usage-quotas-pricing). + +Reserve at most **four planned physical executions/day**, leaving the fifth +for an explicit diagnostic rerun. Disable automatic flaky retries, sharding and +device/OS Cartesian expansion. Inspect remaining quota and other active matrices +before submitting; insufficient quota means NOT_RUN/reschedule, never enabling +billing or creating projects to bypass the allowance. + +| Day of a release rotation | Device | Planned executions | +| --- | --- | --- | +| 1 | S24 | GGUF CPU, GGUF Vulkan, LiteRT CPU, LiteRT GPU = 4 | +| 2 | Tab P12 | Same four profiles = 4 | +| 3 | iPhone 16 Pro | GGUF CPU, GGUF Metal, LiteRT CPU, LiteRT GPU = 4 | +| 4 | iPhone SE 3 / 26.3 | Same four profiles = 4 | + +This proposed core rotation takes **16 executions across at least four quota +days**. It qualifies those selected rows only; it is not the complete supported +platform/release matrix. Older-device, CPU full/compact, OpenCL, NPU and large +feature-pack executions extend the rotation on additional days. Do not spend the +reserved rerun automatically. For a narrow runtime change, run its CPU/GPU pair +on two relevant devices in one day instead of all four profiles on one device. + +Before dispatch, produce a local selection summary with the exact case/model +rows, model bytes, build artifacts, execution count and quota-day assignment. +For a narrow change, choose one representative of each affected hardware/driver +family first; expand on failure or when release coverage requires it. Apply only +to the changed runtime: a LiteRT-only change does not automatically spend quota +on both GGUF profiles. Preserve the omitted rows as NOT_RUN, with their reason. +The published rotation is the initial coverage schedule, not a recurring job. +A LiteRT-only follow-up across the same four core devices therefore needs eight +CPU/GPU executions over at least two quota days, rather than sixteen for both +runtimes. This saves runs by selecting scope, not by claiming fresh GGUF evidence. + +Use the owned Pixel/Mac/iPad for frequent checks and stable performance baselines. +Run Firebase on native pin/backend/packaging changes and release candidates, not +on every documentation or pure-Dart PR. Select virtual packaging cases only when +needed and count them against the shared ten/day limit. + +The initial five pilot submissions finished (one intentionally cancelled). +The last billing check on 2026-09-16 had `billingEnabled: false`; it needed no GCP +credit. Require billing to remain disabled before future submissions. This plan +adds no new submission. Free Test Lab executions have no idle VM to stop. Any later +Compute Engine CUDA testing is a separate action: verify credit eligibility and +remaining balance first, and account for disks/IP/storage after stopping a VM. +Stopping compute does not guarantee every associated resource is free. + +Android Device Streaming is optional interactive debugging, with a separate +30-minute/project/month free allowance at this snapshot; it is not extra +automated execution quota. End a streaming session explicitly. Do not upgrade +to Blaze for either service to make this plan fit. + +## 9. Remote execution, provisioning and cleanup + +The orchestrator owns the full **prepare → upload → run → collect → cleanup** +flow. Start with `gcloud` driven from the owned Mac and a local run journal; +no always-on controller or Terraform deployment is needed for the initial lanes. +CI builds downloadable bundles. It does not automatically create VMs or submit +Firebase tests. A separately authorized remote run consumes those exact bundles +without rebuilding the test app on the destination. + +### One command interface and resumable run record + +Implement these subcommands in `tool/testing/validation.dart`, with provider +logic under `tool/testing/validation/`. They are **proposed commands**, not +available commands to execute today: + +```text +dart run tool/testing/validation.dart plan --target gce-linux-cuda --bundle --profile gguf-cuda --out +dart run tool/testing/validation.dart plan --target firebase-android --bundle --profile litert-cpu --device --out +dart run tool/testing/validation.dart run --plan +dart run tool/testing/validation.dart status --run-id +dart run tool/testing/validation.dart collect --run-id +dart run tool/testing/validation.dart cleanup --run-id +``` + +Also support `gce-windows-cuda` and `firebase-ios` targets. `plan` performs only +local/read-only checks and writes the exact selection, bundle hashes, account, +project, device or VM specification, deadlines, cost/quota preflight and cleanup +policy. Project/account values come from explicit local configuration; never +change the user's default `gcloud` account/project. Require a fresh quota/credit +check immediately before `run` makes remote changes. Missing eligibility or +configuration leaves the selection NOT_RUN with a concrete reason. + +`run` uploads and starts the selected tests, collects available evidence and +cleans up automatically, including failure paths. `status`, `collect` and +`cleanup` reconnect to an existing run; they never create a replacement or +silently rerun inference. An explicit retry gets a new attempt ID and preflight. +Cleanup of an active run first cancels it, then attempts bounded collection. + +Under `.dart_tool/validation/runs//`, persist `run-plan.json`, +`orchestration.json`, append-only `remote-events.jsonl`, and `cleanup.json` next +to the common manifest and test results. Write mutation intent before each +provider call and save operation, instance, disk, matrix and result-location IDs +as soon as known. Use stable run/attempt IDs and reconcile uncertain submissions +before retrying. Record preparation, execution, evidence collection and cleanup +as separate outcomes; passing inference cannot hide failed or unknown cleanup. + +### Disposable Compute Engine CUDA VMs + +| Phase | Planned implementation / completion evidence | +| --- | --- | +| Preflight | Require a separate explicitly selected personal GCP project, active applicable credit and expiry, GPU quota, available region/machine, and a conservative per-run estimate covering compute/GPU, Windows licensing where relevant, disks, IP and transfers. Include other known credit use and a reserve. If coverage cannot be established, do not provision. | +| Create | Create one VM per attempt with an immutable OS image ID, pinned driver/bootstrap inputs, run labels and a persisted resource ledger. Set an absolute deletion deadline at creation, default 60 minutes after dispatch, earlier than credit expiry with a margin. Set auto-delete on every newly created disk; avoid snapshots, reserved addresses, buckets and NAT services for this lane. | +| Bootstrap | Linux shell or Windows PowerShell installs only required runtime/driver prerequisites. Verify driver readiness, actual GPU and CUDA compatibility before uploading tests. Bootstrap is restart-safe and bounded to 20 minutes; failure triggers collection and cleanup. Record actual installed versions, image, GPU, driver and bootstrap hashes. | +| Upload | Fetch and verify the exact CI bundle on the controller, then transfer it over authenticated SSH/SFTP, preferably through IAP. On Windows enable Google's supported SSH package in instance metadata/bootstrap. Use a run-specific remote directory; verify checksums again on the VM. No GitHub token or GCP key is embedded in the bundle or startup metadata. | +| Execute | Invoke the packaged CLI through a tracked shell/PowerShell job using the locked manifest/profile. Download and hash selected models with the existing five-minute preparation bound; run with a 20-minute test timeout. Track the remote job identity so reconnecting observes the existing job. CUDA requires actual offload evidence; it does not establish LiteRT GPU support. | +| Collect | Incrementally copy bounded events/diagnostics to the controller, then retrieve complete JSONL, samples, logs and crash evidence. Allow at most ten minutes for final collection, constrained by the deletion deadline. Missing evidence is reported explicitly; it never extends the VM lifetime automatically. | +| Teardown | In a finally path after success, failure, timeout or cancellation, delete this run's disposable VM and owned disks/resources. Wait for provider operations, re-query the recorded resource IDs and report remaining resources. A stopped instance alone does not satisfy this lane's cleanup contract. | + +For the deadline use Compute Engine's `--termination-time` with +`--instance-termination-action=DELETE`, then read back the effective scheduling +configuration. An absolute deadline avoids extending the allowance after a +restart. Google's deadline can begin termination up to 30 seconds late, so leave +time/credit margin; it is a provider backstop, not exact billing precision. +See [VM runtime limits](https://docs.cloud.google.com/compute/docs/instances/limit-vm-runtime). +This protects against the controller disconnecting; a shell finally block alone +cannot. Do not rely on this backstop for a VM that stops before its deadline: +Compute Engine clears its termination timestamp when stopped. Reconnect and +delete that VM and its disks through the ledger; stopped/unknown state remains +unresolved cleanup. Never remove or extend the deadline automatically. Partial logs may be +lost if the controller cannot reconnect before deletion; report that evidence +gap instead of retaining a billable disk indefinitely. + +Use [IAP forwarding](https://docs.cloud.google.com/iap/docs/using-tcp-forwarding) +for administration and [Windows SSH](https://docs.cloud.google.com/compute/docs/connect/windows-ssh) +for the Windows adapter. Resolve the outbound download path in preflight: if a +temporary external IP is needed, count its cost and restrict administrative +ingress. Do not create a persistent NAT service to hide that dependency. +Keep cloud credentials on the controller using the user's existing login; do +not add cloud secrets or billing permissions to the build workflow. + +Stopping a VM leaves potentially billable resources such as disks and static +addresses; see [Compute Engine stop behavior](https://docs.cloud.google.com/compute/docs/reference/rest/v1/instances/stop). +The cleanup ledger must distinguish deleted, still present and unknown resources. +Restrict deletion to exact resources created for this run, verified by IDs and +ownership; never sweep a project or delete an existing user machine. Reconcile +unfinished local run journals before starting another VM, and block new +provisioning while earlier cleanup is unresolved. Do not automatically reprovision +after eviction, capacity failure or a lost connection. + +Credit coverage is a dispatch prerequisite, not a claim of guaranteed free GCE +usage. [Alerts-only budgets do not cap spending](https://docs.cloud.google.com/billing/docs/how-to/budgets). +If current credit eligibility/balance cannot be verified with sufficient margin, +use owned hardware/free CPU CI and leave CUDA NOT_RUN. Provider-state cleanup +verification and any later billing reconciliation are separate evidence; no +account balance, current VM state or credit expiry was verified by this plan edit. + +### Firebase upload, execution and collection + +Use Flutter integration tests as Android instrumentation or iOS XCTest, as +[Firebase documents](https://firebase.google.com/docs/test-lab/flutter/integration-testing-with-flutter). +Robo crawling is not the correctness harness. The pilot confirmed the package +runs through both wrappers; the maintained harness is still to be implemented. + +1. Freeze source/dependency/model/configuration hashes. Build and locally validate + supported options, signed iOS inputs and device-specific install requirements. + Use Release mode for comparable measurements where the integration runner + supports it; validate that path first. Keep Debug-only measurements in a + separate cohort if Release instrumentation is unavailable. +2. Refresh catalog and quota, choose an exact device/OS and one backend profile. + Submit an explicit one-device matrix with a 20-minute execution timeout, no retries + and video disabled by default. Persist a local run ID and submission state; + record matrix ID immediately. An uncertain submission is reconciled against + existing matrices before any retry, to avoid duplicate quota consumption. +3. Run each backend profile in a separate cloud execution/app process. Include + lifecycle reload tests within that profile. A crash must not prevent the + other backend profiles from producing their own results. +4. Download pinned public models inside the test or use a verified supported + fixture-transfer mechanism. No paid custom model bucket. Bound download at + five minutes and distinguish network preparation failure from inference. +5. Write results incrementally to durable files; capture Android pullable app + artifacts through a verified Test Lab mechanism and iOS XCTest attachments. + Prove retrieval on each platform before relying on it. Keep short sequenced + JSONL summaries in logs as crash fallback, with byte limits and checksums for + any reconstructed fragments. Never parse a truncated line as valid JSON. +6. Collect JUnit, JSON, logs, native crash/tombstone or XCTest diagnostics and + manifest. Reconcile app-completed cases with wrapper/matrix status; a process + crash with zero JUnit cases is still ERROR, not a zero-failure success. +7. Wait for terminal matrix/execution states. On interruption, reconnect via + saved matrix IDs; cancel unnecessary outstanding tests and verify termination. + Export evidence promptly rather than depending on the cloud console forever. + +The Firebase adapter pushes the local verified APK pair using +`gcloud firebase test android run --type=instrumentation --app=... --test=...`, +or the locally built/signed iOS XCTest zip using +`gcloud firebase test ios run --type=xctest --test=...`. Both specify the exact +project/device, `--async`, `--timeout=20m`, `--num-flaky-test-attempts=0`, +`--no-record-video`, a run/attempt label and a unique `--results-dir` per matrix. +Omit `--results-bucket` to retain default Test Lab storage. APKs/XCTest inputs +carry the selected profile/manifest; prove any runtime configuration injection +before depending on it. See the official +[Android CLI](https://docs.cloud.google.com/sdk/gcloud/reference/firebase/test/android/run) +and [iOS CLI](https://docs.cloud.google.com/sdk/gcloud/reference/firebase/test/ios/run). + +A run label or results directory is correlation, not a provider idempotency key. +If the CLI loses its response before the matrix ID is saved, mark submission +UNKNOWN and block replacement submissions until the existing attempt is positively +identified in provider records/console. Do not infer that nothing was submitted +from an empty local journal or an unsuccessful lookup. A later REST adapter can +use a persisted create request ID, but the CLI path must first prove its recovery +behavior; it cannot claim automatic deduplication from labels alone. + +The 20-minute flag limits test execution, not provider queue/setup/cleanup time. +Apply a separate 45-minute controller deadline, request cancellation when it is +exceeded, and continue terminal-state reconciliation on reconnect. Capture the +matrix ID/result URI immediately, poll that same matrix and collect from its +returned location. Use the +[testMatrices cancellation API](https://firebase.google.com/docs/test-lab/reference/testing/rest/v1/projects.testMatrices/cancel) +when aborting; sending cancellation alone is not proof that execution has ended. +Android `--directories-to-pull` must name verified accessible output directories +within the supported roots, accounting for scoped storage. iOS uses verified +XCTest attachments. Retain the earlier incremental log fallback for native crashes. + +Firebase manages device allocation/install/run and device cleanup; we own +submission, cancellation, evidence retrieval and terminal-state checks. Do not +delete the Spark project or its default result bucket as per-run teardown. End +any separately opened Device Streaming session. Serialize our submissions within +the four-planned-physical-runs/day policy, account for other project users, and +never upgrade billing or retry automatically when quota is exhausted. No VM is +created by this adapter. + +Keep the default Test Lab result storage for the Spark pilot; do not provision +paid custom Cloud Storage. Store downloaded evidence locally and sanitized small +reports as CI artifacts with bounded retention when that path is implemented. +Do not rely on Flutter per-test Firebase timings or video segmentation: Firebase +documents limitations, so measure durations inside the suite. + +## 10. Logs, metrics and result contract + +Write `manifest.json`, incremental `events.jsonl`, and bounded `diagnostics/` on +device. One host-side exporter validates those records and derives `results.json`, +`junit.xml`, `samples.csv` and `summary.html`; wrappers must not implement separate +aggregation rules. Firebase's own JUnit remains raw provider evidence and is +reconciled with the derived case report. Preserve samples so reports can be +recomputed without rerunning inference. Common provenance belongs in the manifest; +events reference stable case/model/configuration IDs. Reruns append attempts and +preserve previous failures; never export only the best attempt. These exporters +are still proposed. + +The manifest lists every mandatory expanded row and a stable ID; each has one +terminal result per attempt. Detect missing, duplicate or truncated records. +Missing child rows after a process crash become NOT_RUN with the parent ERROR; +missing rows in an apparently successful run make the report incomplete. The +summary shows selected/attempted/passed counts and missing mandatory IDs. It can +be green only when every selected obligation passed (including explicitly +expected unsupported-operation guards); omitted feature packs remain uncovered. + +Required identity fields: source commit/dirty patch hash, package/Flutter/Dart +versions, native and Web release tags plus artifact checksums, build mode/ABI, +model/companion hashes, case/profile versions, effective config hash, requested +and resolved backend, CPU features, GPU/driver, device ID/OS build, RAM and page +size where measurable, browser/runtime capabilities, provider and cloud IDs. + +Per-case evidence includes monotonic start/end and phase timings; exact synthetic +messages; sanitized rendered prompt/template where observable; generated content, +thinking and tool deltas; token counts with provenance; finish/cancel reason; +expected predicate; actual result; exception type/code/stack; backend initialization +and fallback evidence; output hash; warm-up/sample index; resource availability. +Capture native INFO diagnostics around load/failure, and bounded logs for normal +runs. During timing, buffer bounded text/counters in memory, then serialize after +the stopwatch stops. Keep only small start/phase/crash breadcrumbs on the hot +path; record native logging level and flush overhead so logging changes cannot +masquerade as a speedup. Unknown values are `null` with a reason, never invented +zeroes. Redact credentials, signed URLs, account/device serials and private paths. + +| Metric | Definition / interpretation | +| --- | --- | +| Model preparation ms | Download and checksum separately; excluded from inference TPS | +| Public load ms | `loadModel` entry to return; lazy native initialization may remain | +| Cold first-response ms | From entry to the first `loadModel` on a new engine/process to first visible output of its first request; includes load and lazy initialization, excludes prior download/hash time. Record OS file-cache state as unknown unless controlled. | +| TTFA ms | Generation call to first nonempty public content delta; separate first-any-event/thinking times | +| Native TTFT ms | Runtime first-token timing only if exposed; do not relabel TTFA as TTFT | +| End-to-end output TPS | Authoritative generated visible-output token count / stream wall seconds, when available; include first-output latency | +| Estimated wall TPS | Retokenized visible output / stream wall seconds if authoritative count unavailable; label estimated, use null without tokenizer | +| Native prefill/decode TPS | Native token counters / respective native phase seconds; record whether counters are per request or differenced | +| Post-first-token TPS | `(tokens - 1) / (last-token time - first-token time)` only with actual token timestamps; batched Dart chunks are insufficient | +| Memory | Process RSS/peak and GPU allocation where available; provider OS profiler vs in-process sample distinguished | +| Reliability | Cases attempted/completed, crashes/timeouts, backend fallbacks, correctness failures, coverage NOT_RUN counts | +| ASR/TTS | ASR WER plus real-time factor = elapsed seconds from first audio feed to final transcript / input audio seconds. TTS real-time factor = elapsed seconds from synthesis request to final PCM sample / output audio seconds; excludes playback. Record streaming feed pacing; paced live ASR is a different cohort from unpaced file ASR. | + +One warm-up plus **three measured repetitions** per benchmark profile; report +median/min/max and individual dots. No p95 from three observations. End-of-sequence +may produce fewer than 32 tokens: record actual count and completion cause, never +divide by the requested maximum. Thinking/tool tokens and visible tokens are +different populations and must be named explicitly. + +Continue independent timing cases after semantic assertion failures if the +process is healthy. Retain correctness FAIL and tag their timings +`correctness_failed`; exclude them from passing performance baselines by default. +Crashes/timeouts have missing timings, not zero TPS. Measurement failure does not +erase correctness evidence already written. + +Statuses: `PASS`, `FAIL` (an assertion failed), `ERROR` (crash, timeout, harness, +download or infrastructure failure, with a precise reason), `UNSUPPORTED` +(expected contract limitation), `NOT_RUN` (quota, unavailable hardware, missing +fixture or deliberately out of selected scope). A tested unsupported-operation +guard itself can PASS while the positive feature row remains UNSUPPORTED. +Known issues remain failed/error rows linked to their issue; no green XFAIL mask. + +Illustrative record shape, using the observed pilot arithmetic failure; timing +values are deliberately absent: + +```json +{ + "schema_version": 1, + "case_id": "C04.arithmetic", + "device_id": "SC-51E", + "runtime": "litert_lm", + "requested_backend": "cpu", + "model_id": "chat-litert", + "expected": {"trimmed_regex": "^4[.!]?$"}, + "actual": {"content": "2"}, + "status": "FAIL", + "reason": "semantic_oracle_mismatch", + "metrics": {"decode_tps": null, "wall_tps": null}, + "metrics_unavailable_reason": "pilot_assertion_stopped_before_benchmark", + "issue": "https://github.com/leehack/llamadart/issues/509" +} +``` + +## 11. Report and graph design + +The first report is a portable offline HTML summary with embedded validated data, +status tables, per-sample TPS/latency plots and artifact links. Generate CSV/JSON +from the same records. Add interactive filters, historical trends and quota +visuals after the exporter and physical-device evidence are reliable. No hosted +database, paid dashboard, or GCP service is needed. The views below describe the +complete report design, not six blockers to the first working harness. + +| View | Visual / interaction | Reading rule | +| --- | --- | --- | +| Coverage | Device × runtime/backend heatmap; model/feature filters; labels and icons in addition to colors | PASS green, FAIL red, ERROR orange, UNSUPPORTED patterned gray, NOT_RUN empty gray; click opens exact evidence | +| Throughput | Separate small panels for native decode TPS and end-to-end/estimated wall TPS; three dots with median and min/max whiskers | Same model/config/build cohort only; show sample count and failed-output badge; never combine the two TPS definitions | +| Latency | Download, load, cold first response, warm TTFA and total stream duration, separately labelled | No stacking overlapping timings; missing values shown as unavailable | +| Trend | Per-device/model/backend median versus commit, with sample ranges | Compare identical OS/driver/runtime configuration or start a new cohort; annotate pin changes | +| Failures | Exact expectation and output, phase, exception/crash summary, issue link, native/backend evidence | Expose known failures and missing evidence before aggregate pass percentage | +| Cost/coverage | Execution count used/planned and remaining known quota, deferred rows | Show incomplete coverage plainly; no estimated spend presented as billed cost | + +The pilot's GGUF medians illustrate why separate charts are necessary: S24 CPU +estimated wall TPS 396.7 vs native decode 475.0; iPhone CPU 674.1 vs 747.5; +iPhone Metal 443.4 vs 5264.0. These are tiny-model diagnostic observations, not +a device ranking. Native compute timing excludes work included in the public +stream. Build modes also differ. Do not put these values in a comparable-platform +leaderboard or infer LiteRT TPS from them. + +Performance is initially informational. Establish stable owned-device baselines +before setting thresholds. A candidate alert is a >20% median slowdown with +matching provenance, but three lab samples alone cannot prove a regression: +confirm locally or in another quota-approved run. Correctness/crashes are +blocking independently of speed. No throughput threshold hides a known failure. + +## 12. Implementation sequence and completion criteria + +1. **Implementation authorized:** use current merged pins; reconcile any later + release changes before qualification. + lock reference-qualified core fixtures and resolve model-dependent oracles. + Reuse the three pilot issues to investigate failures separately from harness work. +2. Implement the smallest shared suite, desktop runner and mobile wrapper, plus + immutable manifests and incremental JSON. Integrate existing matrix/E2E + discovery. Validate on owned Mac/Pixel/iPad before consuming cloud quota. +3. Add separate Android/iOS backend profiles, local preflight, artifact retrieval + and terminal-state reconciliation. Test the Firebase adapter with fake + provider responses for uncertain submission, duplicate invocation, quota + exhaustion, cancellation and missing evidence before any cloud use. Use the + first approved cloud runs to prove crash recovery and result export, not just + a happy-path screenshot. +4. Produce offline reports and CI build artifacts; validate Web asset staging and + one tiny WASM model using the maintained Web E2E path. Keep sign/upload/run + actions separate from automatic artifact building. +5. Implement the GCE adapter and exercise its failure paths with fake + provider responses before VM use: uncertain creation, interrupted upload, + duplicate run invocation, timeout/cancellation, missing evidence, permission + failure during cleanup, a prematurely stopped VM and a mismatched resource + owner. Verify that unresolved cleanup blocks another VM. A separately + authorized short GCE run must prove upload, execution, result retrieval and + resource deletion before broad CUDA coverage; unavailable credit defers this + optional lane without blocking the owned-device/Firebase harness. +6. Run the selected Firebase rotation, then older-device CPU full/compact and + feature packs as quota permits. Qualify the optional S24 then Pixel 10 NPU + pack only after its model/library preflight and native reference succeed. + Fill every supported-platform row with exact + PASS/FAIL/ERROR/UNSUPPORTED/NOT_RUN evidence; never equate selected rotation + completion with whole-release qualification. + +The initial harness is usable when its selected bundle runs without the repository, +checks dependencies/models, exercises quick-core public APIs, records proven +backend use, survives independent assertion failures, retains useful crash +evidence, exports valid JSON/JUnit/CSV/basic HTML, and terminates bounded cloud +work. Subsequent milestones expand model/features/platforms; the initial milestone +does not qualify unrun rows. Known product failures remain visible. Full release +readiness still requires the repository's +existing review, platform matrix, affected-family and release gates. diff --git a/doc/testing_matrix.md b/doc/testing_matrix.md index f26126902..068a4c007 100644 --- a/doc/testing_matrix.md +++ b/doc/testing_matrix.md @@ -4,6 +4,13 @@ This repository uses a layered test matrix so contributors can validate the essential runtime, model, feature, and platform paths without forcing every pull request to run large local models or device-only checks. +The proposed [cross-platform validation plan](cross_platform_validation_plan.md) +details reusable test apps, model/backend cases, metrics, a free Firebase +physical-device rotation, and VM/Firebase upload, execution and cleanup. It is +planning work; implementation is deferred until +the current release additions are complete, and its proposed rows are not yet +implemented by the runners below. + Use the matrix for every non-trivial PR: ```bash @@ -401,3 +408,7 @@ When an agent creates or updates a PR: ancestry with `tool/git/safe_pr_head_update.dart` to prevent stale head rewinds. Consult `doc/pr_branch_writer_inventory.md` for writer scope and the remaining GitHub-managed governance boundary. + +The executable quick suite and provider commands are documented in the +[cross-platform validation runbook](cross_platform_validation.md). Discover the +model-free `validation-harness` and opt-in `validation-model-core` matrix rows. diff --git a/example/chat_app/android/app/build.gradle.kts b/example/chat_app/android/app/build.gradle.kts index e9b6b7e1b..9b3448055 100644 --- a/example/chat_app/android/app/build.gradle.kts +++ b/example/chat_app/android/app/build.gradle.kts @@ -28,6 +28,7 @@ android { targetSdk = flutter.targetSdkVersion versionCode = flutter.versionCode versionName = flutter.versionName + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" } buildTypes { @@ -42,3 +43,9 @@ android { flutter { source = "../.." } + +// Match the integration_test plugin bundled with the pinned Flutter SDK. +dependencies { + androidTestImplementation("androidx.test:runner:1.3.0") + androidTestImplementation("androidx.test:rules:1.2.0") +} diff --git a/example/chat_app/android/app/src/androidTest/java/com/example/llamadart_chat_example/MainActivityTest.java b/example/chat_app/android/app/src/androidTest/java/com/example/llamadart_chat_example/MainActivityTest.java new file mode 100644 index 000000000..e396d1daa --- /dev/null +++ b/example/chat_app/android/app/src/androidTest/java/com/example/llamadart_chat_example/MainActivityTest.java @@ -0,0 +1,14 @@ +package com.example.llamadart_chat_example; + +import androidx.test.rule.ActivityTestRule; +import dev.flutter.plugins.integration_test.FlutterTestRunner; +import org.junit.Rule; +import org.junit.runner.RunWith; + +/** Runs the selected Flutter integration entry point in Test Lab. */ +@RunWith(FlutterTestRunner.class) +public class MainActivityTest { + @Rule + public ActivityTestRule rule = + new ActivityTestRule<>(MainActivity.class, true, false); +} diff --git a/example/chat_app/android/gradle.properties b/example/chat_app/android/gradle.properties index fbee1d8cd..01636b9cf 100644 --- a/example/chat_app/android/gradle.properties +++ b/example/chat_app/android/gradle.properties @@ -1,2 +1,5 @@ org.gradle.jvmargs=-Xmx8G -XX:MaxMetaspaceSize=4G -XX:ReservedCodeCacheSize=512m -XX:+HeapDumpOnOutOfMemoryError android.useAndroidX=true +# Keep the existing Kotlin/Gradle DSL while using Flutter 3.47.1. +android.builtInKotlin=false +android.newDsl=false diff --git a/example/chat_app/integration_test/validation_test.dart b/example/chat_app/integration_test/validation_test.dart new file mode 100644 index 000000000..739ac6215 --- /dev/null +++ b/example/chat_app/integration_test/validation_test.dart @@ -0,0 +1,32 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; + +import 'package:llamadart_chat_example/validation/controller.dart'; + +void main() { + final binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + const profile = String.fromEnvironment( + 'VALIDATION_PROFILE', + defaultValue: 'tiny-gguf-cpu', + ); + testWidgets('shared validation: $profile', (tester) async { + await tester.pumpWidget( + const MaterialApp( + home: Scaffold(body: Text('Running llamadart validation')), + ), + ); + final controller = ValidationController(); + addTearDown(controller.dispose); + await controller.run(profile); + binding.reportData = + controller.report?.toJson() ?? {'error': controller.error}; + expect(controller.error, isNull); + expect( + controller.report?.assertionsPassed, + isTrue, + reason: + 'Functional obligations failed; accelerator qualification remains separate in the report.', + ); + }, timeout: const Timeout(Duration(minutes: 18))); +} diff --git a/example/chat_app/ios/Runner.xcodeproj/project.pbxproj b/example/chat_app/ios/Runner.xcodeproj/project.pbxproj index bfa87d3ce..3d20b9f3c 100644 --- a/example/chat_app/ios/Runner.xcodeproj/project.pbxproj +++ b/example/chat_app/ios/Runner.xcodeproj/project.pbxproj @@ -8,7 +8,7 @@ /* Begin PBXBuildFile section */ 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */ = {isa = PBXBuildFile; fileRef = 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */; }; - 331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 331C807B294A618700263BE5 /* RunnerTests.swift */; }; + 331C808B294A63AB00263BE5 /* RunnerTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 331C807B294A618700263BE5 /* RunnerTests.m */; }; 3B3967161E833CAA004F5970 /* AppFrameworkInfo.plist in Resources */ = {isa = PBXBuildFile; fileRef = 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */; }; 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 74858FAE1ED2DC5600515810 /* AppDelegate.swift */; }; 78A318202AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage in Frameworks */ = {isa = PBXBuildFile; productRef = 78A3181F2AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage */; }; @@ -43,7 +43,7 @@ /* Begin PBXFileReference section */ 1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GeneratedPluginRegistrant.h; sourceTree = ""; }; 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GeneratedPluginRegistrant.m; sourceTree = ""; }; - 331C807B294A618700263BE5 /* RunnerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RunnerTests.swift; sourceTree = ""; }; + 331C807B294A618700263BE5 /* RunnerTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = RunnerTests.m; sourceTree = ""; }; 331C8081294A63A400263BE5 /* RunnerTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = RunnerTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = AppFrameworkInfo.plist; path = Flutter/AppFrameworkInfo.plist; sourceTree = ""; }; 74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "Runner-Bridging-Header.h"; sourceTree = ""; }; @@ -81,7 +81,7 @@ 331C8082294A63A400263BE5 /* RunnerTests */ = { isa = PBXGroup; children = ( - 331C807B294A618700263BE5 /* RunnerTests.swift */, + 331C807B294A618700263BE5 /* RunnerTests.m */, ); path = RunnerTests; sourceTree = ""; @@ -278,7 +278,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - 331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */, + 331C808B294A63AB00263BE5 /* RunnerTests.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/example/chat_app/ios/RunnerTests/RunnerTests.m b/example/chat_app/ios/RunnerTests/RunnerTests.m new file mode 100644 index 000000000..6751b7516 --- /dev/null +++ b/example/chat_app/ios/RunnerTests/RunnerTests.m @@ -0,0 +1,35 @@ +@import XCTest; +@import integration_test; + +INTEGRATION_TEST_IOS_RUNNER(RunnerTests) + +// Flutter executes the Dart suite before materializing its XCTest results. +// Keep the latest run's bounded evidence even when its assertions fail. +@implementation RunnerTests (ValidationAttachments) +- (void)tearDown { + NSFileManager *files = NSFileManager.defaultManager; + NSURL *documents = [files URLsForDirectory:NSDocumentDirectory + inDomains:NSUserDomainMask].firstObject; + NSURL *runs = [documents URLByAppendingPathComponent:@"validation/runs" isDirectory:YES]; + NSArray *directories = [files contentsOfDirectoryAtURL:runs + includingPropertiesForKeys:nil options:NSDirectoryEnumerationSkipsHiddenFiles error:nil]; + NSURL *latest = [[directories sortedArrayUsingComparator:^NSComparisonResult(NSURL *a, NSURL *b) { + return [a.lastPathComponent compare:b.lastPathComponent]; + }] lastObject]; + if (latest != nil) { + for (NSURL *url in [files contentsOfDirectoryAtURL:latest + includingPropertiesForKeys:@[NSURLFileSizeKey] options:NSDirectoryEnumerationSkipsHiddenFiles error:nil]) { + NSNumber *size = nil; + [url getResourceValue:&size forKey:NSURLFileSizeKey error:nil]; + if (size != nil && size.unsignedLongLongValue <= 2 * 1024 * 1024 && + [@[@"json", @"jsonl", @"csv", @"xml", @"html"] containsObject:url.pathExtension]) { + XCTAttachment *attachment = [XCTAttachment attachmentWithContentsOfFileAtURL:url]; + attachment.name = url.lastPathComponent; + attachment.lifetime = XCTAttachmentLifetimeKeepAlways; + [self addAttachment:attachment]; + } + } + } + [super tearDown]; +} +@end diff --git a/example/chat_app/ios/RunnerTests/RunnerTests.swift b/example/chat_app/ios/RunnerTests/RunnerTests.swift deleted file mode 100644 index 86a7c3b1b..000000000 --- a/example/chat_app/ios/RunnerTests/RunnerTests.swift +++ /dev/null @@ -1,12 +0,0 @@ -import Flutter -import UIKit -import XCTest - -class RunnerTests: XCTestCase { - - func testExample() { - // If you add code to the Runner application, consider adding tests here. - // See https://developer.apple.com/documentation/xctest for more information about using XCTest. - } - -} diff --git a/example/chat_app/lib/validation/controller.dart b/example/chat_app/lib/validation/controller.dart new file mode 100644 index 000000000..a41bc403e --- /dev/null +++ b/example/chat_app/lib/validation/controller.dart @@ -0,0 +1,156 @@ +import 'dart:convert'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; + +import 'host.dart'; + +/// Coordinates the same suite for interactive and unattended Flutter runs. +class ValidationController extends ChangeNotifier { + /// Allows host injection in widget tests. + ValidationController({ValidationHost? host}) + : host = host ?? createValidationHost(); + + /// Native/browser storage adapter. + final ValidationHost host; + + /// Bounded current run progress. + final List> records = []; + + /// Whether a run is in progress. + bool running = false; + + /// Current case or preparation phase. + String phase = 'Ready'; + + /// Complete or partial report. + ValidationReport? report; + + /// Preparation/adapter failure description. + String? error; + ValidationRunner? _runner; + bool _cancelled = false; + bool _disposed = false; + void _notify() { + if (!_disposed) notifyListeners(); + } + + @override + void dispose() { + _disposed = true; + cancel(); + super.dispose(); + } + + /// Cancels inference and prevents the next case from starting. + void cancel() { + _cancelled = true; + host.cancelPreparation(); + _runner?.cancel(); + } + + /// Loads a checked-in profile and executes it once. + Future run(String profileId) async { + if (running) throw StateError('A validation run is already active'); + if (!RegExp(r'^[a-z][a-z0-9-]{0,63}$').hasMatch(profileId)) { + throw const FormatException('Invalid profile id'); + } + _cancelled = false; + running = true; + phase = 'Preparing model'; + error = null; + report = null; + records.clear(); + _notify(); + var started = false; + try { + final source = await rootBundle.loadString( + 'packages/llamadart_validation/assets/profiles/$profileId.json', + ); + final profile = ValidationProfile.fromJson( + jsonDecode(source) as Map, + ); + if (_cancelled) { + throw StateError('Run cancelled before model preparation'); + } + final runId = 'app-${DateTime.now().toUtc().microsecondsSinceEpoch}'; + await host.start(runId); + started = true; + if (_cancelled) { + throw StateError('Run cancelled before model preparation'); + } + final model = await host.prepare(profile); + if (_cancelled) throw StateError('Run cancelled during preparation'); + _runner = ValidationRunner( + profile: profile, + engine: PublicValidationEngine(), + emit: (event) async { + await host.emit(event); + if (event['type'] == 'case' || event['type'] == 'case_start') { + phase = '${event['case_id']} ${event['status'] ?? 'running'}'; + if (event['type'] == 'case') records.add(event); + _notify(); + } + }, + ); + await _runner!.run( + model.path, + runId: runId, + preparation: model.evidence, + environment: { + 'platform': defaultTargetPlatform.name, + 'web': kIsWeb, + 'build_mode': kReleaseMode + ? 'release' + : kProfileMode + ? 'profile' + : 'debug', + 'source_commit': const String.fromEnvironment( + 'VALIDATION_COMMIT', + defaultValue: 'unknown', + ), + 'source_dirty': const bool.fromEnvironment( + 'VALIDATION_SOURCE_DIRTY', + defaultValue: true, + ), + 'hook_sha256': const String.fromEnvironment( + 'VALIDATION_HOOK_SHA256', + defaultValue: 'unknown', + ), + 'bridge_tag': const String.fromEnvironment( + 'VALIDATION_BRIDGE_TAG', + defaultValue: 'unknown', + ), + 'native_tag': const String.fromEnvironment( + 'VALIDATION_NATIVE_TAG', + defaultValue: 'unknown', + ), + 'litert_tag': const String.fromEnvironment( + 'VALIDATION_LITERT_TAG', + defaultValue: 'unknown', + ), + }, + ); + } catch (exception) { + error = redactDiagnostic('$exception'); + if (started) { + await host.emit({'type': 'preparation_error', 'message': error}); + } + } finally { + try { + if (started) report = await host.finish(); + } catch (exception) { + error = redactDiagnostic('Report persistence failed: $exception'); + } + running = false; + _runner = null; + phase = error != null + ? 'Error' + : report?.qualified == true + ? 'Qualified' + : 'Incomplete / failed'; + _notify(); + } + } +} diff --git a/example/chat_app/lib/validation/host.dart b/example/chat_app/lib/validation/host.dart new file mode 100644 index 000000000..5d800d010 --- /dev/null +++ b/example/chat_app/lib/validation/host.dart @@ -0,0 +1,34 @@ +import 'package:llamadart_validation/llamadart_validation.dart'; + +import 'host_native.dart' + if (dart.library.js_interop) 'host_web.dart' + as platform; + +/// Platform storage and model preparation for the shared validation runner. +abstract interface class ValidationHost { + /// Prepares and verifies the locked model before inference. + Future<({String path, Map evidence})> prepare( + ValidationProfile profile, + ); + + /// Cancels an in-progress download. + void cancelPreparation(); + + /// Opens a unique incremental result journal. + Future start(String runId); + + /// Persists a complete event and crash-recovery breadcrumb. + Future emit(Map event); + + /// Closes the journal and derives reports from it. + Future finish(); + + /// Exports one report to device storage or a browser download. + Future export(String name, String text); + + /// Describes where the host keeps result files. + String get outputLocation; +} + +/// Selects native filesystem or browser storage without importing IO on Web. +ValidationHost createValidationHost() => platform.createHost(); diff --git a/example/chat_app/lib/validation/host_native.dart b/example/chat_app/lib/validation/host_native.dart new file mode 100644 index 000000000..203163eb0 --- /dev/null +++ b/example/chat_app/lib/validation/host_native.dart @@ -0,0 +1,78 @@ +import 'dart:io'; +import 'dart:convert'; +import 'package:http/http.dart' as http; +import 'dart:typed_data'; + +import 'package:file_picker/file_picker.dart'; +import 'package:llamadart_validation/io.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:path/path.dart' as p; +import 'package:path_provider/path_provider.dart'; + +import 'host.dart'; + +/// Creates the mobile/desktop Flutter filesystem host. +ValidationHost createHost() => _NativeHost(); + +class _NativeHost implements ValidationHost { + FileValidationJournal? _journal; + Directory? _output; + http.Client? _client; + @override + void cancelPreparation() => _client?.close(); + + @override + String get outputLocation => _output?.path ?? ''; + + @override + Future<({String path, Map evidence})> prepare( + ValidationProfile profile, + ) async { + _client = http.Client(); + try { + return await prepareModel( + profile, + Directory( + p.join( + (await getApplicationSupportDirectory()).path, + 'validation', + 'models', + ), + ), + client: _client, + ); + } finally { + _client?.close(); + _client = null; + } + } + + @override + Future start(String runId) async { + final base = Platform.isAndroid + ? await getExternalStorageDirectory() + : await getApplicationDocumentsDirectory(); + if (base == null) { + throw StateError('No validation output directory available'); + } + _output = Directory(p.join(base.path, 'validation', 'runs', runId)); + _journal = FileValidationJournal(_output!); + } + + @override + Future emit(Map event) => _journal!.emit(event); + + @override + Future finish() async { + _journal!.close(); + return writeReports(_output!); + } + + @override + Future export(String name, String text) async { + await FilePicker.platform.saveFile( + fileName: name, + bytes: Uint8List.fromList(utf8.encode(text)), + ); + } +} diff --git a/example/chat_app/lib/validation/host_web.dart b/example/chat_app/lib/validation/host_web.dart new file mode 100644 index 000000000..cb8b3ec57 --- /dev/null +++ b/example/chat_app/lib/validation/host_web.dart @@ -0,0 +1,104 @@ +import 'dart:convert'; +import 'dart:async'; +import 'dart:typed_data'; +import 'dart:js_interop'; + +import 'package:crypto/crypto.dart'; +import 'package:http/http.dart' as http; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:web/web.dart' as web; + +import 'host.dart'; + +/// Creates the browser host, which exports its in-memory result journal. +ValidationHost createHost() => _WebHost(); + +class _WebHost implements ValidationHost { + final _lines = []; + http.Client? _client; + @override + void cancelPreparation() => _client?.close(); + @override + String get outputLocation => 'Browser memory; export before closing the tab'; + + @override + Future<({String path, Map evidence})> prepare( + ValidationProfile profile, + ) async { + if (profile.runtime == 'litert') { + throw UnsupportedError( + 'Native LiteRT fixtures are not browser bundles. Supply a separately qualified Web profile.', + ); + } + final watch = Stopwatch()..start(); + final client = _client = http.Client(); + final timer = Timer(const Duration(minutes: 5), client.close); + try { + final response = await client.send( + http.Request('GET', Uri.parse(profile.model['url'] as String)), + ); + if (response.statusCode != 200) { + throw StateError('Model download HTTP mismatch'); + } + final bytes = BytesBuilder(copy: false); + await for (final chunk in response.stream) { + if (bytes.length + chunk.length > (profile.model['bytes'] as int)) { + throw StateError('Model size exceeded'); + } + bytes.add(chunk); + } + if (bytes.length != profile.model['bytes'] || + sha256.convert(bytes.takeBytes()).toString() != profile.modelHash) { + throw const FormatException('Model checksum/size mismatch'); + } + } finally { + timer.cancel(); + client.close(); + _client = null; + } + // The immutable URL retains the .gguf routing suffix. The bridge owns its + // URL cache; report that second transfer separately from native file reuse. + return ( + path: profile.model['url'] as String, + evidence: { + 'sha256': profile.modelHash, + 'bytes': profile.model['bytes'], + 'verified': true, + 'total_ms': watch.elapsedMilliseconds, + 'bridge_transfer': 'immutable URL; bridge may transfer bytes again', + }, + ); + } + + @override + Future start(String runId) async { + _lines.clear(); + } + + @override + Future emit(Map event) async { + final line = jsonEncode(event); + _lines.add(line); + if (line.length <= 32768) { + web.console.log('LLAMADART_VALIDATION $line'.toJS); + } + } + + @override + Future finish() async => + ValidationReport.parse(_lines.join('\n')); + + @override + Future export(String name, String text) async { + final blob = web.Blob( + [text.toJS].toJS, + web.BlobPropertyBag(type: 'application/octet-stream'), + ); + final url = web.URL.createObjectURL(blob); + final anchor = web.document.createElement('a') as web.HTMLAnchorElement + ..href = url + ..download = name; + anchor.click(); + Timer(const Duration(seconds: 1), () => web.URL.revokeObjectURL(url)); + } +} diff --git a/example/chat_app/lib/validation_main.dart b/example/chat_app/lib/validation_main.dart new file mode 100644 index 000000000..304b26f30 --- /dev/null +++ b/example/chat_app/lib/validation_main.dart @@ -0,0 +1,146 @@ +import 'dart:convert'; + +import 'package:flutter/material.dart'; + +import 'validation/controller.dart'; + +void main() => runApp(const ValidationApp()); + +/// Small interactive entry point for the shared cross-platform test suite. +class ValidationApp extends StatefulWidget { + /// Creates the QA app independently of the normal chat application. + const ValidationApp({super.key}); + + @override + State createState() => _ValidationAppState(); +} + +class _ValidationAppState extends State { + final _controller = ValidationController(); + String _profile = const String.fromEnvironment( + 'VALIDATION_PROFILE', + defaultValue: 'tiny-gguf-cpu', + ); + @override + void dispose() { + _controller.cancel(); + _controller.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) => MaterialApp( + title: 'llamadart validation', + theme: ThemeData( + colorSchemeSeed: const Color(0xff247a87), + useMaterial3: true, + ), + home: AnimatedBuilder( + animation: _controller, + builder: (context, _) => Scaffold( + appBar: AppBar(title: const Text('llamadart validation')), + body: Center( + child: ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 850), + child: ListView( + padding: const EdgeInsets.all(24), + children: [ + const Text( + 'Run the same package checks on this device.', + style: TextStyle(fontSize: 22), + ), + const SizedBox(height: 16), + DropdownButtonFormField( + initialValue: _profile, + decoration: const InputDecoration( + labelText: 'Model / backend profile', + ), + items: [ + for (final id in const [ + 'tiny-gguf-cpu', + 'tiny-gguf-metal', + 'tiny-gguf-vulkan', + 'tiny-gguf-cuda', + 'chat-gguf-cpu', + 'chat-gguf-metal', + 'chat-gguf-vulkan', + 'chat-gguf-cuda', + 'chat-litert-cpu', + 'chat-litert-gpu', + ]) + DropdownMenuItem(value: id, child: Text(id)), + ], + onChanged: _controller.running + ? null + : (value) => setState(() => _profile = value!), + ), + const SizedBox(height: 16), + Wrap( + spacing: 12, + runSpacing: 12, + children: [ + FilledButton( + onPressed: _controller.running + ? null + : () => _controller.run(_profile), + child: const Text('Run tests'), + ), + OutlinedButton( + onPressed: _controller.running + ? _controller.cancel + : null, + child: const Text('Cancel'), + ), + OutlinedButton( + onPressed: _controller.report == null + ? null + : () => _controller.host.export( + 'summary.html', + _controller.report!.toHtml(), + ), + child: const Text('Export report'), + ), + OutlinedButton( + onPressed: _controller.report == null + ? null + : () => _controller.host.export( + 'results.json', + jsonEncode(_controller.report!.toJson()), + ), + child: const Text('Export JSON'), + ), + ], + ), + const SizedBox(height: 20), + if (_controller.running) const LinearProgressIndicator(), + Text( + _controller.phase, + style: Theme.of(context).textTheme.titleLarge, + ), + if (_controller.error != null) + SelectableText(_controller.error!), + if (_controller.report != null) ...[ + Text( + 'Assertions: ${_controller.report!.assertionsPassed ? 'passed' : 'failed or incomplete'}', + ), + Text( + 'Accelerator execution: ${_controller.report!.acceleratorVerified ? 'not required' : 'unverified; inspect native evidence'}', + ), + SelectableText(_controller.host.outputLocation), + ], + for (final record in _controller.records) + ListTile( + contentPadding: EdgeInsets.zero, + title: Text('${record['case_id']} · ${record['status']}'), + subtitle: Text( + '${record['reason'] ?? record['expected'] ?? ''}', + ), + ), + ], + ), + ), + ), + ), + ), + ); +} diff --git a/example/chat_app/pubspec.lock b/example/chat_app/pubspec.lock index 6a7db30fc..381fb1d06 100644 --- a/example/chat_app/pubspec.lock +++ b/example/chat_app/pubspec.lock @@ -331,7 +331,7 @@ packages: source: hosted version: "1.0.1" http: - dependency: "direct dev" + dependency: "direct main" description: name: http sha256: "87721a4a50b19c7f1d49001e51409bddc46303966ce89a65af4f4e6004896412" @@ -428,6 +428,13 @@ packages: relative: true source: path version: "0.0.18" + llamadart_validation: + dependency: "direct main" + description: + path: "../../packages/llamadart_validation" + relative: true + source: path + version: "0.0.0" logging: dependency: transitive description: diff --git a/example/chat_app/pubspec.yaml b/example/chat_app/pubspec.yaml index 93a27483d..2daafbc3f 100644 --- a/example/chat_app/pubspec.yaml +++ b/example/chat_app/pubspec.yaml @@ -12,6 +12,9 @@ dependencies: sdk: flutter llamadart: path: ../.. # Use local version + llamadart_validation: + path: ../../packages/llamadart_validation + http: ^1.1.0 llamadart_llama_cpp_flutter: path: ../../packages/llamadart_llama_cpp_flutter llamadart_litert_lm_flutter: @@ -45,7 +48,6 @@ dev_dependencies: test: ^1.26.3 ffi: any - http: any flutter: uses-material-design: true diff --git a/example/chat_app/test/validation_controller_test.dart b/example/chat_app/test/validation_controller_test.dart new file mode 100644 index 000000000..58023d99c --- /dev/null +++ b/example/chat_app/test/validation_controller_test.dart @@ -0,0 +1,86 @@ +import 'dart:async'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:llamadart_chat_example/validation/controller.dart'; +import 'package:llamadart_chat_example/validation/host.dart'; + +class PreparationHost implements ValidationHost { + final started = Completer(); + final pending = Completer<({String path, Map evidence})>(); + bool cancelled = false; + bool finishFails = false; + @override + String get outputLocation => 'test'; + @override + Future start(String id) async {} + @override + Future<({String path, Map evidence})> prepare( + ValidationProfile profile, + ) { + started.complete(); + return pending.future; + } + + @override + void cancelPreparation() { + cancelled = true; + if (!pending.isCompleted) { + pending.completeError(StateError('cancelled download')); + } + } + + @override + Future emit(Map event) async {} + @override + Future finish() async { + if (finishFails) throw StateError('disk full'); + return ValidationReport.parse(''); + } + + @override + Future export(String name, String text) async {} +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + test( + 'cancelling model preparation restores idle and retains incomplete report', + () async { + final host = PreparationHost(); + final controller = ValidationController(host: host); + final run = controller.run('tiny-gguf-cpu'); + await host.started.future; + expect(controller.running, true); + controller.cancel(); + await run; + expect(host.cancelled, true); + expect(controller.running, false); + expect(controller.report?.qualified, false); + controller.dispose(); + }, + ); + test( + 'disposing during preparation cannot notify a disposed controller', + () async { + final host = PreparationHost(); + final controller = ValidationController(host: host); + final run = controller.run('tiny-gguf-cpu'); + await host.started.future; + controller.dispose(); + await run; + expect(controller.running, false); + }, + ); + test('failed report persistence still releases running state', () async { + final host = PreparationHost()..finishFails = true; + final controller = ValidationController(host: host); + final run = controller.run('tiny-gguf-cpu'); + await host.started.future; + controller.cancel(); + await run; + expect(controller.running, false); + expect(controller.error, contains('Report persistence failed')); + controller.dispose(); + }); +} diff --git a/packages/llamadart_validation/README.md b/packages/llamadart_validation/README.md new file mode 100644 index 000000000..3d6341b1e --- /dev/null +++ b/packages/llamadart_validation/README.md @@ -0,0 +1,29 @@ +# llamadart validation + +Private maintainer package (`publish_to: none`). It exercises the exported +llamadart API through the same core on desktop, Flutter mobile, and Web. +It is independent of the public package's implementation and exports. + +Start from the repository root: + +```sh +dart run tool/testing/run_local_e2e.dart --scenario validation-harness +dart run tool/testing/validation.dart local --profile tiny-gguf-cpu +dart run tool/testing/validation.dart build --target desktop --out .dart_tool/validation/bundles/desktop +``` + +See [the runbook](../../doc/cross_platform_validation.md) for models, bundles, +Firebase/GCE setup, run/recovery commands, evidence and cost boundaries. +See [the full plan](../../doc/cross_platform_validation_plan.md) for subsequent +platform and feature qualification. A quick run does not qualify a release. + +`assets/profiles/` locks each model URL/revision/SHA256/size and inference +configuration. `lib/` has no Flutter or filesystem dependency; `lib/io.dart` +is the native filesystem adapter. `bin/run.dart` and `bin/report.dart` are the +CLI entrypoints. Flutter uses the package's profile assets and shared runner. +Native wrappers persist JSONL per event; reports always derive from that journal. + +Exit 0 means this selected run qualified, 1 means failed/incomplete. +Unknown/dirty source provenance, missing counters, unknown backend placement, skipped mandatory cases and missing +records remain incomplete. `release` selection deliberately records unimplemented +feature packs as NOT_RUN until their fixtures and wrappers are qualified. diff --git a/packages/llamadart_validation/analysis_options.yaml b/packages/llamadart_validation/analysis_options.yaml new file mode 100644 index 000000000..572dd239d --- /dev/null +++ b/packages/llamadart_validation/analysis_options.yaml @@ -0,0 +1 @@ +include: package:lints/recommended.yaml diff --git a/packages/llamadart_validation/assets/profiles/chat-gguf-cpu.json b/packages/llamadart_validation/assets/profiles/chat-gguf-cpu.json new file mode 100644 index 000000000..ede7812c4 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-gguf-cpu.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "id": "chat-gguf-cpu", + "runtime": "gguf", + "backend": "cpu", + "model": { + "kind": "chat", + "filename": "Qwen3.5-0.8B-Q4_0.gguf", + "revision": "8fea620810c4afa23dd6443f999a48574c1611a3", + "url": "https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/resolve/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q4_0.gguf", + "sha256": "57d1997790d1744fba5b40a7317df71ea5e2acee28c47e78f0cce39c0703f8cf", + "bytes": 563036064 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/chat-gguf-cuda.json b/packages/llamadart_validation/assets/profiles/chat-gguf-cuda.json new file mode 100644 index 000000000..95791ae37 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-gguf-cuda.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "id": "chat-gguf-cuda", + "runtime": "gguf", + "backend": "cuda", + "model": { + "kind": "chat", + "filename": "Qwen3.5-0.8B-Q4_0.gguf", + "revision": "8fea620810c4afa23dd6443f999a48574c1611a3", + "url": "https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/resolve/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q4_0.gguf", + "sha256": "57d1997790d1744fba5b40a7317df71ea5e2acee28c47e78f0cce39c0703f8cf", + "bytes": 563036064 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/chat-gguf-metal.json b/packages/llamadart_validation/assets/profiles/chat-gguf-metal.json new file mode 100644 index 000000000..31c999b6e --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-gguf-metal.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "id": "chat-gguf-metal", + "runtime": "gguf", + "backend": "metal", + "model": { + "kind": "chat", + "filename": "Qwen3.5-0.8B-Q4_0.gguf", + "revision": "8fea620810c4afa23dd6443f999a48574c1611a3", + "url": "https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/resolve/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q4_0.gguf", + "sha256": "57d1997790d1744fba5b40a7317df71ea5e2acee28c47e78f0cce39c0703f8cf", + "bytes": 563036064 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/chat-gguf-vulkan.json b/packages/llamadart_validation/assets/profiles/chat-gguf-vulkan.json new file mode 100644 index 000000000..88de6b177 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-gguf-vulkan.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "id": "chat-gguf-vulkan", + "runtime": "gguf", + "backend": "vulkan", + "model": { + "kind": "chat", + "filename": "Qwen3.5-0.8B-Q4_0.gguf", + "revision": "8fea620810c4afa23dd6443f999a48574c1611a3", + "url": "https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/resolve/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q4_0.gguf", + "sha256": "57d1997790d1744fba5b40a7317df71ea5e2acee28c47e78f0cce39c0703f8cf", + "bytes": 563036064 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/chat-litert-cpu.json b/packages/llamadart_validation/assets/profiles/chat-litert-cpu.json new file mode 100644 index 000000000..636a1c06c --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-litert-cpu.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "chat-litert-cpu", + "runtime": "litert", + "backend": "cpu", + "model": { + "id": "chat-litert", + "revision": "8414150f2e9dcc82449bcc9c5abc404b399a4d06", + "filename": "Qwen3-0.6B.litertlm", + "sha256": "555579ff2f4fd13379abe69c1c3ab5200f7338bc92471557f1d6614a6e5ab0b4", + "bytes": 614236160, + "kind": "chat", + "url": "https://huggingface.co/litert-community/Qwen3-0.6B/resolve/8414150f2e9dcc82449bcc9c5abc404b399a4d06/Qwen3-0.6B.litertlm" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/chat-litert-gpu.json b/packages/llamadart_validation/assets/profiles/chat-litert-gpu.json new file mode 100644 index 000000000..d5d9a995b --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-litert-gpu.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "chat-litert-gpu", + "runtime": "litert", + "backend": "gpu", + "model": { + "id": "chat-litert", + "revision": "8414150f2e9dcc82449bcc9c5abc404b399a4d06", + "filename": "Qwen3-0.6B.litertlm", + "sha256": "555579ff2f4fd13379abe69c1c3ab5200f7338bc92471557f1d6614a6e5ab0b4", + "bytes": 614236160, + "kind": "chat", + "url": "https://huggingface.co/litert-community/Qwen3-0.6B/resolve/8414150f2e9dcc82449bcc9c5abc404b399a4d06/Qwen3-0.6B.litertlm" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/tiny-gguf-cpu.json b/packages/llamadart_validation/assets/profiles/tiny-gguf-cpu.json new file mode 100644 index 000000000..0951983f6 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/tiny-gguf-cpu.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "tiny-gguf-cpu", + "runtime": "gguf", + "backend": "cpu", + "model": { + "id": "tiny-gguf", + "revision": "99dd1a73db5a37100bd4ae633f4cfce6560e1567", + "filename": "stories15M.gguf", + "sha256": "61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a", + "bytes": 98357920, + "kind": "raw", + "url": "https://huggingface.co/ggml-org/tiny-llamas/resolve/99dd1a73db5a37100bd4ae633f4cfce6560e1567/stories15M.gguf" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + }, + "unicode": { + "expected_prefix": " " + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/tiny-gguf-cuda.json b/packages/llamadart_validation/assets/profiles/tiny-gguf-cuda.json new file mode 100644 index 000000000..36d998b21 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/tiny-gguf-cuda.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "tiny-gguf-cuda", + "runtime": "gguf", + "backend": "cuda", + "model": { + "id": "tiny-gguf", + "revision": "99dd1a73db5a37100bd4ae633f4cfce6560e1567", + "filename": "stories15M.gguf", + "sha256": "61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a", + "bytes": 98357920, + "kind": "raw", + "url": "https://huggingface.co/ggml-org/tiny-llamas/resolve/99dd1a73db5a37100bd4ae633f4cfce6560e1567/stories15M.gguf" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + }, + "unicode": { + "expected_prefix": " " + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/tiny-gguf-metal.json b/packages/llamadart_validation/assets/profiles/tiny-gguf-metal.json new file mode 100644 index 000000000..14767c406 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/tiny-gguf-metal.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "tiny-gguf-metal", + "runtime": "gguf", + "backend": "metal", + "model": { + "id": "tiny-gguf", + "revision": "99dd1a73db5a37100bd4ae633f4cfce6560e1567", + "filename": "stories15M.gguf", + "sha256": "61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a", + "bytes": 98357920, + "kind": "raw", + "url": "https://huggingface.co/ggml-org/tiny-llamas/resolve/99dd1a73db5a37100bd4ae633f4cfce6560e1567/stories15M.gguf" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + }, + "unicode": { + "expected_prefix": " " + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/tiny-gguf-vulkan.json b/packages/llamadart_validation/assets/profiles/tiny-gguf-vulkan.json new file mode 100644 index 000000000..371d0e0ff --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/tiny-gguf-vulkan.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "tiny-gguf-vulkan", + "runtime": "gguf", + "backend": "vulkan", + "model": { + "id": "tiny-gguf", + "revision": "99dd1a73db5a37100bd4ae633f4cfce6560e1567", + "filename": "stories15M.gguf", + "sha256": "61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a", + "bytes": 98357920, + "kind": "raw", + "url": "https://huggingface.co/ggml-org/tiny-llamas/resolve/99dd1a73db5a37100bd4ae633f4cfce6560e1567/stories15M.gguf" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + }, + "unicode": { + "expected_prefix": " " + } + } +} diff --git a/packages/llamadart_validation/bin/report.dart b/packages/llamadart_validation/bin/report.dart new file mode 100644 index 000000000..4435469ef --- /dev/null +++ b/packages/llamadart_validation/bin/report.dart @@ -0,0 +1,23 @@ +import 'dart:io'; + +import 'package:llamadart_validation/io.dart'; + +void main(List args) { + if (args.length != 1 && !(args.length == 3 && args[1] == '--native-log')) { + stderr.writeln('Usage: report [--native-log ]'); + exitCode = 64; + return; + } + try { + exitCode = + writeReports( + Directory(args.first), + nativeLog: args.length == 3 ? File(args[2]).readAsStringSync() : null, + ).qualified + ? 0 + : 1; + } catch (error) { + stderr.writeln(error); + exitCode = 1; + } +} diff --git a/packages/llamadart_validation/bin/run.dart b/packages/llamadart_validation/bin/run.dart new file mode 100644 index 000000000..1de96c5ac --- /dev/null +++ b/packages/llamadart_validation/bin/run.dart @@ -0,0 +1,128 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:llamadart_validation/io.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:path/path.dart' as p; + +Future main(List args) async { + try { + final options = parseOptions(args, { + 'help', + 'list', + 'assets', + 'profile', + 'profile-file', + 'model', + 'cache', + 'out', + 'run-id', + 'environment-file', + }); + final assets = + options['assets'] ?? + p.join(File.fromUri(Platform.script).parent.parent.path, 'assets'); + if (options.containsKey('help')) { + stdout.writeln( + 'llamadart-validate --profile [--model ] ' + '[--out ] [--cache ] [--assets ]\n' + '--list lists bundled profiles. --profile-file loads an explicit locked manifest.', + ); + return; + } + if (options.containsKey('list')) { + for (final file in Directory( + p.join(assets, 'profiles'), + ).listSync().whereType()) { + if (file.path.endsWith('.json')) { + stdout.writeln(p.basenameWithoutExtension(file.path)); + } + } + return; + } + final profileId = options['profile'] ?? 'tiny-gguf-cpu'; + if (!RegExp(r'^[a-z][a-z0-9-]{0,63}$').hasMatch(profileId)) { + throw const FormatException('Invalid profile id'); + } + final profile = ValidationProfile.fromJson( + jsonDecode( + File( + options['profile-file'] ?? + p.join(assets, 'profiles', '$profileId.json'), + ).readAsStringSync(), + ) + as Map, + ); + final runId = + options['run-id'] ?? + 'local-${DateTime.now().toUtc().microsecondsSinceEpoch}'; + final directory = Directory( + options['out'] ?? p.join('.dart_tool', 'validation', 'runs', runId), + ); + final journal = FileValidationJournal(directory); + try { + final prepared = await prepareModel( + profile, + Directory( + options['cache'] ?? p.join('.dart_tool', 'validation', 'model-cache'), + ), + suppliedPath: options['model'], + ); + final bundledEnvironment = File( + p.join( + File(Platform.resolvedExecutable).parent.parent.path, + 'environment.json', + ), + ); + final environmentPath = + options['environment-file'] ?? + (bundledEnvironment.existsSync() ? bundledEnvironment.path : null); + final environment = { + if (environmentPath != null) + ...jsonDecode(File(environmentPath).readAsStringSync()) + as Map, + 'os': Platform.operatingSystem, + 'os_version': Platform.operatingSystemVersion, + 'processors': Platform.numberOfProcessors, + 'dart': Platform.version, + 'build_mode': const bool.fromEnvironment('dart.vm.product') + ? 'release' + : 'jit', + }; + final runner = ValidationRunner( + profile: profile, + engine: PublicValidationEngine(), + emit: journal.emit, + ); + final interrupt = ProcessSignal.sigint.watch().listen( + (_) => runner.cancel(), + ); + try { + await runner.run( + prepared.path, + runId: runId, + environment: environment, + preparation: prepared.evidence, + ); + } finally { + await interrupt.cancel(); + } + } catch (error) { + await journal.emit({ + 'type': 'preparation_error', + 'message': redactDiagnostic('$error'), + }); + rethrow; + } finally { + journal.close(); + final report = writeReports(directory); + stdout.writeln( + 'REPORT ${p.join(directory.absolute.path, 'summary.html')}', + ); + if (!report.qualified) exitCode = 1; + } + } catch (error) { + stderr.writeln(redactDiagnostic('$error')); + exitCode = 1; + } +} diff --git a/packages/llamadart_validation/lib/io.dart b/packages/llamadart_validation/lib/io.dart new file mode 100644 index 000000000..dd8b47cfe --- /dev/null +++ b/packages/llamadart_validation/lib/io.dart @@ -0,0 +1,177 @@ +/// Native filesystem adapters; never imported by the shared/browser suite. +library; + +import 'dart:convert'; +import 'dart:async'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:http/http.dart' as http; +import 'package:path/path.dart' as p; + +import 'llamadart_validation.dart'; + +/// Verifies model bytes before reuse or load; download timing is separate. +Future<({String path, Map evidence})> prepareModel( + ValidationProfile profile, + Directory cache, { + String? suppliedPath, + Duration timeout = const Duration(minutes: 5), + http.Client? client, +}) async { + final started = Stopwatch()..start(); + final target = suppliedPath == null + ? File(p.join(cache.path, profile.modelHash, profile.filename)) + : File(suppliedPath); + var hit = target.existsSync(); + Future valid(File file) async => + await file.length() == profile.model['bytes'] && + (await sha256.bind(file.openRead()).first).toString() == + profile.modelHash; + final checksum = Stopwatch()..start(); + if (hit && !await valid(target)) { + if (suppliedPath != null) { + throw const FormatException('Supplied model hash/size mismatch'); + } + await target.delete(); + hit = false; + } + checksum.stop(); + var downloadMs = 0; + if (!hit) { + if (suppliedPath != null) { + throw const FileSystemException('Supplied model is missing'); + } + await target.parent.create(recursive: true); + final temporary = File( + '${target.path}.${DateTime.now().microsecondsSinceEpoch}.part', + ); + final transport = client ?? http.Client(); + final timer = Timer(timeout, transport.close); + IOSink? sink; + final watch = Stopwatch()..start(); + try { + await (() async { + final response = await transport.send( + http.Request('GET', Uri.parse(profile.model['url'] as String)), + ); + if (response.statusCode != 200) { + throw HttpException('Model download HTTP ${response.statusCode}'); + } + sink = temporary.openWrite(); + var bytes = 0; + await for (final chunk in response.stream) { + bytes += chunk.length; + if (bytes > (profile.model['bytes'] as int)) { + throw const FormatException( + 'Model download exceeded locked byte size', + ); + } + sink!.add(chunk); + } + await sink!.flush(); + await sink!.close(); + sink = null; + })(); + downloadMs = watch.elapsedMilliseconds; + checksum.start(); + if (!await valid(temporary)) { + throw const FormatException('Downloaded model hash/size mismatch'); + } + checksum.stop(); + await temporary.rename(target.path); + } finally { + timer.cancel(); + transport.close(); + await sink?.close(); + if (temporary.existsSync()) await temporary.delete(); + } + } + return ( + path: target.absolute.path, + evidence: { + 'sha256': profile.modelHash, + 'bytes': profile.model['bytes'], + 'verified': true, + 'cache_hit': hit, + 'download_ms': downloadMs, + 'checksum_ms': checksum.elapsedMilliseconds, + 'total_ms': started.elapsedMilliseconds, + }, + ); +} + +/// Writes incremental records synchronously enough to survive a native crash. +class FileValidationJournal { + FileValidationJournal(this.directory) { + directory.createSync(recursive: true); + final file = File(p.join(directory.path, 'events.jsonl')); + if (file.existsSync()) { + throw StateError('Run output already exists; use a new run/attempt'); + } + _file = file.openSync(mode: FileMode.write); + } + final Directory directory; + late final RandomAccessFile _file; + + Future emit(Map event) async { + final line = jsonEncode(event); + _file.writeStringSync('$line\n'); + _file.flushSync(); + if (event['type'] == 'manifest') { + File( + p.join(directory.path, 'manifest.json'), + ).writeAsStringSync(line, flush: true); + } + // Complete, bounded records provide a provider-log fallback. + if (utf8.encode(line).length <= 32768) { + stdout.writeln('LLAMADART_VALIDATION $line'); + } + } + + void close() => _file.closeSync(); +} + +/// Recomputes every report format from an existing run journal. +ValidationReport writeReports(Directory directory, {String? nativeLog}) { + final report = ValidationReport.parse( + File(p.join(directory.path, 'events.jsonl')).readAsStringSync(), + nativeLog: nativeLog, + ); + for (final entry in { + 'results.json': const JsonEncoder.withIndent(' ').convert(report.toJson()), + 'junit.xml': report.toJUnit(), + 'samples.csv': report.toCsv(), + 'summary.html': report.toHtml(), + }.entries) { + File(p.join(directory.path, entry.key)).writeAsStringSync(entry.value); + } + return report; +} + +/// Simple strict long-option parser shared by the private CLI entry points. +Map parseOptions(List args, Set allowed) { + final values = {}; + for (var i = 0; i < args.length; i++) { + final arg = args[i]; + if (!arg.startsWith('--')) { + throw FormatException('Expected option, got $arg'); + } + final parts = arg.substring(2).split('='); + final key = parts.first; + if (!allowed.contains(key) || values.containsKey(key)) { + throw FormatException('Unknown/duplicate option --$key'); + } + if (key == 'help' || key == 'list') { + values[key] = 'true'; + } else if (parts.length > 1) { + values[key] = parts.skip(1).join('='); + } else { + if (++i >= args.length || args[i].startsWith('--')) { + throw FormatException('Missing --$key value'); + } + values[key] = args[i]; + } + } + return values; +} diff --git a/packages/llamadart_validation/lib/llamadart_validation.dart b/packages/llamadart_validation/lib/llamadart_validation.dart new file mode 100644 index 000000000..c8b024a0d --- /dev/null +++ b/packages/llamadart_validation/lib/llamadart_validation.dart @@ -0,0 +1,6 @@ +/// Private, platform-neutral public-package validation and report contracts. +library; + +export 'src/manifest.dart'; +export 'src/runner.dart'; +export 'src/report.dart'; diff --git a/packages/llamadart_validation/lib/src/manifest.dart b/packages/llamadart_validation/lib/src/manifest.dart new file mode 100644 index 000000000..df943a13c --- /dev/null +++ b/packages/llamadart_validation/lib/src/manifest.dart @@ -0,0 +1,212 @@ +import 'dart:convert'; + +import 'package:crypto/crypto.dart'; +import 'package:llamadart/llamadart.dart'; + +/// Canonical encoding for stable experiment identity. +String canonicalJson(Object? value) { + Object? ordered(Object? value) { + if (value is Map) { + final keys = value.keys.cast().toList()..sort(); + return {for (final key in keys) key: ordered(value[key])}; + } + if (value is List) return value.map(ordered).toList(); + return value; + } + + return jsonEncode(ordered(value)); +} + +/// SHA256 identity of a JSON-compatible value. +Object? freezeJson(Object? value) => value is Map + ? Map.unmodifiable( + value.map((key, item) => MapEntry(key as String, freezeJson(item))), + ) + : value is List + ? List.unmodifiable(value.map(freezeJson)) + : value; + +/// SHA256 identity of a JSON-compatible value. +String jsonHash(Object? value) => + sha256.convert(utf8.encode(canonicalJson(value))).toString(); + +/// Locked model and inference selection shared by every host adapter. +class ValidationProfile { + /// Validates an immutable model/profile manifest before use. + ValidationProfile.fromJson(Map input) + : data = freezeJson(jsonDecode(jsonEncode(input))) as Map { + if (data['schema_version'] != 1) { + throw const FormatException('Unsupported validation schema_version'); + } + if (!RegExp(r'^[a-z][a-z0-9-]{0,63}$').hasMatch(id)) { + throw const FormatException('Invalid profile id'); + } + if (!const ['gguf', 'litert'].contains(runtime)) { + throw const FormatException('runtime must be gguf or litert'); + } + final validBackends = runtime == 'litert' + ? ['cpu', 'gpu', 'npu', 'auto'] + : GpuBackend.values.map((value) => value.name).toList(); + if (!validBackends.contains(backend)) { + throw FormatException('Invalid $runtime backend: $backend'); + } + if (!RegExp(r'^[0-9a-f]{64}$').hasMatch(modelHash) || + !RegExp( + r'^[0-9a-f]{40}$', + ).hasMatch(model['revision'] as String? ?? '')) { + throw const FormatException( + 'Model requires SHA256 and immutable revision', + ); + } + final uri = Uri.parse(model['url'] as String); + if (uri.scheme != 'https' || + uri.userInfo.isNotEmpty || + uri.hasQuery || + !uri.path.contains('/${model['revision']}/')) { + throw const FormatException('Use a public, immutable HTTPS model URL'); + } + if (!filename.endsWith(runtime == 'gguf' ? '.gguf' : '.litertlm') || + !RegExp(r'^[a-zA-Z0-9_.-]+$').hasMatch(filename) || + (model['bytes'] as int) <= 0) { + throw const FormatException('Invalid model filename, format or size'); + } + if (!const ['raw', 'chat'].contains(model['kind'])) { + throw const FormatException('Model kind must be raw or chat'); + } + if (contextSize < 128 || + contextSize > 8192 || + maxTokens < 1 || + maxTokens > 512 || + threads < 1 || + threads > 32) { + throw const FormatException('Profile exceeds bounded core limits'); + } + if (!const ['quick', 'release'].contains(selection)) { + throw const FormatException('selection must be quick or release'); + } + } + + /// JSON definition; callers receive a defensive copy through [toJson]. + final Map data; + + /// Stable profile identifier. + String get id => data['id'] as String; + + /// Model format/runtime identity. + String get runtime => data['runtime'] as String; + + /// Explicit runtime selector. + String get backend => data['backend'] as String; + + /// Immutable model definition. + Map get model => data['model'] as Map; + + /// Expected bytes SHA256. + String get modelHash => model['sha256'] as String; + + /// Safe local model basename. + String get filename => model['filename'] as String; + + /// Whether semantic chat cases apply. + bool get isChat => model['kind'] == 'chat'; + + /// Context token budget. + int get contextSize => data['context_size'] as int? ?? 1024; + + /// Maximum generation length. + int get maxTokens => data['max_tokens'] as int? ?? 32; + + /// Native inference threads. + int get threads => data['threads'] as int? ?? 4; + + /// Catalog selection; release includes explicit uncovered obligations. + String get selection => data['selection'] as String? ?? 'quick'; + + /// Optional model-specific prompts and expected regex predicates. + Map get fixtures => + data['fixtures'] as Map? ?? const {}; + + /// Accelerator evidence is mandatory for an explicit accelerator selection. + bool get requiresAcceleratorProof => + !['cpu', 'auto', 'blas'].contains(backend); + + /// Configuration actually passed to the public model loader. + ModelParams get loadParams => ModelParams( + contextSize: contextSize, + gpuLayers: backend == 'cpu' ? 0 : ModelParams.maxGpuLayers, + preferredBackend: runtime == 'gguf' + ? GpuBackend.values.byName(backend) + : GpuBackend.cpu, + liteRtLmBackend: runtime == 'litert' + ? LiteRtLmBackendPreference.values.byName(backend) + : LiteRtLmBackendPreference.auto, + numberOfThreads: threads, + numberOfThreadsBatch: runtime == 'litert' ? 0 : threads, + ); + + /// Deterministic sampler; native prompt reuse is deliberately disabled. + GenerationParams get generationParams => GenerationParams( + maxTokens: maxTokens, + temp: 0, + seed: 1, + topK: 40, + topP: 0.9, + minP: 0, + penalty: 1.1, + presencePenalty: 0, + reusePromptPrefix: false, + ); + + /// Expanded options and explicit defaults for replay and cohort matching. + Map get effectiveConfig => { + 'context_size': contextSize, + 'threads': threads, + 'batch_threads': runtime == 'litert' ? 0 : threads, + 'backend': backend, + 'gpu_layers_hint': loadParams.gpuLayers, + 'max_tokens': maxTokens, + 'temperature': 0, + 'seed': 1, + 'top_k': 40, + 'top_p': 0.9, + 'min_p': 0, + 'repeat_penalty': 1.1, + 'presence_penalty': 0, + 'stop_sequences': [], + 'enable_thinking': false, + 'reuse_prompt_prefix': false, + 'stream_batch_tokens': generationParams.streamBatchTokenThreshold, + 'stream_batch_bytes': generationParams.streamBatchByteThreshold, + 'speculative_decoding': false, + 'grammar': null, + 'thinking_budget': null, + 'activation_type': null, + 'prefill_chunk_size': null, + 'dispatch_dir': null, + 'batch_size': 0, + 'micro_batch_size': 0, + 'flash_attention': 'auto', + 'cache_type_k': 'f16', + 'cache_type_v': 'f16', + 'kv_unified': null, + 'use_mmap': true, + 'use_mlock': false, + 'load_mtp': false, + 'split_mode': 'layer', + 'main_gpu': 0, + 'loras': [], + 'max_parallel_sequences': 1, + 'speculative_rollback_token_max': 0, + 'rope_frequency_base': null, + 'rope_frequency_scale': null, + 'prefer_memory64': null, + 'parallel_file_section_loading': null, + 'runtime_defaults': 'resolved by the recorded runtime artifact', + 'native_log_level': 'info', + 'cache_behavior': 'fresh request; runtime cache unknown', + }; + + /// Detached serialized definition. + Map toJson() => + jsonDecode(jsonEncode(data)) as Map; +} diff --git a/packages/llamadart_validation/lib/src/placement.dart b/packages/llamadart_validation/lib/src/placement.dart new file mode 100644 index 000000000..75a324a0d --- /dev/null +++ b/packages/llamadart_validation/lib/src/placement.dart @@ -0,0 +1,72 @@ +import 'dart:convert'; +import 'package:crypto/crypto.dart'; + +/// Conservative GGUF native-log evidence. A selector or device inventory alone +/// is insufficient: every expected load needs positive offload and compute allocation. +Map inspectPlacement( + Map manifest, + List> cases, + String? log, +) { + final profile = manifest['profile'] as Map? ?? {}; + final backend = profile['backend']; + final required = manifest['accelerator_evidence_required'] == true; + if (!required) { + return {'required': false, 'verified': true, 'reason': 'not_required'}; + } + final result = { + 'required': true, + 'verified': false, + 'reason': 'native placement evidence unavailable', + if (log != null) 'log_sha256': sha256.convert(utf8.encode(log)).toString(), + }; + if (profile['runtime'] != 'gguf' || + log == null || + !['cuda', 'metal', 'vulkan'].contains(backend)) { + return result; + } + final loads = cases + .where( + (c) => + ['C01.load', 'C09.reload', 'C12.recovery'].contains(c['case_id']) && + c['status'] == 'PASS', + ) + .toList(); + final expectedLoads = loads.length; + if (expectedLoads != 3 || + loads.any((c) { + final name = (c['diagnostics'] as Map?)?['backend_name']; + return name is! String || + !name.toLowerCase().contains(backend as String); + })) { + return result; + } + final offloads = RegExp( + r'load_tensors: offloaded (\d+)/(\d+) layers to GPU', + ).allMatches(log).toList(); + final prefix = { + 'cuda': r'CUDA\d+', + 'metal': r'MTL\d+|Metal', + 'vulkan': r'Vulkan\d+', + }[backend]!; + final buffers = RegExp( + '(?:$prefix) compute buffer size =\\s*([0-9.]+) (?:MiB|MB)', + ).allMatches(log).toList(); + final positive = + offloads.length == expectedLoads && + offloads.every( + (m) => int.parse(m[1]!) > 0 && int.parse(m[1]!) <= int.parse(m[2]!), + ) && + buffers.length == expectedLoads && + buffers.every((m) => (double.tryParse(m[1]!) ?? 0) > 0); + return { + ...result, + 'verified': positive, + 'reason': positive + ? 'matching runtime diagnostics, positive native tensor offload and compute buffers for all three loads' + : 'native placement records incomplete or contradictory', + 'expected_loads': expectedLoads, + 'offload_records': offloads.map((m) => m[0]).toList(), + 'compute_records': buffers.map((m) => m[0]).toList(), + }; +} diff --git a/packages/llamadart_validation/lib/src/report.dart b/packages/llamadart_validation/lib/src/report.dart new file mode 100644 index 000000000..f87e67555 --- /dev/null +++ b/packages/llamadart_validation/lib/src/report.dart @@ -0,0 +1,304 @@ +import 'dart:convert'; +import 'dart:math' as math; + +import 'manifest.dart'; +import 'placement.dart'; + +/// Validates the event journal and produces all report formats from one source. +class ValidationReport { + ValidationReport._( + this.manifest, + this.cases, + this.problems, + this.finished, + this.cleanupPassed, + this.placement, + ); + + final Map manifest; + final List> cases; + final List problems; + final bool finished; + final bool cleanupPassed; + final Map placement; + + /// Truncated, duplicate and missing records cannot become passing reports. + factory ValidationReport.parse(String jsonl, {String? nativeLog}) { + final problems = []; + final events = >[]; + var lineNumber = 0; + for (final line in const LineSplitter().convert(jsonl)) { + lineNumber++; + if (line.trim().isEmpty) continue; + try { + events.add(jsonDecode(line) as Map); + } catch (_) { + problems.add('Invalid JSON event at line $lineNumber'); + } + } + final manifests = events.where((e) => e['type'] == 'manifest').toList(); + if (manifests.length != 1) problems.add('Expected exactly one manifest'); + final manifest = manifests.isEmpty ? {} : manifests.first; + if (manifest['schema_version'] != 1) { + problems.add('Unsupported result schema'); + } + if (manifest['profile_hash'] != jsonHash(manifest['profile'])) { + problems.add('Profile hash does not match the manifest'); + } + if (manifest['config_hash'] != jsonHash(manifest['effective_config'])) { + problems.add('Effective configuration hash does not match'); + } + final inventory = manifest['case_ids']; + final expected = inventory is List + ? inventory.whereType().toList() + : []; + if (inventory is! List || expected.length != inventory.length) { + problems.add('Malformed case inventory'); + } + var sequence = 0; + for (var index = 0; index < events.length; index++) { + final event = events[index]; + if (event['type'] == 'manifest') { + if (index != 0) problems.add('Manifest must be first'); + continue; + } + if (![ + 'case_start', + 'case', + 'cleanup', + 'run_end', + ].contains(event['type'])) { + problems.add('Unknown event type'); + } + if (event['sequence'] != sequence++) { + problems.add('Event sequence mismatch'); + } + if (event['type'] == 'run_end' && index != events.length - 1) { + problems.add('Records follow run end'); + } + } + if (expected.isEmpty || expected.toSet().length != expected.length) { + problems.add('Invalid mandatory case inventory'); + } + final records = >{}; + for (final event in events.where((e) => e['type'] == 'case')) { + final id = event['case_id']; + if (id is! String || !expected.contains(id)) { + problems.add('Unexpected case record: $id'); + continue; + } + if (records.containsKey(id)) { + problems.add('Duplicate terminal record: $id'); + } + if (!const [ + 'PASS', + 'FAIL', + 'ERROR', + 'NOT_RUN', + 'UNSUPPORTED', + ].contains(event['status'])) { + problems.add('Invalid terminal status: $id'); + } + records.putIfAbsent(id, () => event); + } + for (final id in expected) { + if (!records.containsKey(id)) { + problems.add('Missing mandatory case: $id'); + records[id] = { + 'type': 'case', + 'case_id': id, + 'status': 'NOT_RUN', + 'reason': 'missing terminal record after crash/interruption', + }; + } + } + final endings = events.where((e) => e['type'] == 'run_end').toList(); + final finished = + endings.length == 1 && endings.single['cancelled'] == false; + if (!finished) problems.add('Run did not complete normally'); + final cleanups = events.where((e) => e['type'] == 'cleanup').toList(); + final cleanup = cleanups.length == 1 && cleanups.single['status'] == 'PASS'; + if (!cleanup) problems.add('Engine cleanup did not complete'); + return ValidationReport._( + manifest, + [for (final id in expected) records[id]!], + problems, + finished, + cleanup, + inspectPlacement(manifest, [ + for (final id in expected) records[id]!, + ], nativeLog), + ); + } + + /// True only for complete mandatory functional obligations. + bool get assertionsPassed => + problems.isEmpty && + cases.isNotEmpty && + cases.every( + (e) => + e['status'] == 'PASS' || + (e['status'] == 'UNSUPPORTED' && e['expected_unsupported'] == true), + ); + + /// Public selector values alone never qualify an accelerator. + bool get acceleratorVerified => placement['verified'] == true; + + /// Missing or uncommitted build identity preserves results but cannot qualify. + List get provenanceProblems { + final environment = manifest['environment'] as Map? ?? {}; + final runtime = (manifest['profile'] as Map?)?['runtime']; + final tag = environment[runtime == 'litert' ? 'litert_tag' : 'native_tag']; + return [ + if (!RegExp( + r'^[a-f0-9]{40}$', + ).hasMatch('${environment['source_commit']}')) + 'Source commit is missing or unknown', + if (environment['source_dirty'] != false) + 'Build source is dirty or its cleanliness is unknown', + if (!RegExp(r'^[a-f0-9]{64}$').hasMatch('${environment['hook_sha256']}')) + 'Native hook identity is missing or unknown', + if (tag is! String || tag.isEmpty || tag == 'unknown') + 'Runtime artifact pin is missing or unknown', + if (environment['web'] == true && + (environment['bridge_tag'] == null || + environment['bridge_tag'] == 'unknown')) + 'Web runtime artifact pin is missing or unknown', + ]; + } + + /// Overall qualification requires correctness, backend proof and provenance. + bool get qualified => + assertionsPassed && acceleratorVerified && provenanceProblems.isEmpty; + + Map toJson() => { + 'schema_version': 1, + 'manifest': manifest, + 'cases': cases, + 'summary': { + 'qualified': qualified, + 'assertions_passed': assertionsPassed, + 'accelerator_verified': acceleratorVerified, + 'accelerator_evidence': placement, + 'provenance_complete': provenanceProblems.isEmpty, + 'provenance_problems': provenanceProblems, + 'expected': cases.length, + for (final status in ['PASS', 'FAIL', 'ERROR', 'NOT_RUN', 'UNSUPPORTED']) + status: cases.where((c) => c['status'] == status).length, + 'problems': problems, + }, + }; + + /// A failing integrity case preserves crash/collection failures in JUnit. + String toJUnit() { + final errors = + cases.where((c) => c['status'] == 'ERROR').length + (qualified ? 0 : 1); + final xml = StringBuffer( + '', + ); + for (final record in cases) { + xml.write( + '', + ); + final status = record['status']; + final content = _escape(jsonEncode(record)); + if (status == 'FAIL') { + xml.write('$content'); + } + if (status == 'ERROR') { + xml.write('$content'); + } + if (status == 'NOT_RUN' || status == 'UNSUPPORTED') { + xml.write(''); + } + xml.write('$content'); + } + if (!qualified) { + xml.write( + '${_escape([...problems, ...provenanceProblems, if (!assertionsPassed) 'Mandatory assertions incomplete or failed', if (!acceleratorVerified) 'Accelerator execution not verified'].join('; '))}', + ); + } + xml.write(''); + return xml.toString(); + } + + /// Measured repetitions only; warm-ups are excluded from comparisons. + List> get samples => cases + .where((c) => c['benchmark'] == true && c['warmup'] == false) + .toList(); + + String toCsv() { + const metrics = [ + 'wall_ms', + 'ttfa_ms', + 'estimated_wall_tps', + 'native_decode_tps', + ]; + String cell(Object? value) => + '"${(value ?? '').toString().replaceAll('"', '""')}"'; + return [ + ['run_id', 'case_id', 'status', ...metrics].join(','), + for (final sample in samples) + [ + manifest['run_id'], + sample['case_id'], + sample['status'], + for (final metric in metrics) + (sample['metrics'] as Map? ?? {})[metric], + ].map(cell).join(','), + ].join('\n'); + } + + /// Standalone escaped HTML with separate native and estimated throughput plots. + String toHtml() { + String chart(String metric, String title) { + final values = + samples + .map((s) => (s['metrics'] as Map? ?? {})[metric]) + .whereType() + .where((n) => n.isFinite && n >= 0) + .toList() + ..sort(); + final max = values.isEmpty ? 1 : math.max(1, values.last); + final stats = values.isEmpty + ? 'Unavailable' + : 'n=${values.length}; median ${((values[(values.length - 1) ~/ 2] + values[values.length ~/ 2]) / 2).toStringAsFixed(2)}; ' + 'min ${values.first.toStringAsFixed(2)}; max ${values.last.toStringAsFixed(2)}'; + return '

${_escape(title)}

$stats

${samples.map((s) { + final value = (s['metrics'] as Map? ?? {})[metric] as num?; + return '
${_escape(s['case_id'])} · ${_escape(s['status'])}' + '
' + '${value?.toStringAsFixed(2) ?? 'unavailable'}
'; + }).join()}
'; + } + + return '' + '' + 'llamadart validation

llamadart validation

' + '

${qualified ? 'QUALIFIED' : 'INCOMPLETE / FAILED'} · ${_escape(manifest['run_id'])}

' + '

Functional assertions: ${assertionsPassed ? 'passed' : 'incomplete or failed'}. ' + 'Accelerator placement: ${placement['required'] != true + ? 'not required' + : acceleratorVerified + ? 'verified native offload' + : 'unverified'}.

' + '

${_escape([...problems, ...provenanceProblems].join('; '))}

' + '' + '${cases.map((c) => '').join()}
CaseStatusReason
${_escape(c['case_id'])}${_escape(c['status'])}${_escape(c['reason'] ?? '')}
' + '${chart('native_decode_tps', 'Native decode tokens/second')}' + '${chart('estimated_wall_tps', 'Estimated visible-output tokens/second')}' + '${chart('ttfa_ms', 'Time to first visible answer (ms)')}' + '

Each series represents one exact model/configuration/build. Failed outputs remain visible. ' + 'Retokenized output counts are estimates; stream chunks are not tokens.

' + '
Complete evidence
${_escape(const JsonEncoder.withIndent('  ').convert(toJson()))}
'; + } +} + +String _escape(Object? value) => const HtmlEscape( + HtmlEscapeMode.attribute, +).convert((value ?? '').toString()); diff --git a/packages/llamadart_validation/lib/src/runner.dart b/packages/llamadart_validation/lib/src/runner.dart new file mode 100644 index 000000000..1214480ca --- /dev/null +++ b/packages/llamadart_validation/lib/src/runner.dart @@ -0,0 +1,642 @@ +import 'dart:async'; + +import 'package:llamadart/llamadart.dart'; + +import 'manifest.dart'; + +/// Sink implemented by file, browser and Firebase host adapters. +typedef ValidationEventSink = Future Function(Map event); + +/// Injectable public-API boundary for model-free harness regression tests. +abstract interface class ValidationEngine { + Future load(String location, ValidationProfile profile); + Future unload(); + Future dispose(); + void cancel(); + Future> diagnostics(); + Future> tokenize(String text); + Future detokenize(List tokens); + Future> generate( + String prompt, + ValidationProfile profile, { + bool raw = false, + int? maxTokens, + bool cancelAfterFirst = false, + List? history, + }); +} + +/// Runs inference through the exported llamadart API on every platform. +class PublicValidationEngine implements ValidationEngine { + LlamaEngine _engine = LlamaEngine(LlamaBackend()); + bool _disposed = false; + + @override + Future load(String location, ValidationProfile profile) async { + if (_disposed) { + _engine = LlamaEngine(LlamaBackend()); + _disposed = false; + } + await _engine.setLogLevel(LlamaLogLevel.info); + await _engine.loadModel(location, modelParams: profile.loadParams); + if (!_engine.isReady) { + throw StateError('loadModel returned without readiness'); + } + } + + @override + Future unload() => _engine.unloadModel(); + @override + Future dispose() async { + await _engine.dispose(); + _disposed = true; + } + + @override + void cancel() => _engine.cancelGeneration(); + @override + Future> tokenize(String text) => + _engine.tokenize(text, addSpecial: false); + @override + Future detokenize(List tokens) => _engine.detokenize(tokens); + + @override + Future> diagnostics() async => { + 'backend_name': await _engine.getBackendName(), + 'reported_gpu_layers': await _engine.getResolvedGpuLayers(), + 'context_size': await _engine.getContextSize(), + 'model_metadata': await _engine.getMetadata(), + // These public values can contain selector hints. They are not placement proof. + 'accelerator_execution_verified': false, + 'accelerator_evidence_reason': + 'requires correlated native driver/offload diagnostics', + }; + + @override + Future> generate( + String prompt, + ValidationProfile profile, { + bool raw = false, + int? maxTokens, + bool cancelAfterFirst = false, + List? history, + }) async { + final text = StringBuffer(); + final thinking = StringBuffer(); + final finish = []; + var chunks = 0; + int? firstUs; + int? cancelUs; + final params = profile.generationParams.copyWith(maxTokens: maxTokens); + final watch = Stopwatch()..start(); + void append(String content) { + if (content.isEmpty) return; + firstUs ??= watch.elapsedMicroseconds; + text.write(content); + if (text.length > 65536) { + cancel(); + throw StateError('Output exceeded the 64 KiB core limit'); + } + if (cancelAfterFirst && cancelUs == null) { + cancelUs = watch.elapsedMicroseconds; + cancel(); + } + } + + String? cancellationAbort; + try { + if (raw) { + await for (final delta in _engine.generate(prompt, params: params)) { + chunks++; + append(delta); + } + } else { + await for (final chunk in _engine.create( + history ?? + [ + LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: prompt, + ), + ], + params: params, + enableThinking: false, + )) { + chunks++; + for (final choice in chunk.choices) { + append(choice.delta.content ?? ''); + thinking.write(choice.delta.thinking ?? ''); + if (thinking.length > 65536) { + cancel(); + throw StateError('Thinking exceeded the 64 KiB core limit'); + } + if (choice.finishReason != null) finish.add(choice.finishReason!); + } + } + } + } on LlamaInferenceException catch (error) { + // The pinned WASM delegate surfaces cancellation as this explicit abort. + if (cancelUs != null && + error.message == 'Generation failed' && + '${error.details}' == 'AbortError: Generation was cancelled.') { + cancellationAbort = error.toString(); + } else { + rethrow; + } + } + watch.stop(); + // Tokenization and diagnostic reads occur after the timed region. + int? estimatedTokens; + try { + estimatedTokens = (await tokenize(text.toString())).length; + } catch (_) {} + BackendPerfContextData? perf; + try { + perf = await _engine.getPerformanceContext(); + } catch (_) {} + final wallMs = watch.elapsedMicroseconds / 1000; + final nativeMs = perf?.decodeMs ?? perf?.evalMs; + final nativeTokens = perf?.evalTokens; + return { + 'prompt': prompt, + if (history != null) 'messages': history.map((m) => m.toJson()).toList(), + 'max_tokens': params.maxTokens, + 'enable_thinking': false, + 'content': text.toString(), + 'thinking': thinking.toString(), + 'chunks': chunks, + 'finish_reasons': finish, + 'cancel_requested': cancelUs != null, + 'cancel_abort_observed': cancellationAbort != null, + 'cancel_abort': ?cancellationAbort, + 'cancel_to_done_ms': cancelUs == null + ? null + : (watch.elapsedMicroseconds - cancelUs!) / 1000, + 'metrics': { + 'wall_ms': wallMs, + 'ttfa_ms': firstUs == null ? null : firstUs! / 1000, + 'native_ttft_ms': null, + 'estimated_output_tokens': estimatedTokens, + 'token_count_source': 'retokenized visible output; not stream chunks', + 'estimated_wall_tps': wallMs > 0 && estimatedTokens != null + ? estimatedTokens * 1000 / wallMs + : null, + 'native_decode_tokens': nativeTokens, + 'native_decode_ms': nativeMs, + 'native_decode_tps': + nativeMs != null && + nativeMs > 0 && + nativeTokens != null && + nativeTokens > 0 + ? nativeTokens * 1000 / nativeMs + : null, + 'native_timing_source': perf == null + ? null + : 'getPerformanceContext per-request counters', + 'native_prompt_tokens': perf?.promptEvalTokens, + 'native_prompt_ms': perf?.promptEvalMs, + 'missing_native_metrics_reason': perf == null + ? 'backend did not expose counters' + : null, + }, + }; + } +} + +/// Single model/profile run. Assertion failures do not suppress later cases. +class ValidationRunner { + ValidationRunner({ + required this.profile, + required this.engine, + required this.emit, + this.caseTimeout = const Duration(seconds: 60), + }); + + final ValidationProfile profile; + final ValidationEngine engine; + final ValidationEventSink emit; + final Duration caseTimeout; + var _sequence = 0; + var _cancelled = false; + bool _closed = false; + bool _poisoned = false; + bool _settled = true; + + /// Request cancellation from the UI or host without marking success. + void cancel() { + _cancelled = true; + engine.cancel(); + } + + /// Expanded obligations, including explicitly unimplemented release cases. + List get caseIds => [ + 'C01.load', + 'C02.unicode', + 'C03.raw', + if (profile.isChat) ...['C04.hello', 'C04.arithmetic', 'C06.history'], + 'C08.cancel', + 'C09.reload', + 'C10.limit', + 'C12.recovery', + 'B01.warmup', + 'B01.1', + 'B01.2', + 'B01.3', + if (profile.selection == 'release') ...[ + 'C05.thinking', + 'C07.tools', + 'C10.stop', + 'C11.batching', + 'C12.guards', + ], + ]; + + /// Captures a run without pretending selector diagnostics prove GPU placement. + Future run( + String location, { + required String runId, + required Map environment, + Map preparation = const {}, + }) async { + try { + await emit({ + 'type': 'manifest', + 'schema_version': 1, + 'run_id': runId, + 'profile': profile.toJson(), + 'profile_hash': jsonHash(profile.toJson()), + 'effective_config': profile.effectiveConfig, + 'config_hash': jsonHash(profile.effectiveConfig), + 'environment': environment, + 'preparation': preparation, + 'case_ids': caseIds, + 'started_at': DateTime.now().toUtc().toIso8601String(), + 'accelerator_evidence_required': profile.requiresAcceleratorProof, + }); + var usable = false; + var poisoned = false; + for (final id in caseIds) { + if (_cancelled || poisoned || (!usable && id != 'C01.load')) { + await _record(id, 'NOT_RUN', { + 'reason': _cancelled + ? 'cancelled' + : poisoned + ? 'prior_timeout' + : 'model_load_failed', + }); + continue; + } + if (id == 'C05.thinking' || + id == 'C07.tools' || + id == 'C10.stop' || + id == 'C11.batching' || + id == 'C12.guards') { + await _record(id, 'NOT_RUN', { + 'reason': 'release feature pack not implemented', + }); + continue; + } + await emit({ + 'type': 'case_start', + 'case_id': id, + 'sequence': _sequence++, + }); + final watch = Stopwatch()..start(); + try { + _settled = false; + final pending = _runCase( + id, + location, + ).whenComplete(() => _settled = true); + final actual = await pending.timeout(caseTimeout); + if (id == 'C01.load') usable = true; + final status = actual.remove('status') as String? ?? 'PASS'; + await _record(id, status, { + ...actual, + 'elapsed_ms': watch.elapsedMicroseconds / 1000, + }); + } on TimeoutException { + poisoned = true; + _poisoned = true; + engine.cancel(); + await _record(id, 'ERROR', { + 'reason': 'case_timeout', + 'timeout_ms': caseTimeout.inMilliseconds, + }); + } catch (error) { + await _record(id, 'ERROR', { + 'reason': 'runtime_exception', + 'error_type': error.runtimeType.toString(), + 'message': redactDiagnostic('$error'), + }); + } + } + } finally { + _closed = true; + try { + await engine.dispose().timeout(const Duration(seconds: 10)); + await emit({ + 'type': 'cleanup', + 'status': _settled ? 'PASS' : 'ERROR', + if (!_settled) + 'message': 'Timed-out backend operation has not settled', + 'sequence': _sequence++, + }); + } catch (error) { + await emit({ + 'type': 'cleanup', + 'status': 'ERROR', + 'sequence': _sequence++, + 'message': redactDiagnostic('$error'), + }); + } + await emit({ + 'type': 'run_end', + 'sequence': _sequence++, + 'cancelled': _cancelled, + }); + } + } + + Future _record(String id, String status, Map values) => + emit({ + 'type': 'case', + 'case_id': id, + 'status': status, + 'sequence': _sequence++, + ...values, + }); + + String get _shortPrompt => profile.isChat + ? 'Reply with one short sentence saying hello.' + : 'Once upon a time'; + + Future _checked(Future Function() operation) async { + void check() { + if (_closed || _poisoned || _cancelled) { + throw StateError('Case no longer active'); + } + } + + check(); + final result = await operation(); + check(); + return result; + } + + Future> _short() => _checked( + () => engine.generate(_shortPrompt, profile, raw: !profile.isChat), + ); + + Map _nonempty(Map output) => { + ...output, + 'expected': 'nonempty finite output', + 'status': (output['content'] as String).trim().isNotEmpty ? 'PASS' : 'FAIL', + }; + + Future> _withDiagnostics( + Map output, + ) async { + final diagnostics = await _checked(() => engine.diagnostics()); + final name = (diagnostics['backend_name'] as String? ?? '').toLowerCase(); + final runtimeResolved = + name.startsWith('litert-lm') == (profile.runtime == 'litert'); + final metadata = diagnostics['model_metadata'] as Map? ?? {}; + final wasmCpu = + name == 'wasm (prototype bridge)' && + metadata['llamadart.webgpu.n_gpu_layers'] == '0' && + metadata['llamadart.webgpu.core_variant'] == 'wasm32'; + final cpuResolved = + (name.contains('cpu') || wasmCpu) && + !RegExp(r'cuda|metal|vulkan|\bgpu\b|\bnpu\b').hasMatch(name); + return { + ...output, + 'diagnostics': diagnostics, + 'status': !runtimeResolved || (profile.backend == 'cpu' && !cpuResolved) + ? 'FAIL' + : output['status'] as String? ?? 'PASS', + if (!runtimeResolved) + 'reason': 'Resolved runtime does not match model format', + if (runtimeResolved && profile.backend == 'cpu' && !cpuResolved) + 'reason': 'Explicit CPU profile did not resolve to CPU diagnostics', + }; + } + + Future> _runCase(String id, String location) async { + switch (id) { + case 'C01.load': + await _checked(() => engine.load(location, profile)); + return _withDiagnostics({}); + case 'C02.unicode': + const text = 'Montréal 👋\n한글 café'; + final tokens = await _checked(() => engine.tokenize(text)); + final decoded = await _checked(() => engine.detokenize(tokens)); + final prefix = + (profile.fixtures['unicode'] as Map?)?['expected_prefix'] + as String? ?? + ''; + final expected = '$prefix$text'; + return { + 'input': text, + 'tokens': tokens, + 'decoded': decoded, + 'expected': expected, + 'tokenizer_prefix': prefix, + 'status': decoded == expected ? 'PASS' : 'FAIL', + }; + case 'C03.raw': + return _nonempty( + await _checked( + () => engine.generate('Once upon a time', profile, raw: true), + ), + ); + case 'C04.hello': + case 'C04.arithmetic': + final arithmetic = id.endsWith('arithmetic'); + final fixture = + profile.fixtures[arithmetic ? 'arithmetic' : 'hello'] as Map?; + final prompt = + fixture?['prompt'] as String? ?? + (arithmetic + ? 'What is 2 + 2? Reply with only the number.' + : _shortPrompt); + final expected = + fixture?['regex'] as String? ?? + (arithmetic ? r'^4[.!]?$' : r'\bhello\b'); + final output = await _checked(() => engine.generate(prompt, profile)); + final text = (output['content'] as String).trim(); + return { + ...output, + 'expected_regex': expected, + 'status': + RegExp(expected, caseSensitive: false).hasMatch(text) && + (output['thinking'] as String).isEmpty + ? 'PASS' + : 'FAIL', + }; + case 'C06.history': + const prompt = 'What is the secret code? Reply with only the code.'; + final output = await _checked( + () => engine.generate( + prompt, + profile, + history: const [ + LlamaChatMessage.fromText( + role: LlamaChatRole.system, + text: 'Remember the secret code exactly.', + ), + LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: 'The secret code is cedar17.', + ), + LlamaChatMessage.fromText( + role: LlamaChatRole.assistant, + text: 'I will remember the code.', + ), + LlamaChatMessage.fromText(role: LlamaChatRole.user, text: prompt), + ], + ), + ); + return { + ...output, + 'expected': 'cedar17', + 'status': (output['content'] as String).trim() == 'cedar17' + ? 'PASS' + : 'FAIL', + }; + case 'C08.cancel': + final prompt = profile.isChat + ? 'Write a long story about a fox. Continue for at least 500 words.' + : 'Once upon a time'; + final control = await _checked( + () => engine.generate( + prompt, + profile, + raw: !profile.isChat, + maxTokens: 256, + ), + ); + final output = await _checked( + () => engine.generate( + prompt, + profile, + raw: !profile.isChat, + maxTokens: 256, + cancelAfterFirst: true, + ), + ); + final recovery = await _short(); + final duration = output['cancel_to_done_ms'] as num?; + final fullTokens = + (control['metrics'] as Map?)?['native_decode_tokens'] as num?; + final cancelledTokens = + (output['metrics'] as Map?)?['native_decode_tokens'] as num?; + final prefixMatches = + (output['content'] as String).isNotEmpty && + (control['content'] as String).startsWith( + output['content'] as String, + ); + final nativeInterruption = + fullTokens != null && + cancelledTokens != null && + cancelledTokens > 0 && + cancelledTokens < fullTokens; + final interrupted = + prefixMatches && + (nativeInterruption || output['cancel_abort_observed'] == true); + + return { + ...output, + 'recovery': recovery, + 'uncancelled_control': control, + 'interruption_observed': interrupted, + 'status': output['cancel_requested'] != true || !interrupted + ? 'NOT_RUN' + : duration != null && + duration <= 5000 && + (recovery['content'] as String).trim().isNotEmpty + ? 'PASS' + : 'FAIL', + 'reason': + 'requires fewer native decoded tokens or an explicit cancellation abort, matching the seeded control prefix, prompt recovery and bounded cancellation latency', + }; + case 'C09.reload': + await _checked(() => engine.dispose()); + await _checked(() => engine.load(location, profile)); + return _withDiagnostics(_nonempty(await _short())); + case 'C10.limit': + final output = await _checked( + () => engine.generate( + _shortPrompt, + profile, + raw: !profile.isChat, + maxTokens: 1, + ), + ); + final count = + (output['metrics'] as Map)['native_decode_tokens'] as num?; + return { + ...output, + 'expected': 'at most one native decoded token', + 'status': count == null + ? 'NOT_RUN' + : count == 1 + ? 'PASS' + : 'FAIL', + if (count == null) + 'reason': 'native token counter unavailable; chunks are not tokens', + }; + case 'C12.recovery': + await _checked(() => engine.unload()); + String? errorType; + try { + await _checked(() => engine.load('$location.missing', profile)); + } on LlamaException catch (error) { + errorType = error.runtimeType.toString(); + } + await _checked(() => engine.unload()); + await _checked(() => engine.load(location, profile)); + final output = await _short(); + return _withDiagnostics({ + ...output, + 'rejected_error_type': errorType, + 'status': + errorType != null && + (output['content'] as String).trim().isNotEmpty + ? 'PASS' + : 'FAIL', + 'expected': 'typed missing-model error and valid recovery', + }); + default: + final prompt = profile.isChat + ? 'List the numbers from one to twenty in English.' + : 'Once upon a time'; + return { + ..._nonempty( + await _checked( + () => engine.generate(prompt, profile, raw: !profile.isChat), + ), + ), + 'benchmark': true, + 'warmup': id == 'B01.warmup', + 'cohort': profile.isChat + ? 'short-generation' + : 'tiny-packaging-diagnostic', + }; + } + } +} + +/// Bounds and removes common credential/path material from diagnostic exports. +String redactDiagnostic(String value) { + final redacted = value + .replaceAll( + RegExp(r'bearer\s+[^\s]+', caseSensitive: false), + 'Bearer [redacted]', + ) + .replaceAll(RegExp(r'https?://[^\s]+\?[^\s]+'), '[signed URL redacted]') + .replaceAll(RegExp(r'/(?:Users|home)/[^\s/]+'), '/[user]'); + return redacted.length <= 8192 + ? redacted + : '${redacted.substring(0, 8192)}[truncated]'; +} diff --git a/packages/llamadart_validation/pubspec.lock b/packages/llamadart_validation/pubspec.lock new file mode 100644 index 000000000..39ceddc98 --- /dev/null +++ b/packages/llamadart_validation/pubspec.lock @@ -0,0 +1,460 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: fdcd9f70f9eb80df3bc5ed0fa67280df2595fd481948df8a9ab082e6a40ad04b + url: "https://pub.dev" + source: hosted + version: "108.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: a51c769bff3b6dfbe9d60199b8606d808290702a296bef0c26a4ca391d414e46 + url: "https://pub.dev" + source: hosted + version: "14.4.0" + archive: + dependency: transitive + description: + name: archive + sha256: "6c5bcd986e06b94e3c40244af471750840a3d2341d1f9763a1100a14add517b4" + url: "https://pub.dev" + source: hosted + version: "4.3.0" + args: + dependency: transitive + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: e2eb0491ba5ddb6177742d2da23904574082139b07c1e33b8503b9f46f3e1a37 + url: "https://pub.dev" + source: hosted + version: "2.13.1" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + cli_config: + dependency: transitive + description: + name: cli_config + sha256: ac20a183a07002b700f0c25e61b7ee46b23c309d76ab7b7640a028f18e4d99ec + url: "https://pub.dev" + source: hosted + version: "0.2.0" + code_assets: + dependency: transitive + description: + name: code_assets + sha256: "828110d598123b5ea96c00c9f3c72105bf79f8ee36c20a39b26209ade421ec57" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + collection: + dependency: transitive + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + coverage: + dependency: transitive + description: + name: coverage + sha256: "956a3de0725ca232ad353565a8290d3357592bf4250f6f298a185e2d949c5d3d" + url: "https://pub.dev" + source: hosted + version: "1.15.1" + crypto: + dependency: "direct main" + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.dev" + source: hosted + version: "3.0.7" + dinja: + dependency: transitive + description: + name: dinja + sha256: "34d4e569ceb3d900ab061f16cae947223968cc63d2484a8a4710d5ecfa4ae3a2" + url: "https://pub.dev" + source: hosted + version: "1.0.0" + ffi: + dependency: transitive + description: + name: ffi + sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694 + url: "https://pub.dev" + source: hosted + version: "4.0.0" + glob: + dependency: transitive + description: + name: glob + sha256: "218aeb56050c714f62a3182775320dfa04602b55074873e24e31bbd39bda96fb" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + hooks: + dependency: transitive + description: + name: hooks + sha256: eaac480a35ec0814146c2c48d96aaa829e0e44a7662c88ae84c9edf4bc35651f + url: "https://pub.dev" + source: hosted + version: "2.2.0" + http: + dependency: "direct main" + description: + name: http + sha256: "87721a4a50b19c7f1d49001e51409bddc46303966ce89a65af4f4e6004896412" + url: "https://pub.dev" + source: hosted + version: "1.6.0" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: "2635216ca6a737e60de577ffa1a48a0bec76ca8a62917cfc1bb88c14c570646f" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + lints: + dependency: "direct dev" + description: + name: lints + sha256: "12f842a479589fea194fe5c5a3095abc7be0c1f2ddfa9a0e76aed1dbd26a87df" + url: "https://pub.dev" + source: hosted + version: "6.1.0" + llamadart: + dependency: "direct main" + description: + path: "../.." + relative: true + source: path + version: "0.8.23" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "31bd099b47c10cd1aeb55146a2d46ce0277630ecef3f7dae54ad7873f36696cd" + url: "https://pub.dev" + source: hosted + version: "0.12.20" + meta: + dependency: transitive + description: + name: meta + sha256: "307249ce4ff29d58a18e97f6345f539382eb9c9c29ecda628900f31de0443dd9" + url: "https://pub.dev" + source: hosted + version: "1.19.0" + mime: + dependency: transitive + description: + name: mime + sha256: bd47de35f07e27267e69c8c8b22edf9473bfee170a60d60fcc93730c5144b7f6 + url: "https://pub.dev" + source: hosted + version: "2.1.0" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + package_config: + dependency: transitive + description: + name: package_config + sha256: ffcf4cf3d6c0b74ac43708d9f56625506e8a68aa935abe9d267a7330f320eb5d + url: "https://pub.dev" + source: hosted + version: "3.0.0" + path: + dependency: "direct main" + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + pool: + dependency: transitive + description: + name: pool + sha256: "4177f68c237ea2128d1bee66ac17b2ce05ba3dbaafcbdd54c5d40a39d0b6b11c" + url: "https://pub.dev" + source: hosted + version: "1.5.3" + posix: + dependency: transitive + description: + name: posix + sha256: bc1bad54ad2b735816e31f8d4600cfde6c7839975085ddfbca48b6c9f7c4044e + url: "https://pub.dev" + source: hosted + version: "6.5.2" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "261236774e8b1d69cfc6b9eabbc96c40f25e7a2d6b171f3385d4f65d5734fb24" + url: "https://pub.dev" + source: hosted + version: "2.2.1" + record_use: + dependency: transitive + description: + name: record_use + sha256: "1cb8564af8d43b464294411db9217f5ec04891c6f22ee2c32d73ae05e88a6bd2" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: c87c3875f91262785dade62d135760c2c69cb217ac759485334c5857ad89f6e3 + url: "https://pub.dev" + source: hosted + version: "1.1.3" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: c0713a43e323c3302c2abe2a1cc89aa057a387101ebd280371d6a6c9fa68516b + url: "https://pub.dev" + source: hosted + version: "2.1.2" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "14c2945847669b44089bb1222f66873d7ff7103c58911917f2a63c5a62327898" + url: "https://pub.dev" + source: hosted + version: "0.10.14" + source_span: + dependency: transitive + description: + name: source_span + sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab" + url: "https://pub.dev" + source: hosted + version: "1.10.2" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "277654b3034d17ac6f9f1cb5595db011b1d5d41e8806866db28e0abaa101c490" + url: "https://pub.dev" + source: hosted + version: "1.12.2" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test: + dependency: "direct dev" + description: + name: test + sha256: de5d145b0afff7921e5e788a880f52d7e5f3ae24068a202f6fd3b58e4ba26323 + url: "https://pub.dev" + source: hosted + version: "1.32.0" + test_api: + dependency: transitive + description: + name: test_api + sha256: "0a10344e901e5b2e63819567951cb6a06673ed6b84f40462188ff5a0c41f371f" + url: "https://pub.dev" + source: hosted + version: "0.7.14" + test_core: + dependency: transitive + description: + name: test_core + sha256: "80f3fb49087454e07e7e07c67578cfdd156c8c3a5227d8b3f47c7b2d019c2e93" + url: "https://pub.dev" + source: hosted + version: "0.6.20" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "5f37239c4851efcef929cea7824e76df7f2f0970aef85d66bbc430afa40e72f0" + url: "https://pub.dev" + source: hosted + version: "15.3.0" + watcher: + dependency: transitive + description: + name: watcher + sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.dev" + source: hosted + version: "3.0.3" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + yaml: + dependency: transitive + description: + name: yaml + sha256: f67cdd8e07d3c6329146aaef1ba043542b3134c12489f553ca9a7435d1068aea + url: "https://pub.dev" + source: hosted + version: "3.1.4" +sdks: + dart: ">=3.11.0 <4.0.0" diff --git a/packages/llamadart_validation/pubspec.yaml b/packages/llamadart_validation/pubspec.yaml new file mode 100644 index 000000000..13aa2753d --- /dev/null +++ b/packages/llamadart_validation/pubspec.yaml @@ -0,0 +1,25 @@ +name: llamadart_validation +description: Private cross-platform validation harness for the public llamadart API. +publish_to: none +environment: + sdk: ^3.10.7 +dependencies: + llamadart: + path: ../.. + crypto: ^3.0.0 + http: ^1.1.0 + path: ^1.8.3 +dev_dependencies: + lints: ^6.1.0 + test: ^1.26.3 +flutter: + assets: + - assets/profiles/ +hooks: + user_defines: + llamadart: + llamadart_native_runtimes: + runtimes: [llama_cpp, litert_lm] + platforms: + ios-x86_64-sim: [llama_cpp] + windows-arm64: [llama_cpp] diff --git a/packages/llamadart_validation/schemas/event.schema.json b/packages/llamadart_validation/schemas/event.schema.json new file mode 100644 index 000000000..155833259 --- /dev/null +++ b/packages/llamadart_validation/schemas/event.schema.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "llamadart validation journal event v1", + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "enum": [ + "manifest", + "case_start", + "case", + "cleanup", + "run_end", + "preparation_error" + ] + }, + "sequence": { + "type": "integer", + "minimum": 0 + }, + "status": { + "enum": [ + "PASS", + "FAIL", + "ERROR", + "UNSUPPORTED", + "NOT_RUN" + ] + }, + "case_id": { + "type": "string" + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "manifest" + } + } + }, + "then": { + "required": [ + "schema_version", + "run_id", + "profile", + "profile_hash", + "effective_config", + "config_hash", + "environment", + "preparation", + "case_ids" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "case" + } + } + }, + "then": { + "required": [ + "case_id", + "status", + "sequence" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "run_end" + } + } + }, + "then": { + "required": [ + "sequence", + "cancelled" + ] + } + } + ] +} diff --git a/packages/llamadart_validation/schemas/profile.schema.json b/packages/llamadart_validation/schemas/profile.schema.json new file mode 100644 index 000000000..cf26f181c --- /dev/null +++ b/packages/llamadart_validation/schemas/profile.schema.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "llamadart validation profile v1", + "type": "object", + "required": [ + "schema_version", + "id", + "runtime", + "backend", + "model" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "id": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]{0,63}$" + }, + "runtime": { + "enum": [ + "gguf", + "litert" + ] + }, + "backend": { + "type": "string" + }, + "selection": { + "enum": [ + "quick", + "release" + ] + }, + "context_size": { + "type": "integer", + "minimum": 128, + "maximum": 8192 + }, + "threads": { + "type": "integer", + "minimum": 1, + "maximum": 32 + }, + "max_tokens": { + "type": "integer", + "minimum": 1, + "maximum": 512 + }, + "model": { + "type": "object", + "required": [ + "revision", + "filename", + "sha256", + "bytes", + "kind", + "url" + ], + "properties": { + "revision": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "filename": { + "type": "string", + "pattern": "^[a-zA-Z0-9_.-]+$" + }, + "bytes": { + "type": "integer", + "minimum": 1 + }, + "kind": { + "enum": [ + "raw", + "chat" + ] + }, + "url": { + "type": "string", + "pattern": "^https://" + } + } + } + } +} diff --git a/packages/llamadart_validation/test/model_io_test.dart b/packages/llamadart_validation/test/model_io_test.dart new file mode 100644 index 000000000..50938b127 --- /dev/null +++ b/packages/llamadart_validation/test/model_io_test.dart @@ -0,0 +1,108 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; +import 'package:llamadart_validation/io.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +class BlockedClient extends http.BaseClient { + final response = Completer(); + bool closed = false; + @override + Future send(http.BaseRequest request) => + response.future; + @override + void close() { + closed = true; + if (!response.isCompleted) { + response.completeError(http.ClientException('cancelled')); + } + } +} + +void main() { + late Directory cache; + late ValidationProfile profile; + final bytes = utf8.encode('model fixture bytes'); + setUp(() { + cache = Directory.systemTemp.createTempSync('validation-model-test-'); + final json = + jsonDecode( + File('assets/profiles/tiny-gguf-cpu.json').readAsStringSync(), + ) + as Map; + json['model']['sha256'] = sha256.convert(bytes).toString(); + json['model']['bytes'] = bytes.length; + profile = ValidationProfile.fromJson(json); + }); + tearDown(() => cache.deleteSync(recursive: true)); + test('verified cache hit never downloads again', () async { + var calls = 0; + final first = await prepareModel( + profile, + cache, + client: MockClient((_) async { + calls++; + return http.Response.bytes(bytes, 200); + }), + ); + expect(first.evidence['verified'], true); + final second = await prepareModel( + profile, + cache, + client: MockClient((_) async => throw StateError('unexpected network')), + ); + expect(second.evidence['cache_hit'], true); + expect(calls, 1); + }); + test( + 'oversized or wrong-hash download leaves no partial or cached model', + () async { + for (final content in [ + List.filled(bytes.length, 0), + [...bytes, 0], + ]) { + await expectLater( + prepareModel( + profile, + cache, + client: MockClient((_) async => http.Response.bytes(content, 200)), + ), + throwsFormatException, + ); + expect(cache.listSync(recursive: true).whereType(), isEmpty); + } + }, + ); + test('invalid supplied file is retained for the user', () async { + final input = File(p.join(cache.path, 'user-model.gguf')) + ..writeAsStringSync('wrong'); + await expectLater( + prepareModel(profile, cache, suppliedPath: input.path), + throwsFormatException, + ); + expect(input.readAsStringSync(), 'wrong'); + }); + test( + 'deadline closes the active request and removes partial files', + () async { + final client = BlockedClient(); + await expectLater( + prepareModel( + profile, + cache, + client: client, + timeout: const Duration(milliseconds: 5), + ), + throwsA(isA()), + ); + expect(client.closed, true); + expect(cache.listSync(recursive: true).whereType(), isEmpty); + }, + ); +} diff --git a/packages/llamadart_validation/test/validation_test.dart b/packages/llamadart_validation/test/validation_test.dart new file mode 100644 index 000000000..a0efc9df5 --- /dev/null +++ b/packages/llamadart_validation/test/validation_test.dart @@ -0,0 +1,415 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:llamadart/llamadart.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:llamadart_validation/src/placement.dart'; +import 'package:test/test.dart'; + +ValidationProfile profile({String backend = 'cpu', bool release = false}) { + final json = + jsonDecode( + File('assets/profiles/chat-litert-cpu.json').readAsStringSync(), + ) + as Map; + json['backend'] = backend; + json['selection'] = release ? 'release' : 'quick'; + return ValidationProfile.fromJson(json); +} + +class FakeEngine implements ValidationEngine { + var disposed = false; + var cancelled = false; + var loads = 0; + var generated = 0; + var wrongArithmetic = false; + var ignoresCancellation = false; + String backendName = 'LiteRT-LM CPU'; + Map metadata = {}; + int? switchAfterLoad; + Completer? pauseReload; + var timeout = false; + var cleanupFails = false; + @override + Future load(String location, ValidationProfile profile) async { + if (location.endsWith('.missing')) { + throw LlamaModelException('missing model'); + } + loads++; + if (loads == 2) await pauseReload?.future; + } + + @override + Future unload() async {} + @override + Future dispose() async { + disposed = true; + if (cleanupFails) throw StateError('cleanup failure'); + } + + @override + void cancel() { + cancelled = true; + } + + @override + Future> diagnostics() async => { + 'model_metadata': metadata, + 'backend_name': switchAfterLoad != null && loads >= switchAfterLoad! + ? 'Metal' + : backendName, + }; + @override + Future> tokenize(String text) async => utf8.encode(text); + @override + Future detokenize(List tokens) async => utf8.decode(tokens); + @override + Future> generate( + String prompt, + ValidationProfile profile, { + bool raw = false, + int? maxTokens, + bool cancelAfterFirst = false, + List? history, + }) async { + generated++; + if (timeout) return Completer>().future; + return { + 'content': history != null + ? 'cedar17' + : prompt.contains('2 + 2') + ? wrongArithmetic + ? '2' + : '4' + : 'hello', + 'thinking': '', + 'prompt': prompt, + 'cancel_requested': cancelAfterFirst, + 'cancel_to_done_ms': cancelAfterFirst ? 1 : null, + 'metrics': { + 'native_decode_tokens': + maxTokens == 1 || (cancelAfterFirst && !ignoresCancellation) + ? 1 + : 8, + 'native_decode_tps': 12, + 'estimated_wall_tps': 10, + 'ttfa_ms': 5, + }, + }; + } +} + +Future<({ValidationReport report, List> events})> run( + FakeEngine engine, { + ValidationProfile? selected, +}) async { + final events = >[]; + await ValidationRunner( + profile: selected ?? profile(), + engine: engine, + emit: (event) async { + events.add(event); + }, + caseTimeout: const Duration(milliseconds: 30), + ).run( + 'model.litertlm', + runId: 'test-run', + environment: { + 'source_commit': List.filled(40, 'a').join(), + 'source_dirty': false, + 'hook_sha256': List.filled(64, 'b').join(), + 'native_tag': 'v0.4.0', + 'litert_tag': '0.17.0-3', + }, + ); + return ( + report: ValidationReport.parse(events.map(jsonEncode).join('\n')), + events: events, + ); +} + +void main() { + test( + 'WASM CPU diagnostics require a CPU-only core and zero GPU layers', + () async { + final selected = profile().toJson()..['runtime'] = 'gguf'; + final model = Map.from(selected['model'] as Map); + model['filename'] = 'model.gguf'; + model['url'] = (model['url'] as String).replaceAll('.litertlm', '.gguf'); + selected['model'] = model; + for (final layers in ['0', '1', null]) { + final engine = FakeEngine() + ..backendName = 'WASM (Prototype bridge)' + ..metadata = { + 'llamadart.webgpu.n_gpu_layers': layers, + 'llamadart.webgpu.core_variant': 'wasm32', + }; + final result = await run( + engine, + selected: ValidationProfile.fromJson(selected), + ); + for (final id in ['C01.load', 'C09.reload', 'C12.recovery']) { + expect( + result.report.cases.firstWhere( + (record) => record['case_id'] == id, + )['status'], + layers == '0' ? 'PASS' : 'FAIL', + ); + } + } + }, + ); + test( + 'missing or dirty provenance preserves assertions but cannot qualify', + () async { + final result = await run(FakeEngine()); + expect(result.report.assertionsPassed, true); + for (final key in [ + 'source_commit', + 'source_dirty', + 'hook_sha256', + 'litert_tag', + ]) { + final events = (jsonDecode(jsonEncode(result.events)) as List) + .cast(); + (events.first['environment'] as Map).remove(key); + final report = ValidationReport.parse( + events.map(jsonEncode).join('\n'), + ); + expect(report.assertionsPassed, true); + expect(report.qualified, false); + expect(report.toJUnit(), contains('run-integrity')); + expect(report.provenanceProblems, isNotEmpty); + } + (result.events.first['environment'] as Map)['source_dirty'] = true; + expect( + ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ).qualified, + false, + ); + }, + ); + test( + 'CPU backend changes during reload or recovery cannot qualify TPS', + () async { + for (final threshold in [2, 3]) { + final result = await run(FakeEngine()..switchAfterLoad = threshold); + expect( + result.report.cases.singleWhere( + (c) => + c['case_id'] == + (threshold == 2 ? 'C09.reload' : 'C12.recovery'), + )['status'], + 'FAIL', + ); + expect(result.report.qualified, false); + } + }, + ); + + test( + 'native GPU evidence requires matching backend and all load allocations', + () { + final manifest = { + 'profile': {'runtime': 'gguf', 'backend': 'metal'}, + 'accelerator_evidence_required': true, + }; + final cases = >[ + { + 'case_id': 'C01.load', + 'status': 'PASS', + 'diagnostics': {'backend_name': 'Metal'}, + }, + { + 'case_id': 'C09.reload', + 'status': 'PASS', + 'diagnostics': {'backend_name': 'Metal'}, + }, + { + 'case_id': 'C12.recovery', + 'status': 'PASS', + 'diagnostics': {'backend_name': 'Metal'}, + }, + ]; + const load = + 'load_tensors: offloaded 7/7 layers to GPU\nsched_reserve: MTL0 compute buffer size = 63.62 MiB\n'; + expect(inspectPlacement(manifest, cases, load * 3)['verified'], true); + expect(inspectPlacement(manifest, cases, load * 2)['verified'], false); + expect( + inspectPlacement( + manifest, + cases, + load.replaceAll('7/7', '0/7') * 3, + )['verified'], + false, + ); + expect( + inspectPlacement( + manifest, + cases, + load.replaceAll('MTL0', 'CPU') * 3, + )['verified'], + false, + ); + expect( + inspectPlacement(manifest, cases, 'GPU found: Metal')['verified'], + false, + ); + cases.first['diagnostics'] = {'backend_name': 'CPU'}; + expect(inspectPlacement(manifest, cases, load * 3)['verified'], false); + }, + ); + + test( + 'late timed-out reload cannot start more inference after cleanup', + () async { + final paused = Completer(); + final engine = FakeEngine()..pauseReload = paused; + final result = await run(engine); + final generated = engine.generated; + expect(result.report.cleanupPassed, false); + paused.complete(); + await Future.delayed(Duration.zero); + expect(engine.generated, generated); + expect(engine.loads, 2); + }, + ); + test('explicit CPU case rejects accelerator diagnostics', () async { + final result = await run(FakeEngine()..backendName = 'Metal'); + expect(result.report.cases.first['status'], 'FAIL'); + expect(result.report.qualified, false); + }); + + test( + 'ignored cancellation cannot pass the cancellation obligation', + () async { + final result = await run(FakeEngine()..ignoresCancellation = true); + expect( + result.report.cases.singleWhere( + (c) => c['case_id'] == 'C08.cancel', + )['status'], + 'NOT_RUN', + ); + expect(result.report.qualified, false); + }, + ); + test('journal sink failure still disposes the engine', () async { + final engine = FakeEngine(); + await expectLater( + ValidationRunner( + profile: profile(), + engine: engine, + emit: (_) async => throw StateError('disk full'), + ).run('model', runId: 'test', environment: {}), + throwsStateError, + ); + expect(engine.disposed, true); + }); + test('validated profile cannot be mutated through nested JSON', () { + final selected = profile(); + expect(() => selected.model['sha256'] = 'changed', throwsUnsupportedError); + }); + + test('rejects floating or unchecked model inputs', () { + for (final field in ['revision', 'sha256']) { + final json = profile().toJson(); + (json['model'] as Map)[field] = 'main'; + expect(() => ValidationProfile.fromJson(json), throwsFormatException); + } + }); + test('canonical identity is independent of JSON key order', () { + expect(jsonHash({'b': 2, 'a': 1}), jsonHash({'a': 1, 'b': 2})); + }); + test('one warmup and three measured samples; lifecycle loads only', () async { + final engine = FakeEngine(); + final result = await run(engine); + expect(result.report.qualified, isTrue); + expect(result.report.samples, hasLength(3)); + expect(engine.loads, 3); + expect(engine.disposed, isTrue); + }); + test('semantic failure preserves output and still measures TPS', () async { + final result = await run(FakeEngine()..wrongArithmetic = true); + final failure = result.report.cases.singleWhere( + (c) => c['case_id'] == 'C04.arithmetic', + ); + expect(failure['status'], 'FAIL'); + expect(failure['content'], '2'); + expect(result.report.samples, hasLength(3)); + expect(result.report.assertionsPassed, isFalse); + }); + test( + 'timeout cancels and prevents overlapping subsequent inference', + () async { + final engine = FakeEngine()..timeout = true; + final result = await run(engine); + expect(engine.generated, 1); + expect(engine.cancelled, isTrue); + expect(engine.disposed, isTrue); + expect(result.report.assertionsPassed, isFalse); + expect( + result.report.cases.where((c) => c['status'] == 'NOT_RUN'), + isNotEmpty, + ); + }, + ); + test('cleanup failure cannot produce a passing report', () async { + final result = await run(FakeEngine()..cleanupFails = true); + expect(result.report.qualified, isFalse); + expect(result.report.toJUnit(), contains('run-integrity')); + }); + test( + 'GPU preference and reported layers are not accelerator proof', + () async { + final result = await run(FakeEngine(), selected: profile(backend: 'gpu')); + expect(result.report.assertionsPassed, isTrue); + expect(result.report.qualified, isFalse); + expect(result.report.toHtml(), contains('unverified')); + }, + ); + test('release selection keeps unimplemented obligations visible', () async { + final result = await run(FakeEngine(), selected: profile(release: true)); + expect(result.report.qualified, isFalse); + expect( + result.report.cases.singleWhere( + (c) => c['case_id'] == 'C07.tools', + )['status'], + 'NOT_RUN', + ); + }); + test( + 'duplicate, missing, malformed or truncated records fail closed', + () async { + final result = await run(FakeEngine()); + final lines = result.events.map(jsonEncode).toList(); + for (final altered in [ + [ + ...lines, + jsonEncode(result.events.firstWhere((e) => e['type'] == 'case')), + ], + lines.take(lines.length - 1).toList(), + [...lines, '{"type":'], + lines + .where((s) => !s.contains('C02.unicode') || s.contains('manifest')) + .toList(), + ]) { + expect(ValidationReport.parse(altered.join('\n')).qualified, isFalse); + } + }, + ); + test('reports escape generated markup and retain warmup exclusion', () async { + final result = await run(FakeEngine()); + result.report.cases.first['content'] = ''; + expect(result.report.toHtml(), isNot(contains('