diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ab4dec1f..354d9af8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,24 @@ jobs: - name: Check Pana Score run: flutter pub global run pana . --exit-code-threshold 0 + validation-harness: + name: Validation harness (model-free) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: subosito/flutter-action@v2 + with: + flutter-version: 3.47.1 + channel: stable + cache: true + - run: dart pub get + working-directory: packages/llamadart_validation + - run: dart analyze --fatal-infos + working-directory: packages/llamadart_validation + - run: dart test + working-directory: packages/llamadart_validation + companion-packages: name: Companion Package (${{ matrix.package }}) runs-on: macos-latest diff --git a/.github/workflows/validation_bundles.yml b/.github/workflows/validation_bundles.yml new file mode 100644 index 000000000..519ee4198 --- /dev/null +++ b/.github/workflows/validation_bundles.yml @@ -0,0 +1,113 @@ +name: Validation bundles + +on: + pull_request: + branches: [main] + paths: + - .github/workflows/validation_bundles.yml + - packages/llamadart_validation/** + - tool/testing/validation.dart + - tool/testing/validation/** + - example/chat_app/lib/validation_main.dart + - example/chat_app/lib/validation/** + - example/chat_app/integration_test/validation_test.dart + - example/chat_app/android/** + - example/chat_app/ios/** + - example/chat_app/pubspec.* + - scripts/build_chat_app_web.sh + workflow_dispatch: + inputs: + profile: + description: Locked quick profile compiled into mobile and Web apps + type: choice + options: [tiny-gguf-cpu, tiny-gguf-lifecycle, tiny-gguf-batching, tiny-gguf-metal, tiny-gguf-vulkan, tiny-gguf-cuda, chat-gguf-cpu, chat-gguf-metal, chat-gguf-vulkan, chat-gguf-cuda, chat-litert-cpu, chat-litert-gpu] + default: tiny-gguf-cpu + +permissions: + contents: read + +concurrency: + group: validation-bundles-${{ github.ref }} + cancel-in-progress: false + +jobs: + desktop: + name: Desktop ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2022, macos-15] + runs-on: ${{ matrix.os }} + timeout-minutes: 35 + steps: + - uses: actions/checkout@v7 + - uses: subosito/flutter-action@v2 + with: + flutter-version: 3.47.1 + channel: stable + cache: true + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - run: dart pub get + - run: dart run tool/testing/validation.dart build --target desktop --profile ${{ inputs.profile || 'tiny-gguf-cpu' }} --out .dart_tool/validation/bundle + - name: Preserve executable permissions in transport + run: tar -czf .dart_tool/validation/bundle.tar.gz -C .dart_tool/validation bundle + - name: Verify extracted desktop command and profiles + shell: bash + run: | + python - <<'PY' + import os + from pathlib import Path + import subprocess + import tarfile + import tempfile + + archive = Path('.dart_tool/validation/bundle.tar.gz').resolve() + with tempfile.TemporaryDirectory(prefix='llamadart-portable-') as scratch: + with tarfile.open(archive) as package: + package.extractall(scratch, filter='data') + extension = '.exe' if os.name == 'nt' else '' + command = Path(scratch) / 'bundle/bin' / ('llamadart-validate' + extension) + subprocess.run([str(command), '--help'], cwd=scratch, check=True, timeout=30) + profiles = subprocess.run([str(command), '--list'], cwd=scratch, check=True, + capture_output=True, text=True, timeout=30).stdout.splitlines() + if 'tiny-gguf-cpu' not in profiles: + raise RuntimeError('Extracted command did not discover bundled profiles') + PY + - uses: actions/upload-artifact@v7 + with: + name: validation-desktop-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + path: .dart_tool/validation/bundle.tar.gz + retention-days: 7 + if-no-files-found: error + + app: + name: App ${{ matrix.target }} + strategy: + fail-fast: false + matrix: + target: [android, web, ios-inputs] + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@v7 + - uses: subosito/flutter-action@v2 + with: + flutter-version: 3.47.1 + channel: stable + cache: true + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '17' + - run: dart run tool/prepare_workspace.dart + - run: dart run tool/testing/validation.dart build --target ${{ matrix.target }} --profile ${{ inputs.profile || 'tiny-gguf-cpu' }} --out .dart_tool/validation/bundle + - name: Preserve executable permissions in transport + run: tar -czf .dart_tool/validation/bundle.tar.gz -C .dart_tool/validation bundle + - uses: actions/upload-artifact@v7 + with: + name: validation-${{ matrix.target }}-${{ github.sha }} + path: .dart_tool/validation/bundle.tar.gz + retention-days: 7 + if-no-files-found: error diff --git a/.gitignore b/.gitignore index f45570e43..112846dbf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ build/ pubspec.lock !example/chat_app/pubspec.lock +!packages/llamadart_validation/pubspec.lock + +# Native runtime profiling output from optional model validation. +*.profraw .flutter-plugins .flutter-plugins-dependencies .metadata diff --git a/CHANGELOG.md b/CHANGELOG.md index 804e32fbf..41985f10b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## Unreleased +* Discover Windows backend libraries in standard compiled Dart CLI bundles. + +* Add locked Gemma 4/Qwen3.5 validation profiles, explicit NPU coverage, and opt-in speech and voice-pipeline diagnostics. + +* Preserve system instructions as plain text when seeding native LiteRT-LM chat history. + - Render structured typed tool results as JSON text for chat templates while preserving string results and source objects. - Preserve Qwen XML tool argument types using declared schemas and reject invalid or undeclared calls without exposing executable tool deltas. diff --git a/doc/cross_platform_validation.md b/doc/cross_platform_validation.md new file mode 100644 index 000000000..c2d96c38d --- /dev/null +++ b/doc/cross_platform_validation.md @@ -0,0 +1,1087 @@ +# Cross-platform validation runbook + +The first implementation supplies the quick core, shared reports, portable +builds, and explicit Firebase/GCE orchestration. It uses the current checkout's +runtime pins. It does not replace the release matrix, qualify unrun platforms, +or schedule paid work. The broader [validation plan](cross_platform_validation_plan.md) +retains the later feature and device milestones. + +Firebase is the primary physical Android/iOS qualification route. The personal +Pixel and iPad are optional debugging devices; the Mac remains the local build, +iOS signing and desktop/browser test host. The next milestone includes Firebase +NPU qualification on Galaxy S24 (Qualcomm SM8650) and Pixel 10 (Tensor G5). +The two SoC-specific model locks and offline input preflight are implemented. +The Android builder now embeds verified model/vendor inputs, and the installed +app checks its SoC before loading. Public-Dart and direct-native control paths +capture per-generation dispatch evidence. S24 execution is verified, but strict +history qualification remains failed; Pixel 10 hardware execution is NOT_RUN. + +## Current readiness + +The quick diagnostic core is usable; the full platform/release suite is incomplete. +Model-backed Mac, browser and Firebase runs have exposed actual product failures, +and the reports retain failed assertions alongside useful timing and device evidence. +Catalog 4 has 129 passing private harness tests at the reviewed pre-integration +head `b13c545f`; fresh head/base checks remain required after merging main. +Draft [PR #515](https://github.com/leehack/llamadart/pull/515) builds Linux x64, +Windows x64 and macOS arm64 desktop bundles plus Android, Web and iOS inputs. +Follow its current CI for exact-head results; build success and extracted CLI +startup are not model execution or complete device qualification. +Native GGUF Unicode corruption was fixed in merged +[PR #516](https://github.com/leehack/llamadart/pull/516). Earlier failed journals +remain historical evidence; reruns must identify the fixed source commit. + +| Area | Current evidence | Remaining qualification | +| --- | --- | --- | +| Quick public API core | Load, Unicode round-trip, raw/chat, history, cancellation/recovery, reload, token bound and short TPS sampling | Catalog 4 adds separate Unicode generation; retain model/backend failures | +| Report integrity | Canonical profile-derived cases/configuration/proof requirements; missing, contradictory, duplicate and interrupted records fail closed | Paired native/public/reference aggregation and optional trend views | +| Portable apps | Local macOS bundle and Android/iOS/Web paths exercised | Refresh exact-head CI and real portable execution evidence; primary-model/device qualification and iOS signing remain separate | +| Cloud lifecycle | Firebase lifecycle exercised; Linux/Windows bootstrap runs collected and resources deleted; provider failure controls tested locally | Bootstrap execution is not end-to-end qualification of the maintained GCE adapter and custom image | +| Accelerators | GGUF native-log proof and S24 per-generation NPU dispatch evidence | LiteRT GPU/Web proof, Pixel 10 NPU and the remaining device rotation | +| Critical feature packs | Catalog 4 executes Unicode generation, thinking on/off, tool choice/result roundtrips, stop markers, unloaded-engine guards and batching/lifecycle controls | Exact-model feature qualification, thinking budgets, tool-bearing batching, broader guards, multimodal, speech qualification and embeddings | + +Do not weaken semantic predicates to obtain a green run. The Gemma original-model +control below passes the strict history predicate; the LiteRT results remain failed. +Three benchmark samples support diagnosis, not a performance regression threshold +or a device ranking. First qualify the portable build workflow, then expand the +critical feature packs with representative locked models before broadening devices. + +## Source and generated artifacts + +- `packages/llamadart_validation/`: private Dart suite, locked profiles, desktop + runner/reporter, JSONL validation and JSON/JUnit/CSV/HTML rendering. +- `example/chat_app/lib/validation_main.dart`: interactive QA app. Run with + `flutter run -t lib/validation_main.dart` from `example/chat_app`. +- `example/chat_app/integration_test/validation_test.dart`: unattended entrypoint; + Android instrumentation and iOS XCTest invoke the same controller. +- `tool/testing/validation.dart`: build, local, report, npu-preflight, plan, run, status, collect, + reconcile, cleanup. Provider helpers live beside it under `tool/testing/validation/`. +- `.github/workflows/validation_bundles.yml`: build-only workflow on relevant PR + changes (tiny CPU profile) or explicit manual selection. No cloud + credentials, model runs, VM creation or Firebase submission in CI. + Desktop jobs extract the transport archive outside the checkout and verify + command startup and bundled profile discovery without loading a model. +- `.dart_tool/validation/`: ignored local models, bundles and journals. Do not + commit weights, signing files, account configs, results or credentials. + +Use Flutter **3.47.1**, its Dart executable, and Python 3.10+ (Windows: `python`, +other hosts: `python3`). Android uses the installed Android SDK/JDK; XCTest needs +Xcode, configured local signing and the owned Mac. gcloud is needed only for +provider operations. Normal chat-app behavior is unchanged. +The `local` command records source/runtime pins for diagnostics. Desktop JIT and +Flutter desktop runs cannot qualify without verified portable payload evidence; +use the built CLI bundle for desktop qualification. Plain `flutter run` without +the builder's identity defines also remains diagnostic. Dirty builds retain all +assertion results and metrics; qualification requires clean committed source, +known runtime identities, and verified model hash/size evidence matching the lock. + +Desktop launch verifies the bundle inventory, runtime files and environment, +rejects runtime overrides and unlisted loader sidecars, then anchors native cache +discovery to the bundle directory. Caller model/cache/output paths retain their +original meaning. `runtime_payload_verified` and `runtime_bundle_sha256` record +this check separately from accelerator placement. The builder rejects local pub +overrides; explicit native library overrides are rejected by the public adapter. +Dart tooling's JIT library search paths remain usable only for diagnostics. +This is payload integrity verification, not code signing or a signature over an +untrusted producer's report. + +## Quick model profiles and cases + +| Profiles | Locked fixture | Use | +| --- | --- | --- | +| `tiny-gguf-{cpu,metal,vulkan,cuda}` | stories15M, 98,357,920 bytes | Packaging, native loading, lifecycle; throughput is a tiny-model diagnostic | +| `tiny-gguf-lifecycle` | Same stories15M lock / CPU | Quick core plus the second dispose/load/generate cycle | +| `tiny-gguf-batching` | Same stories15M lock / CPU | Quick core plus C11 default/adjusted/default batching parity | +| `chat-gguf-{cpu,metal,vulkan,cuda}` | Qwen3.5 0.8B Q4_0, 563,036,064 bytes | GGUF chat, history and instruction checks | +| `chat-litert-{cpu,gpu}` | Qwen3 0.6B LiteRT-LM, 614,236,160 bytes | Native LiteRT public path; explicit GPU proof remains incomplete | +| `gemma3-litert-cpu` | Gemma3 1B IT q4 LiteRT-LM, 584,417,280 bytes | CPU semantic counterpart to the S24 NPU fixture; gated, supply a local authorized model | + +Full revisions and SHA256 values live in profile JSON. The instruction GGUF is +[ggml-org's Q4_0 artifact](https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/blob/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q4_0.gguf), +so its results are a distinct cohort from the Q4_K_M candidate in the original plan. +Native LiteRT files cannot qualify LiteRT Web; that requires a Web model bundle. +The app reports this explicitly before downloading a native LiteRT fixture in Web. + +The Gemma CPU profile uses the same pinned model repository revision as the NPU +profile, with a distinct CPU artifact hash and conversion. It aligns context +1280, four threads, 32 output tokens and thinking enabled. CPU uses the requested +greedy sampler; NPU uses unknown compiled defaults. These are semantic controls, +not an identical-artifact speed comparison. The CPU profile adds the same four +strict history variants as the native NPU diagnostic, producing 17 cases: +canonical, the original literal-system representation, no system and a combined +user prompt. `enable_thinking` is an explicit boolean profile override; +`history_controls` opts LiteRT CPU chat profiles into the extra diagnostic cases. +Other core profiles retain their original inventory and settings. + +```bash +dart run tool/testing/validation.dart local --profile gemma3-litert-cpu \ + --model /path/to/gemma3-1b-it-int4.litertlm \ + --out .dart_tool/validation/runs/gemma-cpu-control-1 +``` + +Use fresh output directories for repetitions. The CPU artifact has native context +capacity 4096 but is loaded with the matched 1280 limit. A Mac result is a desktop +semantic control; it does not fill the separate S24 CPU device obligation. +The builder rejects this gated profile for mobile/Web, and remote submission +rejects it before spending quota. Desktop bundles remain usable with `--model`; +private mobile model transfer must be implemented before enabling that lane. + +The quick inventory is C01 load/diagnostics, C02 Unicode tokenize/detokenize, +C03 raw generation, C04 hello/arithmetic and C06 multi-turn history for chat +fixtures, C08 cancellation/control/recovery, C09 dispose/new engine/reload, +C10 one-token limit, C12 missing-model rejection/recovery, and B01 one warmup +plus three measured generations. Independent assertion failures do not suppress +later metrics; load failure or timeout prevents unsafe later inference. + +Sampling is temperature 0, seed 1, top-k 40, top-p .9, repeat penalty 1.1, +context 1024, four threads, 32 generated tokens, thinking disabled, prompt reuse +disabled. Case overrides (one-token limit and 256-token cancellation) are recorded. +C08 compares the same prompt against an uncancelled control; the pinned WASM +delegate's explicit cancellation AbortError is also recorded as interruption evidence. Merely calling cancel +is not a pass; missing interruption evidence is NOT_RUN. Case timeouts are 60s; +disposal has a 10s bound and unresolved work cannot report successful cleanup. +Native Flutter model acquisition has a ten-minute deadline within the +18-minute integration test and 20-minute Test Lab execution limits. CLI downloads +retain their five-minute default. Download deadline failures report received and +expected bytes, remove partial weights and never become inference/TPS samples. + +Prompts, regex expectations, exact output/thinking, ordered terminal case IDs, +configuration hashes, model hashes, source/runtime pins and environment all appear +in the journal. The raw tiny fixture does not claim chat capability. + +### Focused selections and replay metadata + +The shared runner accepts three profile selections: + +- `quick`: the original short core and one three-sample benchmark series. +- `focused`: the quick core plus cases matching the profile's nonempty, unique + `focus_features` list. Valid IDs are `text`, `unicode`, `thinking`, `history`, + `tools`, `streaming`, `batching`, `lifecycle`, `guards` and `performance`. Text/history/ + performance are already covered by the applicable quick cases. +- `release`: every current extended core obligation, including unfinished cases. + +For example, `"selection": "focused", "focus_features": ["streaming", "tools"]` +adds C07 tools, C10 stop markers and C11 batching. Catalog 4 executes C07; +catalog 3 executes C10 control/stop/recovery, and C11 executes the parity or +rejection contracts below. +`tiny-gguf-lifecycle` is a runnable focused CPU profile: it adds +`C09.reload.second` after the first reload, invalid-input recovery and benchmark. +It is available in the QA app, portable bundles and manual build workflow: + +```bash +dart run tool/testing/validation.dart local --profile tiny-gguf-lifecycle \ + --out .dart_tool/validation/runs/tiny-lifecycle +``` + +`tiny-gguf-batching` selects C11 without also selecting the separate stop-marker +case. It runs the same short prompt with default worker thresholds (8 pieces / +512 bytes), then 1 piece / 1 byte, then defaults again. Content, thinking and +finish reasons must match, every stream must complete in order, and the recorded +options must match each trial. Chunk counts may differ. All three requests retain +output, effective batching thresholds and timing/TPS; they are parity trials, +not extra benchmark samples. + +LiteRT Web separately checks that each nondefault option produces a named +`LlamaUnsupportedException` before inference, then verifies default-request +recovery. This is negative-contract coverage, not native batching qualification. +GGUF Web, direct-native controls and NPU runtime-default sampling remain NOT_RUN +for C11. Tool-bearing parity requires the qualified C07 fixture; unexpected tool +emissions remain visible and cannot pass the text/thinking case. + +Journal/report schema 2 exports catalog version 2, per-feature and per-case +versions, resolved synthetic prompts/tool schemas/predicates, selected cases and +omission reasons. The runner reads the same compiled fixture definitions that it +exports; model-specific text/predicate overrides are explicit hashed profile +fields. Each terminal record binds its case version and fixture hash. Reports +reject missing or rehashed conflicting catalog data, and HTML shows selection +and unselected cases separately from verdicts. Future media/model packs still +need their own fixture locks and reference qualification. + +Schema-1 journals remain readable with their original case inventory; missing +catalog metadata stays unavailable. Catalog-version-1 schema-2 reports are also imported against their original +definitions, with C11 still unimplemented. Unknown versions or new batching +selections claiming the old catalog fail closed. A legacy report is not evidence that the +newer extended selection ran. Focused selection requires schema 2. + +Thinking budgets, tool-bearing batching, expanded unsupported guards, +multimodal/speech/embedding packs and full browser/device rotation remain +subsequent qualification work. NPU requires the verified Android packaging +described below; selecting `npu` alone cannot supply vendor libraries or evidence. +Catalog 4 executes all currently selected core release cases. This does not +qualify the broader planned feature packs or historical runs: existing journals +retain their original catalog and NOT_RUN results. Direct-native controls keep +public-only feature cases NOT_RUN. + +## NPU input preparation + +`npu-qualcomm-sm8650` and `npu-tensor-g5` are **unqualified candidate locks** +with opt-in Android build support. Each records Gemma 3 1B IT revision +`a6306a4e292016480083b73b8dc6f3f939ae04c3`, the SoC-specific file/hash/size, +context 1280, max output 32 and required library architectures. The Qualcomm +file is 690,094,080 bytes; Tensor G5 is 1,678,542,365 bytes. Model access is gated: +stage an authorized copy locally without embedding tokens or signed URLs in +the APK, kit manifest or report. The preflight never downloads a model. + +```sh +dart run tool/testing/validation.dart npu-preflight \ + --profile npu-qualcomm-sm8650 \ + --model /path/to/Gemma3-1B-IT_q4_ekv1280_sm8650.litertlm \ + --kit /path/to/qualcomm-kit \ + --out .dart_tool/validation/npu-preflight.json +``` + +Omit `--model` or `--kit` to obtain the missing-input inventory. `--out` must be +a new file. The command writes hashes, sizes, SoC/runtime identities and checks; +it does not export supplied local paths. Exit 1 and `status: NOT_RUN` remain +intentional even when `inputs_verified: true`: file integrity is not native +execution or qualification. It creates no remote resources and uses no quota. + +The kit contains flat regular library files and `npu-kit.json` with schema 1, +`target`, `runtime_tag`, `litert_revision`, `dispatch_header_sha256`, and a +`libraries` map from basename to `sha256`/`bytes`. Host libraries must be +AArch64 ELF64; the Qualcomm V75 skeleton is Hexagon ELF32. The lock includes +QAIRT 2.47.0.260601 host/DSP hashes, audited vendor dispatch hashes and the +matching diagnostic proxy hashes. Rebuilding a proxy requires reviewing and +updating its lock; a caller-supplied manifest cannot bless a different probe. +An older prebuilt, different SoC, tampered file, missing file or symlink fails +the input check. Dynamic dependency resolution still needs final-APK/device +verification; this inventory check does not execute or resolve a shared library. +The inspected S24 stub needs device `libcdsprpc.so`; its DSP skeleton needs +Hexagon `libc++.so.1` and `libc++abi.so.1`. The 2026-09-17 installed S24 pilot +initialized and executed the NPU through both adapters, establishing access for +that exact device/runtime/kit combination. Other targets remain unverified. +Android host libraries cannot substitute for DSP libraries with the same basename. + +For runtimes `0.17.0-3` and `0.17.0-5`, the actual LiteRT dependency is +`9fe5be45564c868408e6514c8aabb83e211a0911`. Its dispatch header adds `get_hooks` +to the nested interface relative to LiteRT v2.2.0 while retaining the same API +version. The version string alone cannot establish table-layout compatibility. +The native owner repository's diagnostic proxy is built against the exact +headers and accepts only the audited same-source vendor binaries. It preserves +vendor calls and exports lifetime counters for synchronous completions/failures, +async submissions/failures and synchronous calls in flight. Async submission is +not completion; positive counts establish some NPU work, never all-NPU placement. + +The builder stages the model as an uncompressed APK asset and the verified kit +as extracted native libraries. The final APK checker streams every model/library +entry and rechecks sizes/hashes; remote upload repeats this check. The Android +host checks `Build.SOC_MODEL`, API and ABI before loading, checks installed library +hashes, copies the model to the app's private cache, and verifies its hash again. +It declares the required device library (`libcdsprpc.so` or +`libedgetpu_litert.so`) and configures the app-local DSP search directory. These +checks cannot prove that the Firebase sandbox grants access to its device driver. +Use the dated device results below for actual execution evidence. + +Build separate bundles from clean committed source; each contains interactive +`qa-app.apk`, unattended `app.apk` and instrumentation `test.apk`: + +```sh +dart run tool/testing/validation.dart build --target android \ + --profile npu-qualcomm-sm8650 \ + --kit /path/to/qualcomm-kit \ + --model /path/to/Gemma3-1B-IT_q4_ekv1280_sm8650.litertlm \ + --execution-path native_c_api \ + --out .dart_tool/validation/bundles/s24-npu-native +# Repeat with --execution-path public_api and a separate output directory. +``` + +These locally staged bundles contain authorized model weights and licensed vendor +libraries. Keep them private; the public CI bundle workflow does not accept NPU +kit/model inputs. No Hugging Face token, signed download URL or SDK credential is +packaged. Kit `license-*` files are retained in the APK. + +The direct native control bypasses `LlamaEngine` and its backend/worker bindings, +calling the pinned C API on a dedicated isolate. It now runs twelve cases: load, +hello, arithmetic, four history controls, reload, warmup and three throughput +repetitions. The history controls retain the exact `cedar17` oracle and compare +canonical native system/history seeding, the former public path's literal JSON system +content, history without a system message, and one combined user prompt. Each +records the JSON bytes supplied to the C API, response, timing and dispatch +counters. The native system setter receives JSON content, not a complete message +object; the literal-JSON variant deliberately preserves the original incorrect +public serialization for regression diagnosis. Setup timing and its dispatch snapshot are separate +from send/decode timing; total dispatch proof includes preface initialization. +It uses the +same Gemma artifact, dispatch kit, context 1280, four threads and per-request +output cap 32 as the public LiteRT service. The public NPU runtime skips session +sampler overrides, so both paths retain compiled model/runtime defaults. Requested +seed/temperature/top-k/top-p remain recorded, but effective NPU sampling is +explicitly unknown; these runs cannot claim seeded deterministic sampling. +Gemma keeps the native conversation default for thinking; the Qwen pilot's +explicit thinking-disable setting is not reused. The public NPU path runs the +14-case quick chat inventory, including Unicode tokenization, raw/history, +cancel/control/recovery, one-token limit and missing-model recovery. + +Every generation records before/after probe counters. The reporter recomputes +deltas and rejects absent, reset, failed, in-flight or async-only proof. Positive +proof means **NPU participation; CPU partition coverage unknown**, never all-NPU. +Native controls are labelled separately and do not qualify the public Dart path. +Native decode TPS/token counts and native TTFT are separate from wall throughput; +the blocking native control cannot observe visible-answer TTFA, so that field +is null. Warmup is retained but excluded from the three-sample charts. + +Main `21135e37dadf60882ea427db6078ccc90f84a28a` adopts runtime `0.17.0-5`. +Its published manifest retains upstream LiteRT-LM +`e9fd8c53ff968071774206163027dd84bedfe925` and the same LiteRT dependency above. +The NPU input guard now requires this runtime and a separately recorded kit +audit; preserve the old kit and reports. The new runtime's desktop linkage fixes +do not establish that Gemma NPU history is fixed. The native repository's +[Qwen tokenizer repair](https://github.com/leehack/litert-lm-native/blob/3eb4079397d19e2058e176fcdb5ab15b28a70ad0/docs/qwen3_tokenizer_repair.md) +creates a distinct model artifact; the locked original Qwen fixture is unchanged. + +Use the normal Firebase `plan`/`run`/`collect`/`cleanup` flow below with the exact +S24 or Pixel 10 profile/device pairing. Run the native control first and stop if +it cannot initialize. Do not automatically submit both bundles or bypass the +selected Spark quota or Blaze budget guard. The compatible S24 CPU Gemma semantic control, separate Unicode +generation fixture and aggregate paired-control qualification remain future +work; the existing Qwen CPU retry is not a matched Gemma control. A per-run green +report is not completion of the full NPU pack. The `validation-harness` row covers +the model-free identity/proof/input checks and APK tampering tests. + +## Build and run + +```sh +# Model-free safety and report regression tests. +dart run tool/testing/run_local_e2e.dart --scenario validation-harness + +# Uses a shared verified model cache; creates a fresh output directory. +dart run tool/testing/validation.dart local --profile chat-litert-cpu + +# Build on each native OS/architecture; weights are downloaded when running. +dart run tool/testing/validation.dart build --target desktop --profile tiny-gguf-cpu --out .dart_tool/validation/bundles/desktop +dart run tool/testing/validation.dart build --target android --profile tiny-gguf-vulkan --out .dart_tool/validation/bundles/android +dart run tool/testing/validation.dart build --target web --out .dart_tool/validation/bundles/web +dart run tool/testing/validation.dart build --target ios-inputs --out .dart_tool/validation/bundles/ios-inputs +# On the owned Mac with existing local Xcode signing configured: +dart run tool/testing/validation.dart build --target ios --profile tiny-gguf-cpu --out .dart_tool/validation/bundles/ios +``` + +Desktop bundles include `bin/llamadart-validate`, `bin/llamadart-report`, native +code assets, profiles and remote wrappers. +The hidden `.dart_tool/llamadart/litert_lm/` directory inside desktop bundles +contains the pinned LiteRT runtime, re-extracted from its SHA256-verified archive. +Preserve it when copying or extracting a bundle; the public runtime discovers it +relative to the executable. This avoids depending on the build machine's cache. +Windows arm64 bundles retain GGUF only; the manifest records LiteRT unavailable, +and an explicitly selected LiteRT build profile is rejected on that ABI. +Run outside the repository: + +```sh +bin/llamadart-validate --profile tiny-gguf-cpu --environment-file environment.json --out results +bin/llamadart-report results +# Explicit GGUF accelerator evidence can be derived from that run's native log: +bin/llamadart-report results --native-log results/stderr.log +``` + +Windows executables have `.exe`. Keep the complete bundle layout. GitHub artifacts +contain `bundle.tar.gz` so executable permissions survive download; extract it +before use. Bundle manifests enumerate every shipped byte and record build OS/ABI, +SDKs, source commit/dirty state, native pins and the compiled mobile profile. +Resolved suite/app dependency locks are included as provenance too. +Remote dispatch rejects changed, dirty, wrong-OS or wrong-profile bundles. + +Android produces `qa-app.apk` for interactive installation and a matched +`app.apk`/`test.apk` instrumentation pair. These are Debug builds, explicitly +labelled; do not compare their TPS to Release builds. Both native runtimes are +bundled, making APKs larger than a single-runtime deployment. + +`ios-inputs` contains committed source, no signing credentials. `ios` invokes +`build-for-testing` and packages `tests.zip`; signing must already be configured. The unique existing project team is applied +to the test target too; use `--team TEAMID` when selecting another configured team. +Building the iOS bundle does not submit a test; use the explicit Firebase +`plan`/`run` steps below. Web builds use the maintained bridge-staging script. Serve with isolation +headers, for example: + +```sh +python3 tool/testing/serve_static_with_headers.py --directory .dart_tool/validation/bundles/web --port 7367 +``` + +## Firebase setup, submission and collection + +The [device rotation and NPU cases](cross_platform_validation_plan.md#8-firebase-device-selection-and-free-rotation) +include S24, Tab P12, iPhone 16 Pro, iPhone SE 3, Pixel 10 and iPad 10. The planned +initial selection is 16 CPU/GPU core executions, six NPU reference/public-Dart/CPU +control executions and two targeted iPad GPU/lifecycle executions: **24 across +at least seven Spark quota days**, with at most four planned physical executions/day. +NPU submissions require the verified APK packaging and preflight first. iOS cases +cover CPU/Metal/LiteRT GPU; Apple NPU is not exposed by the current backend. +Native XCTest coverage does not qualify Safari/iPadOS browser execution. + +By default, use an explicitly selected **unbilled Spark project** and account. Preflight checks +billing is disabled and the selected physical model/OS exists in the live catalog. +Pick one row per submission; never use an implicit device matrix. Check available +physical quota in Firebase before creating a receipt. Receipt evidence expires +after 15 minutes. The local journal permits at most four physical submissions in +a rolling 24 hours; other clients still share the provider's quota. + +Copy `tool/testing/validation/firebase.example.json` to an ignored local path and +fill account, project, exact device/OS and a freshly checked quota receipt. + +```sh +dart run tool/testing/validation.dart plan --target firebase-android --config .dart_tool/validation/firebase.json --bundle .dart_tool/validation/bundles/android --out .dart_tool/validation/plan.json +dart run tool/testing/validation.dart run --plan .dart_tool/validation/plan.json +``` + +Use `firebase-ios` with the signed `ios` bundle. Plan creation is local/read-only; +`run` rechecks identity, budget evidence and live provider state before submission. +The CLI uses explicit `--project` and `--account`, one execution capped at 20 minutes, no +flaky retries and no video. It records the matrix ID, polls terminal state, copies +the default Test Lab results, then verifies completion or cancellation. It does +not enable billing or create a custom result bucket. + +### Explicit Blaze runs + +Blaze is an opt-in mode for separately authorized runs. Copy +`tool/testing/validation/firebase_blaze.example.json` to an ignored local path. +The operator links billing separately; the runner never upgrades a project. +Preflight requires `billing_mode: "blaze"`, an enabled billing link to the exact +`billing_account`, and current physical execution quota. Spark plans still reject +billed projects. A quota receipt proves execution capacity, not free minutes. + +Supply a fresh `budget` receipt with an authorization/pricing evidence reference, +fixed `window_start`, `expires_at` no more than 24 hours later, and positive +`maximum_run_usd` and `maximum_total_usd`. At least one hour must remain before +dispatch. Refresh `verified_at` within 15 minutes of each run, without moving +the authorized window or increasing its cap. All amounts are USD: record any +conservative conversion, tax and auxiliary-cost allowance in the evidence. +Do not copy a CAD credit balance into `available_usd`. + +`funding: "credit"` also requires a fresh credit receipt that verifies Test Lab +eligibility, at least twice the entire batch budget available, and two hours +before credit expiry. A balance alone or a scope of "certain usage" does not +establish eligibility. `funding: "approved_charges"` is only for an explicit +user authorization covering out-of-pocket charges; it is not implied by having +a payment method or requesting a credit-funded upgrade. + +For free execution on Blaze, select `funding: "free_allowance"` and supply a fresh +`free_allowance` receipt (`verified_at`, `evidence`, `remaining_physical_minutes`). +Verify the full project inventory, including other clients and pre-upgrade use; +round each physical test's **test-process duration** up to whole minutes. Queue, +installation and result-collection durations are not the billable test duration. +Subtract this usage from the published 30-minute daily physical allowance. Unknown +or active executions prevent relying on that calculation. The receipt must be +within the budget window. This mode needs no assumed credit eligibility or paid +authorization; gross USD reservations remain for traceability. + +Set `test_timeout_minutes` to an integer from 1 to 20 (default 20). Verified free +minutes must cover that timeout plus a one-minute rounding reserve. The journal +also reserves minutes for submissions made after the receipt was checked, so +reusing a receipt cannot spend those minutes twice. Refresh the full project +inventory after each completed execution before reclaiming unused reserved time; +do not simply update its timestamp. These controls cannot exclude submissions +from another client after the check, so keep this dedicated project serialized. + +The journal reserves the full per-run estimate before any submission, rounding +up to cents, and rejects runs exceeding the batch cap. Earlier Blaze submissions +for the same project within that window consume the cap even when failed or +cancelled; preflight failures do not. An uncertain submission still blocks all +new remote work until reconciled. Keep one shared run journal for the batch; +do not delete it or move the window to reset the allowance. + +Each run remains one physical device, at most a 20-minute provider timeout and zero +flaky retries. The estimate must cover at least the full timeout at the current +[$5/device-hour rate](https://firebase.google.com/docs/test-lab/usage-quotas-pricing), +before deducting any free minutes or credits. For example, a $2/run reservation +and $6 batch cap permit three runs. Check current prices and add other costs +where applicable. This is a local submission guard, not a Cloud Billing hard +cap: it cannot control other clients, determine promotional eligibility, or +reconcile the provider invoice. Actual usage, applied credits and charged cost +remain separate evidence. Budget alerts also do not cap charges. + +The gcloud asynchronous response contains a console URL, not a matrix object. +The adapter reads the matrix creation receipt and verifies its project and unique +run label through the Testing API before using it. If submission is interrupted, +recover the existing matrix with `reconcile`; never repeat `run` to find out +whether a submission succeeded. An unverified candidate ID is diagnostic only. + +Android pulls external app result files. The first live S24 run recovered a +complete journal through this path, but its collected console log contained no +validation JSONL; console-only recovery is not yet qualified on Flutter devices. +iOS attaches bounded result files to XCTest; collection exports `.xcresult` +attachments on macOS. Missing, truncated or conflicting evidence remains incomplete. +Physical device export/crash behavior is an explicit live-qualification step; +a build and fake-provider tests alone do not prove it. + +A model-preparation error can occur before the suite manifest exists. The current +collector retains those raw device files under `remote-results`, but cannot +produce a validated suite report from them. Record preparation as ERROR and +inference as NOT_RUN, with TPS unavailable. Zero cases and successful artifact +collection are not a passing run; a structured preparation-error envelope remains +a follow-up. Do not insert requested bundle metadata as observed runtime evidence. + +## GCE setup and teardown + +Run the GCE controller from the owned Mac or Linux. Windows is supported as the +CUDA test guest; Windows-hosted GCE orchestration is not yet qualified. + +Copy `tool/testing/validation/gce.example.json` to an ignored path. Configure an +immutable **GPU-ready image**, exact expected driver, compatible machine/accelerator, +zone, an existing network and a targeted IAP TCP/22 firewall tag. Linux needs +Python, `timeout` and `sha256sum`; Windows needs Google-supported SSH and PowerShell. +This initial adapter consumes an already qualified image; it does not install GPU +drivers or change project networking. Check quotas and image/machine compatibility +before the first approved run. Instance readiness rechecks the actual driver. + +The GCE profile must explicitly select GGUF CUDA and the uploaded desktop bundle +must be built for the destination x64 OS. Each run requires fresh evidence that +promotional credit applies to **all** expected costs, has at least twice the +estimated run cost available, and expires at least two hours after dispatch. +The receipt is an operator-verified billing-console record, not an automatic +credit API lookup. A budget alert is not a guarantee against charges. + +```sh +dart run tool/testing/validation.dart plan --target gce-linux-cuda --profile tiny-gguf-cuda --config .dart_tool/validation/gce.json --bundle .dart_tool/validation/bundles/linux --out .dart_tool/validation/plan.json +dart run tool/testing/validation.dart run --plan .dart_tool/validation/plan.json +``` + +Use `gce-windows-cuda` with a Windows x64 bundle and Windows-ready image. The +adapter records intent before creating one VM, requires provider-confirmed deletion +after 60 minutes, persists instance/disk IDs before setup, verifies the driver, +uploads/checks the bundle over IAP, runs under a 20-minute watchdog, collects +results, deletes the owned VM/boot disk, and reads the inventories back. +It creates no reusable volumes, static IPs or service accounts. An ephemeral +public IP provides outbound model download access and belongs in the cost estimate. + +Deletion is the default cleanup. **Stopped VMs can retain charged disks**; stopping +also clears the runtime deadline. Provider deletion deadlines and local finally +cleanup are complementary, not a zero-cost guarantee. If identity is uncertain, +cleanup refuses to delete a potentially unrelated resource, records UNKNOWN and +blocks another run. No cloud run is permitted under the $0-out-of-pocket policy +when applicable credit cannot be verified. + +## Recovery and evidence + +```sh +dart run tool/testing/validation.dart status --run-id qa-EXACT-ID +dart run tool/testing/validation.dart collect --run-id qa-EXACT-ID +dart run tool/testing/validation.dart cleanup --run-id qa-EXACT-ID +# Only after finding the exact ID in the provider console: +dart run tool/testing/validation.dart reconcile --run-id qa-EXACT-ID --remote-id EXACT-PROVIDER-ID +``` + +Use the ID printed in the saved plan (the placeholder above is not a valid ID). +OS locks serialize provider operations and release if the controlling process dies. +SIGINT requests cancellation; cleanup still runs after the current bounded provider +operation. Reusing a run ID cannot submit twice. A missing creation/submission +response is UNKNOWN, never permission to retry. If no remote ID was received, +inspect the provider console using the exact project/run label, preserve the +journal, and use `reconcile` before recovery; do not delete the journal to bypass +the blocker. Reconciliation checks the Firebase project/matrix/run label, or the +GCE numeric instance ID, zone, label and owned automatic boot disk. It cannot +replace an established identity and does not erase the original failure. Then +run `status`, `collect` and `cleanup`. Recovery commands never submit a replacement. + +`orchestration.json` stores phase, exact IDs, provider state and collection state; +`cleanup.json` stores verified/unknown cleanup. `remote-summary.json` is the +combined verdict: provider success **and** validated Dart assertions/provenance +**and** complete retrieval **and** verified cleanup. Test-only report success +cannot override provider failure or unresolved infrastructure. Collection also +matches the source commit, cleanliness, hook hash and all runtime pins against +the uploaded bundle; desktop evidence must match its bundle manifest hash. + +Each run exports `events.jsonl`, `manifest.json`, `results.json`, `junit.xml`, +`samples.csv`, and `summary.html`. HTML has case status/output, native decode TPS, +estimated visible-output TPS, TTFA, and median/min/max for three measured samples. +Warmup is retained but excluded from these comparisons. Missing counters are null; +chunks are never called tokens. Backend-native decode timing and retokenized +wall-time estimates stay in separate series. Three samples are informational, +not a performance regression gate or a cross-device ranking. + +The reporter validates the embedded profile, derives its canonical mandatory case +inventory and effective configuration, and checks the journal against both. Hash +consistency alone is insufficient. Accelerator proof is derived from the backend; +an event flag cannot waive it. The current catalog grants no expected-unsupported +exemptions, so a producer cannot qualify a skipped case by setting +`expected_unsupported: true`. Old reports are evidence snapshots; revalidation +with a newer catalog must preserve the original and write a separate result. +Reimport requires `preparation.verified == true` and the exact model SHA256/byte +size from the profile. Historical desktop journals without runtime payload proof +retain their assertion results but are incomplete under the current qualification +gate; do not copy new verification flags into old journals. + +Explicit CPU rows reject contradictory GPU diagnostics. GGUF accelerator reports +require matching backend diagnostics plus positive native tensor offload and +compute allocations for all three successful loads. Device presence or requested +GPU layers alone does not qualify execution. LiteRT NPU uses the checked per-generation dispatch adapter described above. +LiteRT GPU and browser accelerator proof still require qualified evidence +adapters; they remain incomplete. + +Normal model runs are opt-in. Model-free suite/provider tests run in CI. Before +claiming another platform qualified, attach the exact commit, model/backend, +command, provider/device identity, combined verdict, cleanup and native evidence. + +## Initial owned-machine observations (2026-09-17) + +The portable macOS arm64 bundle ran outside the repository. Tiny GGUF CPU/Metal +completed inference/lifecycle/TPS, while native Unicode corruption remains tracked +in [#511](https://github.com/leehack/llamadart/issues/511). The tiny model's expected +SentencePiece leading space is an explicit fixture normalization; corrupted +Unicode is still a failure. A Web/WASM UI run preserved Unicode and proved +cancellation through the pinned delegate's explicit AbortError and successful +recovery. The CPU identity check recognizes the pinned WASM name only with zero +GPU layers and its CPU core metadata. +Web native-token counters were unavailable, so C10 stayed NOT_RUN. + +Qwen3.5 GGUF also exposed native Unicode corruption and returned `Cedar17` for the +case-sensitive `cedar17` history oracle. LiteRT CPU ran 13/14 cases successfully +but answered `2` for `2 + 2`; retain the semantic failure and relate it to +[#509](https://github.com/leehack/llamadart/issues/509), without inferring the same +cause before a native reference comparison. These runs are diagnostics, not +release qualification or stable performance baselines. + +## Maintained Firebase CPU pilot (2026-09-17) + +Four physical executions used the unbilled Spark project. All reached terminal +provider states, artifacts were collected, and completion was verified. No GCE +VM was created. Android file pulls and iOS XCTest attachment export both recovered +complete quick-core journals when model preparation succeeded. + +| Device / profile | Outcome | Median native decode TPS | +| --- | --- | ---: | +| Galaxy S24 / tiny GGUF CPU | 10 PASS, 1 FAIL: C02 Unicode corruption | 538.5 | +| iPhone 16 Pro / tiny GGUF CPU | 10 PASS, 1 FAIL: C02 Unicode corruption | 708.3 | +| Galaxy S24 / Qwen3 LiteRT CPU | Preparation ERROR; inference NOT_RUN | Unavailable | +| iPhone 16 Pro / Qwen3 LiteRT CPU | 13 PASS, 1 FAIL: C04 arithmetic returned `2` | 8.9 | + +GGUF used native `v0.4.0`; LiteRT used `0.17.0-3`. Unicode matches +[#511](https://github.com/leehack/llamadart/issues/511); the arithmetic observation +matches [#509](https://github.com/leehack/llamadart/issues/509) and does not by +itself establish a Dart-layer defect. The three benchmark samples after warmup +remain available despite those independent assertion failures. Android is Debug +and iOS is Release; the tiny GGUF model is a packaging fixture, so these numbers +are not a device ranking or a comparison of backend performance. + +The Android LiteRT attempt ended after 302 seconds during model download, before +any suite manifest or inference. Its old diagnostic recorded a closed connection +without partial byte progress. The corrected mobile host now has a ten-minute +download deadline and reports byte counts on timeout. The final iOS bundle used +that host, downloaded and verified the 614 MB fixture in about 65 seconds, then +ran the suite. The original failed Android attempt remains in the evidence; +the separately selected retry below establishes download recovery for one run. + +The remaining mobile milestones are reliable preparation-error envelopes, +semantic-failure investigation, and the planned accelerator qualification packs. +These CPU observations do not qualify accelerator paths or a release. + +### Android LiteRT CPU retry after Blaze upgrade + +Run `qa-1789668430755373` / `matrix-2thou79z4k2ud` used the corrected clean +`b860bfe6c13958bede6df7088d1f53e05eac8f7c` Debug bundle on S24/API 36. It +downloaded the 614,236,160-byte Qwen3 model in 169.332 seconds, verified its locked +SHA256, and finished preparation in 175.961 seconds within the ten-minute +download deadline. This is one successful retry, not a network reliability rate. + +The result was **13 PASS, 1 FAIL**: C04 arithmetic again returned `2` for `2 + 2`, +matching [#509](https://github.com/leehack/llamadart/issues/509). Median native +decode throughput was 12.34 TPS, estimated wall throughput 8.53 TPS, and visible +TTFA 2,054.97 ms across three measured samples after warmup. The provider recorded +232 seconds of test process time (four rounded free minutes), and collection and +completion were verified. The scheduled September 18 retry was paused to prevent +a duplicate. Qwen CPU is not the matched Gemma CPU control for the NPU pack. + +## Galaxy S24 NPU pilot (2026-09-17) + +Both installed apps used clean bundle source `2288eed19c482e7774df2c93b1e7d9a96e6e9c4e`, +Flutter 3.47.1, Debug mode, LiteRT-LM `0.17.0-3`, QAIRT `2.47.0.260601`, and the +locked `Gemma3-1B-IT_q4_ekv1280_sm8650.litertlm` model. The device reported +`SC-51E`, `SM8650`, arm64-v8a and API 36. Model and kit hashes were verified on +installation and per-generation synchronous vendor-call completions established +**NPU participation; CPU partition coverage unknown**. + +| Execution path | Result | Native decode TPS, median | Estimated wall TPS, median | Collection/completion | +| --- | --- | ---: | ---: | --- | +| Direct native C API control | 8 PASS; NPU proof in 7 generations | 80.08 | 72.45 | COMPLETE / VERIFIED | +| Public llamadart API | 13 PASS, 1 FAIL; NPU proof in 14 generations | 82.27 | 72.24 | COMPLETE / VERIFIED | + +These are three short 32-token measured generations after warmup, not a sustained +benchmark or evidence that one adapter is faster. Both use compiled NPU sampling +defaults: requested temperature/seed overrides are not applied, and effective +sampler values remain unknown. Native-control TTFT median was 36.37 ms; public +visible-answer TTFA median was 135.56 ms. They measure different boundaries and +must not be substituted for each other. + +The public history case expected `cedar17` and returned 32 repeated `7` characters. +It remains FAIL and blocks public NPU qualification. Load, Unicode tokenizer +round-trip, raw generation, hello, arithmetic, cancellation/recovery, reload, +one-token output limit and invalid-path recovery passed. Track the exact inputs +and controls in [history investigation #513](https://github.com/leehack/llamadart/issues/513). +The native control does not yet test this history input, so its success does not +establish whether the failure belongs to Dart, the runtime, or the compiled model. + +Native run `qa-1789667329901788` / `matrix-pt34kzd9mgxsa` used six seconds of test +process time; public run `qa-1789667821365130` / `matrix-21m98jmyipzo2` used 45 seconds. +Both were capped at ten minutes after a fresh project-wide free-minute check. +Each consumes one rounded free physical minute. All raw outputs, configuration, +provenance, timing samples and dispatch counters are retained with the run reports. +The matched Gemma CPU control, separate Unicode generation fixture and aggregate +NPU pack qualification remain outstanding; the Qwen CPU retry is independent. + +The final project-wide inventory at 18:15 UTC found all seven September 17 +physical executions complete, with no unresolved executions. Individually +rounded test times totalled **17 of the 30 free physical minutes**: 11 from the +earlier Spark runs and six from this three-run batch. The gross USD 6 journal +reservation is a dispatch guard, not a charge. No VM or custom storage bucket was +created. The default Test Lab result bucket is +[provided at no charge](https://docs.cloud.google.com/sdk/gcloud/reference/firebase/test/android/run#description), +and complete copies of the reports are retained locally. + +### Native history replay on current main / LiteRT-LM 0.17.0-5 + +The follow-up control used clean source +`fb8d2beb4db65f1b8987d69022049b4d0ae4fbda`, including main +`21135e37dadf60882ea427db6078ccc90f84a28a`, with the same Gemma model, QAIRT kit +and S24/API 36. Runtime `0.17.0-5` was verified from the release archive through +the APK: its core SHA256 `502d017d8375c0796adf5f720da29fb1915b2a70103e3cf8f6e8e8880ac0f614` +becomes `5424262e8a396b1cc32813a5d3f061c0ccba06cdff2ddc61d5bfdc40a249d2c4` +after the reproduced Android NDK symbol-stripping operation. + +Run `qa-1789670325547288` / `matrix-1serhazjregr5` completed **8 PASS, 4 FAIL**, +with no ERROR/NOT_RUN and NPU participation verified in all eleven generations. +Every history variant retained the exact, case-sensitive `cedar17` oracle: + +| Direct native history control | Output | Verdict | +| --- | --- | --- | +| Normal system content plus prior messages | `Cedar17` | FAIL: capitalization | +| Public path's literal-JSON system content, same prior messages | 32 repeated `7` characters | FAIL | +| Prior messages without system content | `Cedar17` | FAIL: capitalization | +| Four text contents joined into one user prompt | `7777\n` | FAIL | + +This reproduces degeneration without the public worker/streaming adapter on the +new runtime. The literal-system input conflicts with the C API's system-content +contract; normal content removed degeneration in this observed comparison, but +strict recall still failed. One generation per variant and unknown compiled +sampling do not establish a failure rate or isolate every cause. The native +conversation-creation snapshots showed no vendor calls for these controls, so +this result does not establish a separate initialization-prefill defect. +[#513](https://github.com/leehack/llamadart/issues/513) retains the exact setter +JSON and next controls: correct the serialization boundary, replay public Dart, +and compare a compatible Gemma CPU/model reference. No public `0.17.0-5` rerun +or production serialization change is claimed by this control. + +The independent short benchmarks had median native decode 80.58 TPS, estimated +wall 71.71 TPS and native TTFT 37.05 ms. Collection was COMPLETE and terminal +cleanup VERIFIED. The provider recorded 46 seconds, consuming one rounded free +minute. The final 18:52 UTC project inventory found eight completed physical +executions, **18 of 30 free minutes used**, twelve remaining and no unresolved +executions. No VM was created or additional run scheduled. + +### Public history replay after the system-content fix + +Clean source `d0bd029b07e31649ab5760756dcc1841ba487282` corrects the service's +system-message boundary: pass plain joined text to the runtime, which JSON-encodes +it once for the C API. Literal JSON, quotes, newlines, Unicode, multiple and +empty system messages are covered; the existing history-order and unsupported +system-media assertions remain. Three regression assertions failed against the +old service. All 161 LiteRT VM tests, full repository analysis and changed-file +format checks pass, with every executable history-seeding line covered. + +The one public API retest used the same S24, model hash, QAIRT kit and audited +`0.17.0-5` core as the preceding native control. Run `qa-1789672372212252`, +matrix `matrix-23gzng6la12qr`, completed **13 PASS, 1 FAIL**, with no ERROR/NOT_RUN, +complete provenance and NPU participation verified across fourteen generations. +`C06.history` now returns **`Cedar17`**, matching the canonical native control, +instead of the original repeated `7` output. The exact lowercase `cedar17` +oracle remains unchanged and failed; the NPU profile remains unqualified and +[#513](https://github.com/leehack/llamadart/issues/513) stays open. The history +request recorded 52 prompt tokens, four decoded tokens, two public chunks and +five completed vendor calls; CPU partition coverage is still unknown. + +This fixes the demonstrated serialization contract mismatch and removes the +degeneration in this single public observation. It does not establish a failure +rate or attribute the remaining capitalization and combined-prompt failures to +model behavior, model conversion or the NPU runtime. A compatible Gemma CPU/model +reference and repeated controlled comparison remain the next diagnostic steps. +Compiled NPU sampling defaults are unknown; requested temperature/seed are not +applied by this path. + +Three measured short benchmark samples after warmup gave median native decode +**77.84 TPS**, estimated wall **66.13 TPS** and time to first public output +**152.17 ms**. Native TTFT is unavailable on this public path. These benchmarks +assert bounded nonempty output, not number-list semantics, and the earlier +82.27 TPS sample is not a statistically controlled performance baseline. + +Collection is COMPLETE and terminal cleanup VERIFIED. The test process took +46 seconds, consuming **one rounded free minute**. The 19:22 UTC project audit +found nine completed physical executions, **19 of 30 free minutes used**, eleven +remaining and zero unresolved executions. No paid test minutes were needed, no +VM was created and no further run was scheduled. The interactive report, exact +JSON, immutable usage receipt and runtime audit are retained under +`.dart_tool/validation/system-fix-20260917/` and the run's `report/` directory. + +### Repeated Gemma CPU controls on macOS + +The local CPU comparison uses clean source +`2b604775ac49f0532e2a0a9f34024c93d3998c10`, Apple M4 Max/macOS 26.6.2 arm64, +and LiteRT-LM `0.17.0-5`. The CPU artifact from the same pinned model repository +is `gemma3-1b-it-int4.litertlm`, 584,417,280 bytes, SHA256 +`1325ae366d31950f137c9c357b9fa89448b176d76998180c08ceaca78bba98be`. +The core library SHA256 is +`42a1fa7cc0666ceda1bb00f864065e7b6bcae041234bd501ecc1c56de54b7530`, matching +the cached release archive and published release manifest's macOS smoke record. + +Three fresh public-API processes each completed **13 PASS, 4 FAIL**, with no +ERROR/NOT_RUN and complete provenance. A separate local Python/ctypes diagnostic +used the upstream C API directly, without Dart, and repeated all four history +inputs three times with fresh engines. Its exact setter JSON matches the prior +NPU controls; its outputs match the public CPU path in all twelve comparisons. + +| Input | Public CPU, each of three runs | Direct-native CPU, each of three repetitions | +| --- | --- | --- | +| Canonical system and prior history | `Cedar17` | `Cedar17` | +| Former public literal-JSON system content | 32 repeated `7` characters | 32 repeated `7` characters | +| History without system content | `Cedar17` | `Cedar17` | +| Combined user prompt | 32 repeated `7` characters | 32 repeated `7` characters | + +Every row still fails exact `cedar17`. These failures therefore occur without +Qualcomm/NPU execution and without the Dart service, worker or streaming adapter. +This narrowed attribution but did not distinguish the converted model/tokenizer, +shared LiteRT runtime, and original model behavior. The subsequent original-model +and prompt comparison below narrows that boundary further; keep +[#513](https://github.com/leehack/llamadart/issues/513) open. + +Both CPU paths use context 1280, four threads, max output 32, thinking enabled +and greedy decoding. The public requested top-k 40/temperature zero resolves to +top-k 1 in the service; the direct C API used TopP sampler type 2, top-k 1, +top-p 0.9, temperature zero and seed 1. NPU still has unknown compiled sampling. +Platform, artifact/conversion, maximum native context capacity, and sampling +differ between CPU and NPU. Do not treat their TPS ratio as accelerator speedup +or fill the S24 CPU row with this Mac evidence. + +Across nine short public benchmark samples (three per run after separate +warmups), median native decode was **74.06 TPS**, estimated wall **67.46 TPS**, +and time to first public output **137.33 ms**. This is diagnostic throughput, +not semantic qualification. No Firebase run, VM or paid resource was created; +no cloud test minutes were consumed. The three run directories are +`.dart_tool/validation/runs/gemma-cpu-20260917-{1,2,3}`; comparison JSON, +direct-native script/results/logs and runtime/model audit are under +`.dart_tool/validation/gemma-cpu-20260917/`. + +### Original Gemma and tokenizer reference (2026-09-17) + +The original `google/gemma-3-1b-it` model at revision +`dcc83ea841ab6100d6b47a070329e1ba4cf78752` was acquired through existing authorized +access and verified against repository Git blobs/LFS hashes. Its safetensors file +is 1,999,811,208 bytes, SHA256 +`3d4ef8d71c14db7e448a09ebe891cfb6bf32c57a9b44499ae0d1c098e48516b6`. +The local reference used Transformers 5.17.0, PyTorch 2.14.0, Tokenizers 0.23.2, +CPU float32, four threads, eager attention, greedy decoding, seed 1 and at most +32 new tokens. It used the model's original chat template and the exact four +role/content inputs from the prior direct-native control. + +All four variants returned `cedar17\n` in each of three repetitions: **12/12 PASS** +under the existing trimmed, case-sensitive predicate. No assertion was relaxed. +This is a semantic reference, not a comparable speed or quantization benchmark. + +The CPU LiteRT model embeds a SentencePiece tokenizer byte-identical to the +original: SHA256 +`1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c`. +For every input, the native conversation render exactly matches the original +prompt after accounting for BOS. The native tokenizer IDs plus metadata BOS ID 2 +match the original Transformers input IDs. The pinned upstream session code adds +that BOS separately on the first turn; this comparison checks rendering and the +tokenizer API, not a trace of tensors passed to the executor. + +The NPU tokenizer differs in IDs 256000–262143 (6,144 vocabulary entries), with +the same normalizer and core special-token IDs. None of the observed input IDs +uses those changed entries. That difference is not evidence of the cause; actual +NPU prompt tokenization was not captured in this local comparison. Both LiteRT +files use non-Jinja role affixes with BOS ID 2; the NPU metadata additionally +specifies the 1280-token limit. + +These results rule out an impossible oracle and a mismatched CPU tokenizer file +for these inputs. They do not separate quantization/conversion effects from +LiteRT executor/runtime behavior. Continue #513 with a controlled alternative +conversion or upstream execution comparison; keep the strict failures and S24 CPU +gap open. No Firebase execution, VM or paid resource was used. Hash audits, +original inputs/token IDs/outputs, native rendered prompts, extracted tokenizer +metadata, dependency versions and comparison JSON are retained privately under +`.dart_tool/validation/gemma-reference-20260917/`. + +The report validator was also tightened after five regression tests demonstrated +false qualifications from omitted obligations, conflicting rehashed settings, +invalid profiles, waived accelerator flags and self-granted unsupported status. +All 43 harness tests and 44 provider/input tests pass. Revalidating copies of the +three CPU journals and latest public/native S24 journals preserves their exact +verdicts and counts, with no new integrity problems; originals remain unchanged. + +## Planned platform/backend coverage + +Inspect the model/use-case coverage catalog without downloading models or +starting cloud resources: + +```bash +dart run tool/testing/validation.dart coverage +dart run tool/testing/validation.dart coverage --platform android-arm64 --backend npu +dart run tool/testing/validation.dart coverage --use-case stt +dart run tool/testing/validation.dart coverage --use-case tts +``` + +This JSON is a planning inventory, not a qualification report. `NOT_RUN` means +execution evidence is still required; `UNVERIFIED` identifies an artifact or +compatibility gap; `UNSUPPORTED` identifies a current runtime/API boundary. +Actual results remain in collected run reports. The catalog never emits PASS. + +Gemma 4 E2B and Qwen3.5 0.8B are primary chat families. Gemma 4 Tensor G5 and +Qualcomm SM8750 NPU rows require immutable artifacts and matched vendor kits +before executable profiles can be added. SM8750 does not qualify S24 SM8650. +Existing Gemma 3 NPU profiles remain separate legacy controls. No Qwen3.5 NPU +combination is established; Apple, desktop and Web NPU paths are unsupported. + +Dedicated STT/TTS models are required exceptions to the primary chat families. +GGUF Qwen3-ASR and Qwen3-TTS need separate platform/backend execution evidence. +Typed LiteRT ASR is native CPU-only; LiteRT TTS and NPU speech are unsupported. +Current TTS produces complete audio, not playable streaming chunks. Speech +quality, real-time factor and app microphone/playback checks remain separate +from chat token throughput and accelerator availability. + +The catalog regressions run in the existing `validation-harness` local E2E +scenario and private package tests. Candidate rows cannot be passed to the +builder as runnable profiles; existing model-lock and NPU preflight requirements +remain mandatory. Browser/delegate and device-specific model memory checks are +still required for every actual run. + +### Primary model profiles and runnable speech packs + +Gemma 4 E2B now has immutable `gemma4-gguf-{cpu,metal,vulkan,cuda}` and +`gemma4-litert-{cpu,gpu}` text profiles. Qwen3.5 0.8B retains the existing +`chat-gguf-*` Q4_0 profiles and adds `qwen35-litert-{cpu,gpu}` INT8 profiles. +The new profiles disable thinking, retain strict core predicates, and record +resolved sampling and TPS with the existing reporter. Native LiteRT profiles +are not Web or NPU artifacts. Multimodal/projector profiles remain separate work. + +```bash +dart run tool/testing/validation.dart local --profile gemma4-gguf-cpu --model /models/gemma-4-E2B-it-Q4_K_S.gguf +dart run tool/testing/validation.dart speech --pack stt --backend cpu --out /tmp/new-stt-run +dart run tool/testing/validation.dart speech --pack tts --backend cpu --out /tmp/new-tts-run +dart run tool/testing/validation.dart speech --pack litert-asr --backend cpu --model /models/moonshine_tiny_5s_i8.tflite --tokenizer /models/moonshine_tokenizer.json --out /tmp/new-litert-asr-run +dart run tool/testing/validation.dart voice --chat-profile gemma4-gguf-cpu --chat-model /models/gemma-4-E2B-it-Q4_K_S.gguf --out /tmp/new-voice-run +``` + +GGUF speech downloads or verifies the locked Qwen3-ASR/Qwen3-TTS model and +projector. Dedicated LiteRT ASR requires supplied files matching the checked-in +Moonshine model/tokenizer hashes; immutable source URLs are in +`packages/llamadart_validation/assets/speech/litert-asr.json`. +Every entry point runs under the existing subprocess host's 15-minute deadline. +The `validation-speech-stt`, `validation-speech-tts`, +`validation-speech-litert-asr` and `validation-voice-round-trip` scenarios are +registered in the local E2E runner; use `--model-path`, `--mmproj-path`, or +`--tokenizer-path` to reuse local inputs. The voice scenario uses Gemma 4 CPU; +the direct command also accepts the other primary CPU chat profiles. + +Speech reports contain per-case PASS/FAIL, exact locks and fixture identity, +raw/reference transcript, WER, processing time, first partial/first playable +audio timing where available, real-time factor, and generated WAV artifacts. +Cases cover generation, immediate cancellation, subsequent request, invalid +input/recovery, independent reload and cleanup. GGUF STT additionally compares +file and bytes inputs. TTS rejects silent, nonfinite or truncated output; +playability is not a listening-quality assertion. Its first playable audio is +the final buffer, never a progress callback. The voice report preserves the +transcript and chat response and writes the synthesized response WAV. + +These new speech commands are **diagnostic local runners**, not yet portable +bundle/Firebase adapters or accepted qualification-report imports. They never +set `qualified=true`. Exit zero means the selected functional assertions passed, +not accelerator or perceptual qualification. Full microphone/playback, +noise/language/voice fixtures, mobile/Web speech packaging, and historical +speech dashboards remain open. A supported backend request still requires +actual hardware execution evidence before marking a platform/backend row green. + +### Preparation progress on interrupted runs + +Native desktop and Flutter validation write a separate `preparation.jsonl` with `preparation_progress` +records before inference: download started/finished, checksum started/verified +(or rejected), and ready only after size and SHA256 verification. Active byte +processing emits at most one progress update per ten seconds per stage; a stalled +network does not emit a heartbeat. Records contain profile ID, locked model hash, +processed/expected bytes and elapsed preparation milliseconds, never a URL or +local model path. Subtract stage timestamps to separate transfer from verification +cost. Existing completed-run `download_ms` and `checksum_ms` remain available. + +These flushed JSONL records also appear in provider logs with the +`LLAMADART_PREPARATION` prefix. They leave the manifest-first suite protocol +unchanged and survive a provider timeout where no suite +manifest was written, but cannot qualify a run or replace missing test results. +The eight-minute S24 Gemma 4 attempt reached model loading only after about 7m44s; +its old logs do not distinguish transfer and checksum cost. Use these records +before choosing a longer timeout or another model delivery strategy, and recheck +free allowance before any device dispatch. + +### Desktop CUDA payloads + +Linux x64 and Windows x64 validation bundles explicitly include CPU, Vulkan and +CUDA modules through the private harness hook configuration. Bundling fails if +any is missing; selecting a CUDA profile alone does not override Dart build-hook +defaults. The GPU driver remains a host prerequisite, and a shipped CUDA module +is not execution or placement evidence. Earlier 31867c10c CI bundles use the +CPU/Vulkan defaults and must not be used for CUDA qualification. + +### Terminal Firebase recovery and release catalog 3 + +Firebase polling retains safe failure categories and HTTP status codes without +response bodies, authentication values or request URLs. Transient read failures +retry at most three consecutive attempts under the original run deadline; the +controller never retries submission. Interrupted runs stay unqualified. Failed +Firebase runs cancel/verify terminal state before collecting final artifacts; +pre-terminal snapshots cannot be labelled complete. GCE still collects before +VM/disk deletion. An unknown cleanup state continues to block new submissions. +Manual Firebase cleanup invalidates earlier collections and refreshes provider +status. Follow it with `collect`, which refreshes terminal status again before +retrieval. A failed recovery persists an incomplete, unqualified assessment; +cancelled preparation-only runs cannot qualify from old artifacts. + +Windows compiled CLI bundles resolve native backends from their verified +`bin/../lib` layout, before unrelated working-directory or hook caches. +Explicit native overrides and executable-adjacent bundles retain precedence. + +Windows VM uploads and result collection explicitly use legacy SCP (`-O`), +because the Google Windows SSH image used in live validation closed the default +SFTP-based transfer. Linux retains the default protocol. Both remain IAP-tunneled. + +Vulkan offload and compute-buffer records do not establish physical GPU use when +the log identifies a software device such as llvmpipe, lavapipe or SwiftShader. +Those runs remain unverified, including mixed software/hardware inventories. +Each Vulkan compute-buffer device also needs matching, unambiguous native +discovery or selected-model evidence identifying recognized GPU hardware. +Both sources must agree when present. Discovery capability columns are stripped +only after checking the original device text for software identities; absent, +conflicting or unknown device names fail closed. CUDA-ready VM images need a Vulkan +loader and a hardware ICD before they can qualify Vulkan or LiteRT WebGPU lanes. + +Catalog 3 executes C10 stop-marker comparison and C12 unloaded-engine readiness +rejection/reload recovery. Stop tests retain an unrestricted control, exact +pre-marker output, forwarded stop configuration and subsequent-request recovery; +a model that does not emit the control marker fails the oracle. Readiness tests +require the public `LlamaContextException` contract. Direct native reference +controls do not stand in for either public API case. Catalogs 1 and 2 retain +these cases as unimplemented and cannot claim their execution. Catalog 4 adds +Unicode generation, thinking on/off and tool choice/result controls. Thinking +budgets, tool-bearing batching and model-specific qualification remain separate. + + +### Current Qwen tool and history reference (2026-09-19) + +The unchanged catalog-4 Qwen CPU profile from suite `b13c545f`, run in an +isolated setup against main `699969b0` after PRs #531 and #529, records +**16 PASS, 1 FAIL, 0 ERROR**. C07 auto/required/none, exact tool arguments, +typed-Map result follow-ups and recovery pass. C06 still returns `Cedar17` +instead of the strict expected `cedar17`; no predicate was relaxed. This is JIT +diagnostic evidence, not a sealed current-PR portable qualification report. + +For the locked Qwen3.5-0.8B-Q4_0 model (SHA256 +`57d1997790d1744fba5b40a7317df71ea5e2acee28c47e78f0cce39c0703f8cf`), +unmodified upstream `llama-server` at native v0.4.1's exact upstream commit +`b29c606e28a01b1bc8c1351026a0fa6e616bf6c4` matches all 11 diagnostic trials: +rendered prompts, input token IDs and public/raw Dart outputs. The original +history remains intact. Three baseline repetitions return `Cedar17`; disabling +the repetition penalty returns `17`, while changing the stored code to `maple42` +returns `maple42` and removing history produces a different output. The same +explicit-case instruction still returns `Cedar17` on both paths. + +This does not demonstrate a Dart history-loss or parser defect, and it does not +distinguish original model behavior, quantization or upstream numerical execution. +Keep the exact conformance failure and `qualified=false`. A separate multi-secret +history-transport diagnostic, if added, must not replace this obligation. +The [tracking diagnosis](https://github.com/leehack/llamadart/issues/514#issuecomment-5738623925) +records source/model/settings and attribution limits. Gemma3 LiteRT issue #513 +remains a distinct investigation; this CPU GGUF control does not qualify other +models, runtime backends or devices. diff --git a/doc/cross_platform_validation_plan.md b/doc/cross_platform_validation_plan.md new file mode 100644 index 000000000..05af1824f --- /dev/null +++ b/doc/cross_platform_validation_plan.md @@ -0,0 +1,1149 @@ +# Lightweight cross-platform validation plan + +> Model-priority update: Gemma 4 E2B and Qwen3.5 0.8B are the primary +> chat families; STT and TTS remain required dedicated-model exceptions. +> `dart run tool/testing/validation.dart coverage` exposes explicit planned +> platform/backend/use-case rows, including Gemma 4 Tensor G5, SM8750 and +> the SM8650 compatibility gap. Older Gemma 3 NPU profiles below remain +> legacy controls, not Gemma 4 qualification. See +> [the coverage catalog guide](cross_platform_validation.md#planned-platformbackend-coverage) +> and [tracker #514](https://github.com/leehack/llamadart/issues/514) for the +> current priorities and required speech acceptance cases. + + +Status: **initial quick-core harness implemented; broader qualification remains planned** +(2026-09-17). The maintainer authorized implementation after the earlier deferral. +The implementation starts from current merged main and preserves its runtime pins; +the separate release task still owns pending runtime changes. See the +[implementation runbook](cross_platform_validation.md) for available commands, +actual bundles, verified behavior and remaining qualification work. Pilot +observations below are dated 2026-09-16; the device catalog and maintained-harness +pilot were refreshed on 2026-09-17. The runbook records the newer outcomes. +Proposed later coverage is not an assertion that those rows now pass. + +The objective is a small, repeatable test of the **public llamadart package**, +including native-library packaging, model routing and application lifecycle. +Use free CI and the Mac for routine checks, and **Firebase as the primary +physical Android/iOS test route**. The personal Pixel and iPad are optional +debugging devices, never prerequisites for mobile qualification. Include +Qualcomm and Tensor LiteRT-LM NPU qualification in the next milestone. Keep the +operating budget at **$0 out of pocket**, even after promotional GCP credit expires. +The maintainer upgraded the isolated QA project to Blaze on 2026-09-17. +Execution remains conditional on verified remaining free minutes, verified +Test Lab credit coverage or a separate explicit out-of-pocket authorization; the free rotation remains the +sustainable default. See the runbook's [Blaze controls](cross_platform_validation.md#explicit-blaze-runs). + +## 1. Deliverables and boundaries + +One private shared Dart suite supplies cases, model manifests, assertions and +reporting. A small desktop CLI and thin Flutter test app invoke that suite. +The Flutter app supports local interactive runs and unattended integration tests; +Android instrumentation and iOS XCTest wrap the same cases for Firebase. +The app shows selected model/backend, progress, cancellation, result and export. + +CI produces versioned QA bundles with checksums and provenance: desktop CLI +bundles including native code assets, Android app/test APKs, iOS build inputs, +and a deployable Web bundle with its pinned runtime assets. Build each native +target on a supported host/toolchain; do not assume one host cross-compiles all +targets. Initially build/sign physical-iOS XCTest bundles on the owned Mac; +do not add signing credentials to CI as part of this plan. + +Register maintained scenarios in `tool/testing/run_local_e2e.dart` and +`tool/testing/test_matrix.dart`; classify any new example/companion with +`tool/prepare_workspace.dart`. Reuse the existing feature smokes and benchmark +helpers. Do not create a parallel collection of unregistered repro scripts. +Tests call public APIs; direct native/upstream calls are **diagnostic controls** +and cannot substitute for a passing public-package case. + +Use three selections from one case catalog; do not maintain three suites: + +| Selection | Contents | When | +| --- | --- | --- | +| Quick core | Packaging, Unicode, raw/chat stream, history, cancellation/recovery, one reload, token limit and one timing series per representative model/backend | First device qualification and relevant runtime changes; inference target under five minutes, to be measured | +| Change-focused | Quick core plus the thinking/tools/stop/batching or feature-pack cases affected by the change | Relevant PRs; run expensive positive cases on selected devices, cheap negative contracts in CI | +| Release selection | All applicable C01–C12 subcases, affected-family packs and required platform/packaging rows | Release/native-pin qualification; schedule across quota days | + +Keep the existing case IDs, expected results and release coverage when reducing +a quick run. Report omitted cases as NOT_RUN with the selection reason. Large +multimodal/speech downloads remain feature packs. The Firebase four-device core +rotation is a useful subset of release evidence, not the full release selection. + +The first implementation milestone is deliberately small: manifests, shared +cases, existing CLI/mobile/Web adapters, reliable result capture and a basic +offline report. Reuse the chat app for Flutter/Apple companion packaging checks; +do not require new desktop Flutter UIs in addition to CLI bundles. Advanced +trend charts and model conversion remain later milestones. The next milestone +adds Firebase mobile qualification and NPU packaging/evidence; the current +CPU/GPU harness remains usable while those additions are implemented. + +### Proposed repository layout + +Keep the suite in the **llamadart repository**, next to the package it validates. +Use one private Dart package named `llamadart_validation`. The package, adapters, +commands and workflow below now exist. The nested `cases/`, `manifest/` and +`results/` directories remain a possible organization as the catalog grows; +the initial implementation keeps these responsibilities in small `lib/src/` files: + +```text +packages/llamadart_validation/ + pubspec.yaml # publish_to: none; depends on local llamadart + lib/llamadart_validation.dart # shared runner and result contract + lib/src/cases/ # C01-C12 and optional feature packs + lib/src/manifest/ # model/profile validation and selection + lib/src/results/ # events, assertions and metric definitions + assets/ # pinned manifests, prompts, small media fixtures + schemas/ # manifest, event and result JSON schemas + bin/run.dart # desktop CLI; native host adapter + bin/report.dart # single host-side JSON/JUnit/CSV/HTML exporter + test/ # fast tests of selection, assertions and reports + +example/chat_app/ + lib/validation_main.dart # dedicated interactive QA entry point + lib/validation/ # thin Flutter/device/browser adapters + integration_test/validation_test.dart # unattended entry point, same suite + test_driver/integration_test.dart # reuse existing integration-test driver + android/ # app/test APK instrumentation wiring + ios/ # physical-device XCTest wiring + +tool/testing/validation.dart # build/plan/run/status/collect/cleanup CLI +tool/testing/validation/ # GCE/Firebase adapters and VM bootstrap inputs +.github/workflows/validation_bundles.yml # host-specific QA builds and artifacts +doc/cross_platform_validation_plan.md # scope, matrix and reporting contract +``` + +The shared `lib/` stays usable from Dart and Flutter Web: platform adapters +provide model/fixture access, storage and device diagnostics. Keep filesystem, +process and Flutter imports in their host adapters. The desktop CLI and Flutter +entry points depend on the shared package, which calls the public `llamadart` +API; the core package has no dependency on the suite. Do not add QA exports to +`lib/llamadart.dart` or QA commands to the user-facing CLI example. + +Prompts, model hashes, inference profiles and small licensed fixtures have one +checked-in source under `assets/`. Bundle preparation stages the selected inputs +for desktop, Flutter and Web; adapters resolve them through the same manifest +IDs without assuming a checkout or working directory. Model weights remain +outside Git. Use `.dart_tool/validation/model-cache//` for the local +cache, `.dart_tool/validation/bundles//` for build outputs and +`.dart_tool/validation/runs//` for collected events, logs and reports. +Device adapters write to their sandbox and export into that run directory; +browser adapters provide download/export. These generated paths stay ignored. + +Register the private package in `tool/prepare_workspace.dart` and give it +explicit analyze/test coverage, since root analysis currently excludes +`packages/**`. Register orchestration selections in the existing E2E runner and +testing matrix. Keep existing core unit/integration/E2E tests in place; migrate +overlapping smoke logic incrementally once the shared cases preserve its checks. +Extend the current Web build script to accept the QA entry point while retaining +its runtime-asset staging and validation. The bundle workflow builds artifacts; +Firebase submission remains an explicit invocation with the quota preflight. + +Native runtime/build fixes remain in their owning sibling repositories; this +package owns public-Dart validation and its adapters. The earlier Firebase +pilot under `.dart_tool/firebase_pilot/20260916/` remains historical evidence, +not the maintained suite location. + +### Where to get the built apps + +The download location after publication and a successful workflow run will be +**llamadart → GitHub Actions → Validation Bundles → selected run → Artifacts**. +The local `.github/workflows/validation_bundles.yml` implementation names desktop +artifacts `validation-desktop---` and app artifacts +`validation--`. The bundle manifest records the locked profile. +Draft [PR #515](https://github.com/leehack/llamadart/pull/515) publishes this +workflow and runs the tiny CPU build on relevant PR changes. Desktop jobs also +extract the archive into a temporary directory and verify command startup/profile +discovery there without inference. Use a successful run's artifact inventory; +do not infer hosted availability from a local build. + +| Target | Download / runnable output | +| --- | --- | +| Android arm64 | Installable QA APK and its matching instrumentation test APK for unattended/Firebase runs | +| Windows x64/arm64, when buildable | CLI bundle containing `llamadart-validate.exe`, required native libraries and run instructions | +| Linux x64/arm64, when buildable | CLI bundle containing `llamadart-validate`, required native libraries and run instructions | +| macOS arm64/x64, when buildable | CLI bundle containing `llamadart-validate`, required native libraries and run instructions | +| Web | Deployable QA site bundle with pinned runtime assets; serve it with the required headers | +| iOS/iPadOS | CI build inputs initially; build/sign the physical-device app and XCTest bundle on the owned Mac | + +Local builds stage the same outputs under +`.dart_tool/validation/bundles////`. Each bundle +includes its manifest, checksums, selected small fixtures and model-fetch/run +instructions; model weights are separate. Keep CI artifacts on bounded retention +and regenerate from the recorded revision and pinned inputs when needed. No +separate download server or automatic GitHub Release publication is required. + +The interactive app source stays in `example/chat_app/`, using the dedicated +`lib/validation_main.dart` build target. Desktop v1 is the CLI from +`packages/llamadart_validation/bin/run.dart`. Produce target-specific builds on +supported toolchains; a successful build remains separate from running and +qualifying it on actual hardware. + +## 2. Pilot findings and tracked work + +Pilot source was main `a5df1c4fcbb1766d26efb5b1d9becda191df89a3`, Flutter +3.47.1 / Dart 3.13.1, llama.cpp artifact `v0.4.0`, LiteRT-LM `v0.17.0-3`. +It ran actual Flutter/llamadart code on physical devices, not a model-only service. + +| Path | Galaxy S24 SC-51E, API 36, Debug | iPhone 16 Pro, actual iOS 18.3.2, Release | +| --- | --- | --- | +| GGUF CPU | App records passed before a later GPU crash; overall execution failed | Passed | +| GGUF GPU | Vulkan SIGSEGV during compute-pipeline compilation | Metal passed | +| LiteRT CPU | Answer `2` instead of `4` | Answer `2` instead of `4` | +| LiteRT GPU | Incoherent mixed-language output; native WebGPU/Vulkan adapter identified | Answer `2` instead of `4`; Metal identified | + +Track the distinct findings: + +- [llamadart-native #79](https://github.com/leehack/llamadart-native/issues/79): + Galaxy S24 GGUF Vulkan crash. The vendor compiler appears in the stack; + originating driver/runtime/artifact/integration ownership remains unproven. +- [llamadart #509](https://github.com/leehack/llamadart/issues/509): shared + LiteRT Qwen3 arithmetic failure. A wrong small-model answer alone does not + establish a Dart regression; compare identical native and model-reference runs. +- [litert-lm-native #51](https://github.com/leehack/litert-lm-native/issues/51): + Android GPU output divergence. OpenCL was unavailable, then WebGPU selected an + Adreno 750 Vulkan adapter. The fallback is evidence, not an established cause. + +Existing [native tokenizer #48](https://github.com/leehack/litert-lm-native/issues/48), +[desktop empty-chat #505](https://github.com/leehack/llamadart/issues/505), +[desktop LiteRT GPU #506](https://github.com/leehack/llamadart/issues/506), +[Windows CUDA helper #504](https://github.com/leehack/llamadart/issues/504), and +[older Android CPU qualification #476](https://github.com/leehack/llamadart/issues/476) +remain separate unless investigation establishes a common cause. + +An initial pilot-wrapper mistake set LiteRT `numberOfThreadsBatch=4`; the corrected +runs used zero. Those initial typed rejections are **not runtime bugs**. +The pilot collected no LiteRT TPS because its arithmetic assertion ran before +timing. Android Debug and iOS Release results are not a performance comparison. +These are historical observations on the named commit/artifacts. The three pilot +issues were still open when checked on 2026-09-17; neither an issue's state nor +a different commit's passing result establishes the current candidate's outcome. + +## 3. Platform and backend coverage + +The [support matrix](../website/docs/platforms/support-matrix.md) and capability +probes remain authoritative. The table below is a test selection plan, not a +promise that every backend/model combination works. Reconcile it with the +active merged pins before qualification. A published library, successful build, +or selectable backend is not inference qualification. + +| Target | GGUF profiles | LiteRT profiles | Where / coverage limit | +| --- | --- | --- | --- | +| Android arm64 physical | CPU, Vulkan; OpenCL in targeted pack | CPU, GPU; Qualcomm/Tensor NPU with matched model/dispatch libraries | Firebase device rotation; personal Pixel optional | +| Android arm64 virtual | CPU, install/load, 4K/16K page-size packaging | CPU where artifact supports it; explicit unsupported cases | Free Firebase virtual quota; no physical GPU/ISA qualification | +| Android x64 emulator | CPU; Vulkan and OpenCL targeted if actually exposed | CPU, GPU only with backend proof | Local/free CI emulator; unavailable GPU profiles remain NOT_RUN; current Firebase virtual catalog is arm64 | +| iOS arm64 physical | CPU, Metal | CPU, GPU; no Apple NPU backend | Firebase iPhones and iPad 10; personal iPad optional | +| iOS arm64 simulator | CPU, available Metal path separately labelled simulator | CPU, available GPU path separately labelled simulator | Owned Mac; does not qualify physical-device drivers or memory | +| iOS x86_64 simulator | CPU / available Metal | Negative packaging contract: no LiteRT artifact | Intel host if available, otherwise NOT_RUN | +| macOS arm64 | CPU, Metal | CPU, GPU | Owned Mac if matching architecture; record actual hardware | +| macOS x64 | CPU, available Metal | CPU; GPU unsupported by documented x64 bundle | Available free runner or Intel host; GPU gaps explicit | +| Linux x64 | CPU; CUDA, Vulkan, HIP/BLAS targeted | CPU; explicit GPU through Vulkan with `0.17.0-5` | Free CI CPU; GPU requires a compatible driver and suite execution proof | +| Linux arm64 | CPU; Vulkan/BLAS targeted | CPU | Available arm64 runner/hardware; no x64 emulation as arm64 proof | +| Windows x64 | CPU; CUDA, Vulkan/BLAS targeted | CPU; explicit GPU through D3D12 with `0.17.0-5` | Free CI CPU, accessible Windows hardware; GGUF CUDA requires actual NVIDIA GPU | +| Windows arm64 | CPU, Vulkan/BLAS where available | Negative artifact contract unless support is added | Hook/build coverage plus explicit runtime gap without hardware | +| Web, Chrome | WASM CPU and WebGPU separately | Browser CPU/GPU with Web-compatible model | CI WASM; Mac real browser GPU; mobile browser coverage separate | +| Web, Safari/iPadOS | WASM and WebGPU when exposed | Browser runtime capability-dependent | Mac Safari; iPadOS browser lane remains NOT_RUN until a browser adapter or optional device run qualifies it; Firebase native XCTest is not browser evidence | +| Web, Firefox | WASM plus explicit capability checks | Only what active browser runtime exposes | Compatibility lane; unavailable WebGPU is not a GPU pass | + +Quick core applies to each selected supported runtime row. Release selection +accounts for the full supported matrix; unavailable hardware remains an explicit +gap. Feature positives require a compatible fixture and runtime. Exhaustive +option/unsupported guards run in cheap unit/integration lanes; device profiles +retain backend probes and one error/recovery check, expanding when those paths +change. Unexpected unsupported behavior in a promised supported row is a failure. + +Backend evidence must include requested selector, effective backend name, +loaded native modules, actual adapter/driver and offload/delegate diagnostics. +An echoed `gpuLayers=999`, `availableBackends` list, or GPU preference is +insufficient. An automatic CPU fallback can pass an **auto** policy case but +cannot pass the explicit GPU row. Missing proof leaves GPU qualification +incomplete. + +## 4. Models and immutable manifests + +Do not multiply every model by every device. Run representative core models on +the device rotation; exercise affected model families and feature packs on +selected capable Firebase devices or desktop hosts. A custom manifest can add a +user's model without silently replacing a failing standard fixture. + +| Model ID | Proposed artifact | Purpose / scheduling | +| --- | --- | --- | +| `tiny-gguf` | stories15M.gguf, 98.36 MB | Packaging, raw stream, backend/lifecycle; not an instruction-quality oracle | +| `chat-gguf` | Qwen3.5-0.8B Q4_K_M GGUF | Representative chat/thinking/tools; qualify template and semantic fixtures first | +| `dense-state` | Qwen2.5-0.5B Q4_K_M GGUF | Dense context/state/prompt-reuse checks, targeted pack | +| `chat-litert` | Qwen3-0.6B.litertlm, 614.24 MB | Native LiteRT core plus retained arithmetic/tokenizer diagnostics | +| `npu-qualcomm-sm8650` | Gemma 3 1B IT, Qualcomm SM8650-specific LiteRT bundle, 4-bit per-channel, about 658 MB | Next-milestone S24 NPU qualification; separate artifact from CPU/GPU models | +| `npu-tensor-g5` | Gemma 3 1B IT, Tensor G5-specific LiteRT bundle, 8-bit per-channel, about 1.7 GB | Next-milestone Pixel 10 NPU qualification; separate vendor runtime and artifact | +| `tools-litert` | FunctionGemma 270M compatible LiteRT bundle | Optional native tool fixtures; verify actual template/tool support first | +| `gemma-gguf` | Gemma 4 E2B GGUF + matching projector, about 4 GB total | Vision/audio/thinking/tools and large-memory coverage | +| `gemma-litert-native` | Native Gemma 4 E2B LiteRT bundle, about 2.6 GB | Native multimodal pack | +| `gemma-litert-web` | Web-compatible Gemma 4 bundle, about 2 GB | LiteRT Web single-turn lane; never substitute the native bundle | +| `embedding` | EmbeddingGemma 300M Q8 GGUF, about 334 MB | Embeddings and batch consistency | +| `asr-gguf` | Qwen3-ASR GGUF + matching projector, about 1 GB | File transcription; Web WAV-only where supported | +| `asr-litert` | Moonshine tiny INT8 + tokenizer, about 54 MB | Native CPU streaming ASR | +| `tts-gguf` | Qwen3-TTS GGUF + matching projector, about 1.5 GB | Experimental typed synthesis and playback/export | + +The pilot fixtures below and the checked-in quick/NPU profiles have immutable +model locks. S24 NPU execution is observed but history qualification fails; +Tensor NPU remains unexecuted. Other artifacts/sizes +are **selection candidates**, not locked or reference-qualified inputs. +Before a row is executable, resolve +the exact repository revision, filename, byte size, SHA256, format, quantization, +tokenizer/template identity, license/access requirements, companion hashes, +supported feature flags and measured memory envelope. Reject a manifest with +missing hashes or floating revisions. Do not download gated assets without access. + +Pinned pilot fixtures: + +| ID | Repository / revision / filename | SHA256 | +| --- | --- | --- | +| tiny-gguf | `ggml-org/tiny-llamas` / `99dd1a73db5a37100bd4ae633f4cfce6560e1567` / `stories15M.gguf` | `61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a` | +| chat-litert | `litert-community/Qwen3-0.6B` / `8414150f2e9dcc82449bcc9c5abc404b399a4d06` / `Qwen3-0.6B.litertlm` | `555579ff2f4fd13379abe69c1c3ab5200f7338bc92471557f1d6614a6e5ab0b4` | + +Hash downloaded files before load; record cache hit/miss and bytes. Download and +checksum time are separate from inference. Do not assume Firebase preserves app +model caches between executions. Large packs must pass disk/memory preflight and +are excluded from older/constrained-device core unless explicitly qualified. +Within one execution, prepare/hash each immutable model once, then reuse that +verified file for its cases. Load one model at a time; retain it across independent +requests, reset conversation/cache state, and reload only for explicit lifecycle +cases. Hash checks and downloads never run inside timing windows. Start with +tiny-gguf plus chat-gguf for GGUF profiles, and chat-litert for LiteRT profiles; +the full model inventory is not an automatic download list. + +## 5. Case catalog and expected results + +All cases record expectation, actual output, assertion result and capability +decision before reporting aggregate status. Evaluate independent assertions +without throwing away earlier evidence. Cases below are shared specifications; +the model manifest determines supported positive rows and typed negative rows. + +| ID | Input / action | Expected result and scope | +| --- | --- | --- | +| C01 packaging, routing and load | Verify binary/model hashes; load by `.gguf` or `.litertlm`; probe capabilities | Correct runtime family, ready state and effective backend evidence; missing library/corrupt model produces actionable error. Native/Web assets distinguished. | +| C02 tokenizer and Unicode | Encode/decode `Hello, Montréal! 안녕하세요 👋\n two spaces`; plain-text mode with explicit BOS/special-token policy | Exact round trip where advertised, stable IDs for the pinned tokenizer; capture literal byte-level spellings. LiteRT Web asserts typed unsupported tokenizer path. | +| C03 raw stream | Tiny GGUF: `Once upon a time`, max 32; native chat-capable models use their locked raw fixture | One nonempty valid, finite ordered stream with no crash/hang. Benchmark repetitions are scheduled once separately, not repeated for every case. A stories model need not answer instructions. | +| C04 visible chat and thinking control | `Reply with one short sentence saying hello.` and `What is 2+2? Answer only with the number.`; thinking disabled | Greeting contains `hello` case-insensitively; arithmetic trimmed matches `^4[.!]?$`; no hidden-thinking leakage. Existing LiteRT failures remain visible until reference triage. | +| C05 thinking and budget | Same arithmetic prompt with thinking on, max 256; native GGUF budget 0 and 32 on a qualified reasoning fixture | Final visible answer; correctly separated thinking when emitted; no leaked delimiters. Budget behavior checked against native counters/reference semantics, not character count. LiteRT/Web reject native GGUF-only budget controls. | +| C06 conversation and system messages | System `Answer briefly.`; user `Remember this code: cedar-17.`; assistant acknowledgement; user `What code did I ask you to remember? Reply only with the code.` | Native/GGUF history retains `cedar-17`; captured serialized messages preserve system role. LiteRT Web single-turn limitation is explicitly reported; no claimed history pass. | +| C07 tools | Tool `get_weather(city: string)`; user `Call get_weather for Montréal.`; modes auto, required, none; fixed tool response | Required emits the named tool and parsed `{"city":"Montréal"}` where supported; none emits no tool; auto accepts permitted text or valid call. Tool response continuation tested where supported. Unsupported constrained modes fail explicitly. | +| C08 cancellation and reuse | Start long generation; cancel after first nonempty delta, await completion, then issue hello on a new/reset conversation | Stream terminates within 5 seconds, engine remains usable, no late deltas after completion; regeneration passes. If generation already finished, mark cancel subcase NOT_RUN and use the locked longer fixture. | +| C09 dispose and reload | Load/generate/dispose, then new engine/load/generate once in quick core; twice for lifecycle-focused changes or release selection | Clean shutdown; no live worker/stream handles; repeat completion. Record memory after each cycle without demanding identical RSS. | +| C10 limits and stop markers | Locked fixture reliably emitting a chosen marker, plus max-token limit case | Advertised stop/limit semantics, correct finish classification, marker suppression according to API contract; no uncontrolled continuation. Unsupported stop options rejected. | +| C11 native stream batching | Compare supported defaults with one explicitly supported batching configuration at deterministic settings | Same reconstructed content/tool/thinking, ordered completion; chunk count may differ. LiteRT Web must reject nondefault native batching controls. | +| C12 failure and recovery | Missing file, bad model hash, unsupported option, unsupported feature, cancelled download; reload valid fixture | Typed actionable errors, no false success or permanently stuck worker; valid recovery completes. Package/model format misrouting and missing companion are covered. | + +For model-dependent expectations (C04–C07/C10), first obtain reference behavior +from the exact fixture and sampler. Keep semantic failures distinct from transport, +parser and runtime failures. Do not demand exact free-form prose or universal +CPU/GPU token identity. Determinism checks are within a pinned cohort; parity +assertions use the specific contract/reference under test. + +LiteRT Web runs C01, applicable single-turn C03/C04, lifecycle C08/C09 and C12, +plus explicit unsupported tokenizer/history/tool/native-control cases. It does +not inherit native conversation capabilities just because both use `.litertlm`. + +The catalog enumerates C01–C12; a run manifest expands only its selection into +mandatory device/model/backend/subcase rows before execution: + +- Quick core: C01–C04, C06, C08, C09's single reload, C10's token-limit subcase, + and C12's invalid-input/recovery subcase, plus one benchmark series. +- Change-focused: add C05 for reasoning changes, C07 for tool/template changes, + C10's stop-marker case and C11 for streaming/stop/batching changes, and the + applicable feature packs. Load/dispose/worker/lifecycle changes require C09's + second reload and the relevant full C12 failure/recovery subcases. + Parser/template changes require affected-family + fixtures and positive/negative modes, even if that expands the quick selection. +- Release: all applicable C01–C12 subcases and existing required release rows. + +Web applies the same selection rules to its documented positive and negative +contracts. A model lacking a runtime-supported feature leaves that positive +subcase NOT_RUN with a fixture reason; it is not runtime UNSUPPORTED. Selecting +only tiny-gguf cannot qualify chat. Missing a mandatory row makes that selection +incomplete; accounting for a missing test never turns it into a pass. + +## 6. Feature packs + +| Pack | Fixtures / assertions | Selection | +| --- | --- | --- | +| Structured output | Prompt `Return an object with count 3 and ok true.`; schema with required integer `count`, boolean `ok`, no additional properties. Parse exactly; test invalid types/unknown keys, partial-stream suppression, malformed-final rollback and tool modes with thinking prefixes. | Supported GGUF grammar path; compiled grammar acceptance/rejection and affected-family upstream parity remain mandatory existing checks. LiteRT grammar negative case. | +| State and prompt reuse | Dense model: full recomputation vs reused prefix; save/restore same continuation; malformed/version-mismatched state | Native and capable GGUF Web bridge. Web virtual-file persistence is not durable reload storage; test explicit export/import separately. LiteRT unsupported. | +| Embeddings | `A cat sits on a mat.`, `A feline rests on a rug.`, `The engine uses diesel.`; single vs batch | Correct dimension, finite/nonzero vectors, single/batch tolerance; reference-qualified similarity ordering. GGUF supported paths; LiteRT negative. | +| Vision | Bundled synthetic red square on white; `What color is the square? Answer with one word.` | `red`, media reaches model, missing/wrong projector rejected. Native/GGUF Web and native LiteRT compatible bundles; browser LiteRT unsupported as documented. | +| Audio understanding | Fixed WAV saying `The meeting is on Tuesday.`; ask day of meeting | `Tuesday` with model-qualified tolerance; file/blob paths and invalid media errors. No live microphone needed for deterministic lab case. | +| ASR | Fixed licensed PCM WAV `The quick brown fox jumps over the lazy dog.`; batch and chunked streaming where supported | Normalized transcript matches reference, correct sample-rate validation, finalization/cancel/restart; log WER and real-time factor. Moonshine native CPU; Qwen3 native and qualified Web WAV route. | +| TTS | Text `Hello from llamadart.` and a supported speaker configuration | Finite nonempty PCM, valid rate/channels/duration, playable/exportable WAV and cancellation; listening quality remains separate local QA. | +| LoRA | Locked matching base/adapter; baseline, adapter active, reset | Supported load/runtime semantics, reference-qualified output effect; reject incompatible/aLoRA/version-skew cases. Native LiteRT only one default-scale load-time adapter; reject runtime updates/stacking/scaling. | +| Speculative | Same target/prompt/sampler with drafting off/on; locked compatible draft if needed | Output parity against appropriate upstream reference, accepted/drafted counts and speed; distinguish MTP, n-gram, external draft and native LiteRT controls. No unsupported Web substitution. | +| Runtime controls | Threads 1/4, relevant activation type/prefill setting; full/compact CPU packaging; explicit CPU/GPU/fallback | Effective values/backend proof, supported behavior and typed invalid-combination failures. Unsupported/NPU deployments do not silently pass on CPU. | +| App/device/browser | Download cancel/resume/cache, background/foreground, rotation, memory warning, Web worker restart, denied microphone, audio playback/export | App stays responsive, progress/cancel honest, recoverable errors and safe cleanup. Owned hardware covers human audio and sustained thermals; cloud media fixtures cover deterministic paths. | + +Run affected-family fixtures when templates/parsers change; a convenient tiny +model is only pipeline evidence. Full and compact Android CPU packages both +need physical lower-ISA and modern-device coverage for the relevant release. +Neither a modern phone running compact nor an emulator closes issue #476. + +## 7. Prompt and inference configuration contract + +Store case prompts as versioned UTF-8 data. Reports include literal test prompts, +message roles, tool schemas, template identity/options, media fixture hashes, +expected predicates and case version. Use synthetic fixtures only; never collect +personal chat history or microphone content for unattended runs. + +Original profile design sketch (the executable JSON contract is now in +`packages/llamadart_validation/assets/profiles/`; this sketch is not CLI input): + +```yaml +schema_version: 1 +profile: native-quick +model_id: chat-litert +case_ids: [C01, C02, C03, C04, C06, C08, C09, C10.limit, C12.recovery] +load: + context_size: 1024 + threads: 4 + batch_threads: 0 # LiteRT; GGUF profile uses 4 + backend: cpu # separate execution for explicit gpu +generation: + max_tokens: 32 + temperature: 0 + seed: 1 + enable_thinking: false +benchmark: + id: short-generation + prompt: "List the numbers from one to twenty in English." + warmups: 1 + measured_runs: 3 +timeouts: + generation_seconds: 60 + model_prepare_seconds: 300 # CLI; native Flutter uses 600 + cloud_execution_minutes: 20 +``` + +GGUF raw timing uses `Once upon a time`; instruction-model timing uses the list +prompt above. C05/C08 and feature packs override token budgets explicitly. +By default benchmark only chat-gguf or chat-litert once per backend, with one +warm-up and three measured samples total; do not multiply repetitions by case +count. Tiny-gguf timings are packaging diagnostics unless explicitly selected as +a separate benchmark cohort. If both models are benchmarked, report two series. +Default context is 1024; tools/feature packs can request 2048 with a distinct +configuration hash. Standard backend comparisons use the same settings and model. +Do not compare this 32-token core smoke to long-context throughput. + +At build/run time expand **all** remaining package/runtime defaults into the +effective record (top-k, top-p, min-p, penalties, stop strings, batch thresholds, +cache/prompt-reuse policy, GPU hint, activation type, prefill and speculative +settings). Include unsupported/null provenance rather than inventing values. +The frozen runtime revision plus effective configuration hash identifies the +experiment. Capability validation happens locally before a cloud submission. + +For warm measurements, retain the loaded model but start a fresh conversation +and reset request/KV state unless testing reuse. Record actual cache/reuse +behavior; if it cannot be disabled or verified, label the cohort as warm-cache +and do not compare it to uncached prefill. Native initialization may be lazy: +`loadModel` return time and first inference readiness are separate measurements. + +## 8. Firebase device selection and free rotation + +The live `gcloud firebase test ... models list` catalog on **2026-09-17** contained +205 Android entries (196 physical, nine virtual) and six iOS models. Counts +include device/form-factor variants, not 196 distinct useful inference targets; +capacity is not a reservation. The iOS catalog contained iPad 10, iPhone 8, +11 Pro, 14 Pro, 16 Pro and SE 3. Refresh supported OS/version and capacity before +every submission using the [Android](https://firebase.google.com/docs/test-lab/android/available-testing-devices) +and [iOS](https://firebase.google.com/docs/test-lab/ios/available-testing-devices) +catalog guidance. + +Choose for a new driver/SoC generation, CPU instruction baseline, OS boundary or +form factor. Plan coverage independently of access to personal mobile devices. + +| Priority | Device ID / catalog OS | Added coverage | Initial cases | +| --- | --- | --- | --- | +| A | Galaxy S24 `SC-51E` / API 36; low capacity | Pilot's verified Adreno 750 crash/output-divergence reproducer | Four isolated GGUF CPU/Vulkan and LiteRT CPU/GPU profiles; #79/#51 diagnostic controls | +| A | Lenovo Tab P12 `TB370FU` / API 35; medium capacity | MediaTek Dimensity 7050 / Mali-G68, non-Pixel/non-Qualcomm driver stack, Android tablet | Same four core profiles; tablet app flow when relevant | +| A | iPhone 16 Pro `iphone16pro` / 18.3; medium capacity | Repeatable pilot Apple reference; actual OS was 18.3.2 | Four CPU/Metal and LiteRT CPU/GPU core profiles | +| A | iPhone SE 3 `iphonese3` / 26.3; medium capacity | Newer catalog OS on a different, older Apple device generation | Four core profiles, lifecycle and small-screen flow; retain 18.4 as optional same-model OS control | +| B | Galaxy A05s `a05s` / API 35; low capacity | Snapdragon 680 / Adreno 610, older/budget CPU and GPU; candidate lower-ISA qualification device | GGUF CPU full + compact first; probe CPU features and selected module before claiming older-ISA coverage; GPU and LiteRT next | +| B | iPhone 8 `iphone8` / 16.6; medium capacity | Older Apple hardware and nearest available OS to current iOS minimum 16.4 | Tiny GGUF CPU/Metal first; representative model only after memory preflight; LiteRT and resource failures explicitly recorded | +| A, targeted | iPad 10 `ipad10` / 16.6; medium capacity | Apple tablet and older-OS coverage without a personal iPad | GGUF Metal and LiteRT GPU core plus layout/lifecycle; CPU rows remain NOT_RUN unless separately selected | +| C | Pixel 5 `redfin` / API 30; high capacity | Older Android OS when installation/runtime compatibility changes | Packaging/CPU core, then affected backend; different purpose from A05s ISA checks | + +Hardware family sources: [Lenovo Tab P12 specifications](https://psref.lenovo.com/syspool/Sys/PDF/Lenovo_Tablets/Tab_P12/Tab_P12_Spec.pdf), +[Samsung A05s specifications](https://www.samsung.com/africa_en/smartphones/galaxy-a/galaxy-a05s-silver-128gb-sm-a057fzsgafc/), +and [Qualcomm Snapdragon 680 brief](https://www.qualcomm.com/content/dam/qcomm-martech/dm-assets/documents/product_brief_-_snapdragon_680_4g_mobile_platform.pdf). +Probe actual RAM, CPU features, GPU/driver and OS build in each execution; do not +infer a lab SKU's RAM or instruction support from its marketing name. S24 regional +variants can differ; retain `SC-51E`, not just `Galaxy S24`. + +Virtual candidates in the current catalog: + +- `MediumPhone.arm`, API 29 and 36: older/current Android CPU and installation + checks, subject to the final test app's minimum SDK. +- `MediumPhone_ps16k.arm`, API 36: real 16K-page emulator packaging/load checks. +- `MediumPhone_ps16k_backcompat.arm`, API 36: separate compatibility-mode case + when changing native alignment/loading; never substitute it for native 16K. + +These are arm64 virtual entries. Keep Android x64 coverage in a local/CI emulator. +GPU-capable virtual infrastructure does not prove a physical Adreno/Mali driver, +device NPU, lower-ISA CPU selection or real-device memory behavior. + +### Next-milestone LiteRT-LM NPU test pack + +Qualify **Galaxy S24 first, then Pixel 10**. The S24 pilot now establishes NPU +participation through both native and public adapters, but public history fails +([#513](https://github.com/leehack/llamadart/issues/513)); full qualification remains +blocked. See the [dated result](cross_platform_validation.md#galaxy-s24-npu-pilot-2026-09-17). +The later [native replay on current main](cross_platform_validation.md#native-history-replay-on-current-main--litert-lm-0170-5) +reproduced repeated tokens with the public system JSON on `0.17.0-5`; canonical +native history recalled `Cedar17`, still failing exact capitalization. Fixing +serialization alone does not yet qualify the model/path. +The [public replay after the fix](cross_platform_validation.md#public-history-replay-after-the-system-content-fix) +now also returns `Cedar17`: 13 PASS, one strict history FAIL, with NPU +participation verified. The service correction is locally committed; it is not +a merged release fix. [Repeated Mac CPU controls](cross_platform_validation.md#repeated-gemma-cpu-controls-on-macos) +now reproduce all four history failures in three public runs and three direct +C API repetitions, so the behavior is not confined to NPU or Dart. Original-model +and tokenizer/template reference checks remain necessary; the separate S24 CPU +device row remains unrun. +Pixel 10 remains planned. Neither catalog availability nor a built bundle proves +NPU inference on another target. +Google's [LiteRT-LM NPU guide](https://developers.google.com/edge/litert/next/litert_lm_npu) +documents SoC-specific Gemma 3 1B models for Qualcomm SM8650 and Google Tensor G5. +The [Google Tensor SDK](https://developers.google.com/edge/tensor-sdk) remains +labelled beta; verify required SDK/model access before selecting that row. + +| Order | Device / catalog OS | Required NPU target | Purpose | +| --- | --- | --- | --- | +| 1 | Galaxy S24 `SC-51E` / API 36 | Qualcomm SM8650, Snapdragon 8 Gen 3; matching Qualcomm dispatch and QAIRT/HTP libraries | NPU participation verified; resolve history failure and finish matched controls before qualification | +| 2 | Pixel 10 `frankel` / API 36 | Google Tensor G5; matching Google Tensor dispatch/runtime | Independent vendor path; catalog snapshot reports high capacity | +| Alternate | Pixel 10 Pro `blazer` / API 36 | Tensor G5, separately recorded device/OS/driver cohort | Use only if the base Pixel 10 is unavailable; snapshot reports low capacity | +| Later | Galaxy S25 Ultra `pa3q` / API 35 or 36; OnePlus 11 `CPH2449` / API 34 | Corresponding SM8750 or SM8550 model/runtime | Optional Qualcomm-generation expansion after the first two paths work | + +Device/OS entries are from the 2026-09-17 Firebase catalog snapshot. Confirm the +actual `ro.soc.model`, ABI, OS build and driver on device, then require an exact +manifest match before loading. S24 variants with other SoCs are not substitutes. +See [SC-51E hardware specifications](https://www.docomo.ne.jp/support/product/sc51e/spec.html) +and [Pixel 10 Tensor G5 specifications](https://blog.google/products-and-platforms/devices/pixel/tensor-g5-pixel-10/). +Catalog presence establishes access to a device, not its NPU permissions or +working dispatch libraries inside the Test Lab app sandbox. + +The Pixel 9 Pro's Tensor G4 and the Tab P12's Dimensity 7050 are absent from +the documented LLM NPU model table, so they are not positive NPU targets in this +pack. Current llamadart Apple backends expose CPU/GPU only. Existing GPU tests +remain useful and do not qualify any of these devices' NPUs. + +**Preparation implemented:** the two immutable SoC-specific candidate profiles +and `validation.dart npu-preflight` now check local model/kit hashes, runtime +identity and host/DSP library architectures. The S24 model was staged and checked; +same-source Qualcomm/Tensor dispatch libraries were compiled in the native owner +worktree. Its diagnostic proxy distinguishes completed synchronous calls from +failures and async submissions, with model-free forwarding/negative tests. +These are input/build checks, not device execution evidence. See the +[NPU input runbook](cross_platform_validation.md#npu-input-preparation). + +**Android implementation:** the opt-in builder embeds verified local weights and +vendor libraries, checks final APK entries, and repeats those checks before remote +upload. The installed host verifies SoC/API/ABI and installed file hashes before +loading. Separate `public_api` and `native_c_api` bundles capture raw dispatch +snapshots and retain distinct path identities in the same report contract. Both +NPU paths use compiled runtime sampling defaults: the current public NPU adapter +does not apply requested session sampler overrides. Effective sampler values +remain unknown, so the deterministic cancellation-prefix check may stay NOT_RUN. The +native control calls the C API on a dedicated isolate, bypassing public Dart +backend/worker bindings. Native source/build ownership remains in the native repo. + +The public path runs 14 quick chat cases; the control now runs twelve (load, +hello/arithmetic, four history controls, reload, warmup and three measured +generations). The additional controls compare canonical system/history seeding, +the observed public system JSON, history without system content, and a combined +prompt; all retain the exact `cedar17` oracle and per-generation NPU proof. +N01/N03/N04/N06 +are covered to this bounded scope. N02's additional Unicode generation fixture, +the compatible CPU Gemma control, and aggregate matching of the reference/public +reports remain unimplemented; C02 tokenizer coverage runs only on the public +path. A single passing run cannot complete the whole NPU pack. The installed S24 +apps now prove driver access, initialization and NPU participation, and report +measured TPS; the public history failure remains a qualification blocker. Other +device rows remain NOT_RUN until matching installed apps execute. Positive +counters prove NPU participation with CPU partition coverage unknown, not full +NPU placement. + +**Preflight before spending a device execution:** + +1. Resolve model access and freeze the exact `.litertlm` revision, SHA256, + quantization, compiled SoC, context limit and tokenizer/template metadata. + Use the vendor-specific Gemma fixture, not the pilot's generic Qwen3 bundle. + No gated download or SDK terms acceptance is implied by adding this plan. +2. Resolve compatible native runtime, vendor dispatch library and all dependent + libraries, including Qualcomm QAIRT/HTP host and DSP libraries where required. + Pin their versions/checksums and verify permitted packaging. Inspect final + APK contents and native dependencies; the inspected pilot APK contained no + Qualcomm NPU/dispatch libraries. A library search path alone cannot supply them. +3. Arrange model/library transfer into the application sandbox through a supported + test mechanism. The upstream `/data/local/tmp` CLI recipe is a diagnostic + reference, not proof an installed Flutter app can access the same files or + DSP libraries. Keep credentials out of APKs, logs and exported artifacts. +4. Use explicit `LiteRtLmBackendPreference.npu` and a validated + `liteRtLmDispatchLibDir`. Keep llama.cpp-only options at supported defaults + (`numberOfThreadsBatch=0`). Lock context 1280 for the documented Gemma NPU + fixtures, max output 32, requested temperature 0 and seed 1, and one warm-up + plus three measured runs. The current NPU runtime cannot accept the requested + sampler overrides; record compiled runtime defaults and unknown effective + sampling explicitly. Do not label these runs greedy or seeded. Do not apply Qwen-specific + thinking/template options to Gemma. Native automatic controls stay at their + documented defaults unless the exact NPU artifact supports an override. +5. Build a minimal direct-native reference test and the public Dart test with + the same model and native/runtime/library inputs. Both must run as installed + app tests without root, protected vendor-directory writes or special device + modifications. Missing access/library/model prerequisites leave the row + NOT_RUN with a concrete reason; do not burn cloud quota on an incomplete APK. + +| Case | Action | Required evidence / expected output | +| --- | --- | --- | +| N01 identity and load | Load the exact SoC-matched Gemma artifact through explicit NPU selection | Correct artifact/library hashes; native dispatch initialization and actual NPU graph/partition execution evidence. A selector or `availableBackends` string alone is insufficient. | +| N02 semantic reference | Native reference and public Dart each run C04 hello/arithmetic, the C02 tokenizer encode/decode fixture, and a separate generation prompt `Reply with exactly: Montréal 👋` | Apply C04 semantic oracles and C02 exact tokenizer round-trip rules separately. The generation fixture expects trimmed `Montréal 👋` after reference qualification; it cannot substitute for tokenizer coverage. Preserve raw native text and Dart content/finish handling; no corruption or silent empty response. | +| N03 bounded throughput | `List the numbers from one to twenty in English.`; one warm-up, three measured generations | Coherent output and clean finish; cold readiness, TTFA, actual token counts, native prefill/decode TPS when available, estimated wall TPS and memory. Missing counters remain null. | +| N04 lifecycle | Cancel a qualified long fixture after first output; reload and generate again | Bounded stream termination and usable engine, no stale callbacks or native crash; use the C08/C09 rules. | +| N05 failure contracts | Unit/local fixture checks for missing dispatch dependency and mismatched SoC/model manifest | Preflight rejects incompatible manifests; missing native dependency produces actionable diagnostics. Do not deliberately load an incompatible compiled model on the lab device. | +| N06 explicit backend integrity | Inspect native execution/delegate evidence during each successful generation | Record partition placement and any CPU fallback. A CPU-only result fails the requested NPU row; mixed execution is labelled hybrid, never reported as all-NPU. Missing proof leaves qualification incomplete. | + +The direct-native control must be compared on the same target device/OS and +exact compiled artifact. A separate CPU Gemma model can help diagnose prompt or +model quality, but its weights/quantization/conversion may differ: log those +differences and do not present its TPS ratio as a pure NPU speedup. Similarly, +do not run an NPU-compiled file on CPU unless the artifact explicitly supports it. + +Initially schedule three separate executions per device: **direct-native NPU +reference, public llamadart NPU, and a compatible CPU semantic control**. Each +test adapter can contain multiple cases and timing repetitions within that one +execution. Keep the native reference and Dart path in separate app processes so +a crash or initialization state cannot conceal the difference. In a first smoke, +stop before the Dart cloud submission if the native control cannot initialize; +retain the unspent quota and record Dart qualification as NOT_RUN. + +Append S24 on day 5 and Pixel 10 on day 6 of the proposed rotation: **six NPU +qualification executions over two additional quota days**, bringing core plus +this initial NPU pack to 22 executions across at least six days. The targeted +iPad row on day 7 brings the initial mobile selection to **24 executions over +at least seven quota days**. This stays within four planned physical executions/day +and preserves the fifth daily slot for an explicit diagnostic rerun. Do not add +NPU to an already-full four-profile core day. +Further repetitions, alternate devices, or artifact experiments require extra +quota days under Spark; no automatic retries or automatic billing upgrade. + +Three controls per device are the initial qualification budget, not an obligation +for every later Dart-only edit. After qualification, rerun current public-Dart +NPU cases on relevant changes; rerun the native control when native/model/vendor +inputs, template/sampling settings, SoC, OS or driver change, or results diverge. +Reuse a CPU semantic reference only when its model/configuration/template and +device/OS identities still match. Show reused +controls as dated reference links, never current-head passes. An explicit release +or issue-verification requirement to rerun a control overrides this optimization. + +Reports add NPU/hybrid rows to the existing heatmap and TPS panels, with vendor, +SoC, compiled-model hash, dispatch/QAIRT version, partition placement and proof +links. A usable NPU result requires correct public-package output, lifecycle +completion and verified NPU execution; missing evidence or native failure never +becomes a green backend badge. No NPU run is scheduled by this document update. + +### Budget and scheduling + +The original rotation below describes Spark, which requires an **unbilled +project**. Project `llamadart-device-qa-20260916` is now on Blaze and uses its +separate free-minute guard; the original Spark execution count is no longer its +free allowance. Firebase currently allows five physical executions and ten virtual +executions per project/day on Spark. Each device configuration, retry and shard +can consume another execution; five devices times four profiles is twenty +executions, not five. See [official quotas](https://firebase.google.com/docs/test-lab/usage-quotas-pricing). + +Reserve at most **four planned physical executions/day**, leaving the fifth +for an explicit diagnostic rerun. Disable automatic flaky retries, sharding and +device/OS Cartesian expansion. Inspect remaining quota and other active matrices +before submitting; insufficient quota means NOT_RUN/reschedule, never automatically +enabling billing or creating projects to bypass the allowance. + +An explicitly authorized Blaze batch uses the same bundle and cleanup flow with +a live exact-account check and a gross-cost reservation for every submission. +The local cap replaces the Spark four-run guard only for that explicit mode; +free minutes and expected credits are not subtracted from its reservations. +For zero-cost Blaze runs, verify remaining physical minutes against each selected +provider timeout plus a rounding reserve. Spread the rotation across additional +days if its elapsed test time would exceed 30 free physical minutes in one day. +Keep the authorization window fixed, preserve the shared journal, and refresh +quota and funding evidence before each run. No automatic paid retry is permitted. +The first batch completed the S24 native NPU reference, public llamadart NPU +after reference initialization, and the existing Qwen CPU retry. All three were +collected and completion-verified; the public history and CPU arithmetic failures +remain explicit. That retry does not satisfy the matched Gemma CPU control still +required by the NPU pack. The batch consumed six rounded free physical minutes; +the full day's inventory, including earlier Spark tests, totalled 17 of 30. +Credit balances with unspecified service coverage do not authorize paid dispatch. +Blaze's verified free minutes can fund a shorter run: the runner reserves its +full provider timeout plus a rounding minute, then requires a fresh project-wide +usage check before reclaiming the unused reservation. Never treat the upgraded +physical execution-count quota as extra free minutes. + +| Day of a release rotation | Device | Planned executions | +| --- | --- | --- | +| 1 | S24 | GGUF CPU, GGUF Vulkan, LiteRT CPU, LiteRT GPU = 4 | +| 2 | Tab P12 | Same four profiles = 4 | +| 3 | iPhone 16 Pro | GGUF CPU, GGUF Metal, LiteRT CPU, LiteRT GPU = 4 | +| 4 | iPhone SE 3 / 26.3 | Same four profiles = 4 | +| 5 | S24 / SM8650 | Native NPU reference, public llamadart NPU, compatible CPU semantic control = 3, after NPU preflight | +| 6 | Pixel 10 / Tensor G5 | Same three NPU qualification/control profiles = 3, after NPU preflight | +| 7 | iPad 10 / 16.6 | GGUF Metal and LiteRT GPU core, each with tablet layout/lifecycle = 2 | + +The four-device CPU/GPU core takes **16 executions across at least four +quota days**; including NPU and the targeted iPad checks takes **24 executions +across at least seven quota days**. Missing NPU prerequisites defer those rows +with an explicit NOT_RUN reason; they do not require a personal device or block +independent CPU/GPU checks. This qualifies selected rows only, not the complete +supported platform/release matrix. Older-device, CPU full/compact, OpenCL and +large feature-pack executions extend the rotation on additional days. Do not +spend the reserved rerun automatically. For a narrow runtime change, run its CPU/GPU pair +on two relevant devices in one day instead of all four profiles on one device. + +Before dispatch, produce a local selection summary with the exact case/model +rows, model bytes, build artifacts, execution count and quota-day assignment. +For a narrow change, choose one representative of each affected hardware/driver +family first; expand on failure or when release coverage requires it. Apply only +to the changed runtime: a LiteRT-only change does not automatically spend quota +on both GGUF profiles. Preserve the omitted rows as NOT_RUN, with their reason. +The published rotation is the initial coverage schedule, not a recurring job. +A LiteRT-only follow-up across the same four core devices therefore needs eight +CPU/GPU executions over at least two quota days, rather than sixteen for both +runtimes. This saves runs by selecting scope, not by claiming fresh GGUF evidence. + +Use the Mac and free CI for frequent checks. Run Firebase as the primary mobile +lane on native pin/backend/packaging changes and release candidates, not on every +documentation or pure-Dart PR. Personal Pixel/iPad runs are optional diagnostics +and never a prerequisite to lab submission. Select virtual packaging cases only +when needed and count them against the shared ten/day limit. + +The initial five pilot submissions finished (one intentionally cancelled). +The initial CPU pilot ran with billing disabled and needed no GCP credit. A later +live check on 2026-09-17 verified `billingEnabled: true` after the explicitly +authorized Blaze upgrade. Spark plans correctly reject this billed project; +future tests require the explicit Blaze configuration. Subsequent S24 runs +established native/public NPU participation and CPU download recovery, while +retaining semantic failures. See the [dated run evidence](cross_platform_validation.md#galaxy-s24-npu-pilot-2026-09-17). +Test Lab executions have no idle VM to stop. Any later +Compute Engine CUDA testing is a separate action: verify credit eligibility and +remaining balance first, and account for disks/IP/storage after stopping a VM. +Stopping compute does not guarantee every associated resource is free. + +Android Device Streaming is optional interactive debugging, with a separate +30-minute/project/month free allowance at this snapshot; it is not extra +automated execution quota. End a streaming session explicitly. The free rotation +does not require Blaze for either service; Device Streaming is outside the +authorized automated-test batch. + +## 9. Remote execution, provisioning and cleanup + +The orchestrator owns the full **prepare → upload → run → collect → cleanup** +flow. Start with `gcloud` driven from the owned Mac and a local run journal; +no always-on controller or Terraform deployment is needed for the initial lanes. +CI builds downloadable bundles. It does not automatically create VMs or submit +Firebase tests. A separately authorized remote run consumes those exact bundles +without rebuilding the test app on the destination. + +### One command interface and resumable run record + +Implement these subcommands in `tool/testing/validation.dart`, with provider +logic under `tool/testing/validation/`. They are **proposed commands**, not +available commands to execute today: + +```text +dart run tool/testing/validation.dart plan --target gce-linux-cuda --bundle --profile gguf-cuda --out +dart run tool/testing/validation.dart plan --target firebase-android --bundle --profile litert-cpu --device --out +dart run tool/testing/validation.dart run --plan +dart run tool/testing/validation.dart status --run-id +dart run tool/testing/validation.dart collect --run-id +dart run tool/testing/validation.dart cleanup --run-id +``` + +Also support `gce-windows-cuda` and `firebase-ios` targets. `plan` performs only +local/read-only checks and writes the exact selection, bundle hashes, account, +project, device or VM specification, deadlines, cost/quota preflight and cleanup +policy. Project/account values come from explicit local configuration; never +change the user's default `gcloud` account/project. Require a fresh quota/credit +check immediately before `run` makes remote changes. Missing eligibility or +configuration leaves the selection NOT_RUN with a concrete reason. + +`run` uploads and starts the selected tests, collects available evidence and +cleans up automatically, including failure paths. `status`, `collect` and +`cleanup` reconnect to an existing run; they never create a replacement or +silently rerun inference. An explicit retry gets a new attempt ID and preflight. +Cleanup of an active run first cancels it, then attempts bounded collection. + +Under `.dart_tool/validation/runs//`, persist `run-plan.json`, +`orchestration.json`, append-only `remote-events.jsonl`, and `cleanup.json` next +to the common manifest and test results. Write mutation intent before each +provider call and save operation, instance, disk, matrix and result-location IDs +as soon as known. Use stable run/attempt IDs and reconcile uncertain submissions +before retrying. Record preparation, execution, evidence collection and cleanup +as separate outcomes; passing inference cannot hide failed or unknown cleanup. + +### Disposable Compute Engine CUDA VMs + +| Phase | Planned implementation / completion evidence | +| --- | --- | +| Preflight | Require a separate explicitly selected personal GCP project, active applicable credit and expiry, GPU quota, available region/machine, and a conservative per-run estimate covering compute/GPU, Windows licensing where relevant, disks, IP and transfers. Include other known credit use and a reserve. If coverage cannot be established, do not provision. | +| Create | Create one VM per attempt with an immutable OS image ID, pinned driver/bootstrap inputs, run labels and a persisted resource ledger. Set an absolute deletion deadline at creation, default 60 minutes after dispatch, earlier than credit expiry with a margin. Set auto-delete on every newly created disk; avoid snapshots, reserved addresses, buckets and NAT services for this lane. | +| Bootstrap | Linux shell or Windows PowerShell installs only required runtime/driver prerequisites. Verify driver readiness, actual GPU and CUDA compatibility before uploading tests. Bootstrap is restart-safe and bounded to 20 minutes; failure triggers collection and cleanup. Record actual installed versions, image, GPU, driver and bootstrap hashes. | +| Upload | Fetch and verify the exact CI bundle on the controller, then transfer it over authenticated SSH/SFTP, preferably through IAP. On Windows enable Google's supported SSH package in instance metadata/bootstrap. Use a run-specific remote directory; verify checksums again on the VM. No GitHub token or GCP key is embedded in the bundle or startup metadata. | +| Execute | Invoke the packaged CLI through a tracked shell/PowerShell job using the locked manifest/profile. Download and hash selected models with the existing five-minute preparation bound; run with a 20-minute test timeout. Track the remote job identity so reconnecting observes the existing job. CUDA requires actual offload evidence; it does not establish LiteRT GPU support. | +| Collect | Incrementally copy bounded events/diagnostics to the controller, then retrieve complete JSONL, samples, logs and crash evidence. Allow at most ten minutes for final collection, constrained by the deletion deadline. Missing evidence is reported explicitly; it never extends the VM lifetime automatically. | +| Teardown | In a finally path after success, failure, timeout or cancellation, delete this run's disposable VM and owned disks/resources. Wait for provider operations, re-query the recorded resource IDs and report remaining resources. A stopped instance alone does not satisfy this lane's cleanup contract. | + +For the deadline use Compute Engine's `--termination-time` with +`--instance-termination-action=DELETE`, then read back the effective scheduling +configuration. An absolute deadline avoids extending the allowance after a +restart. Google's deadline can begin termination up to 30 seconds late, so leave +time/credit margin; it is a provider backstop, not exact billing precision. +See [VM runtime limits](https://docs.cloud.google.com/compute/docs/instances/limit-vm-runtime). +This protects against the controller disconnecting; a shell finally block alone +cannot. Do not rely on this backstop for a VM that stops before its deadline: +Compute Engine clears its termination timestamp when stopped. Reconnect and +delete that VM and its disks through the ledger; stopped/unknown state remains +unresolved cleanup. Never remove or extend the deadline automatically. Partial logs may be +lost if the controller cannot reconnect before deletion; report that evidence +gap instead of retaining a billable disk indefinitely. + +Use [IAP forwarding](https://docs.cloud.google.com/iap/docs/using-tcp-forwarding) +for administration and [Windows SSH](https://docs.cloud.google.com/compute/docs/connect/windows-ssh) +for the Windows adapter. Resolve the outbound download path in preflight: if a +temporary external IP is needed, count its cost and restrict administrative +ingress. Do not create a persistent NAT service to hide that dependency. +Keep cloud credentials on the controller using the user's existing login; do +not add cloud secrets or billing permissions to the build workflow. + +Stopping a VM leaves potentially billable resources such as disks and static +addresses; see [Compute Engine stop behavior](https://docs.cloud.google.com/compute/docs/reference/rest/v1/instances/stop). +The cleanup ledger must distinguish deleted, still present and unknown resources. +Restrict deletion to exact resources created for this run, verified by IDs and +ownership; never sweep a project or delete an existing user machine. Reconcile +unfinished local run journals before starting another VM, and block new +provisioning while earlier cleanup is unresolved. Do not automatically reprovision +after eviction, capacity failure or a lost connection. + +Credit coverage is a dispatch prerequisite, not a claim of guaranteed free GCE +usage. [Alerts-only budgets do not cap spending](https://docs.cloud.google.com/billing/docs/how-to/budgets). +If current credit eligibility/balance cannot be verified with sufficient margin, +use owned hardware/free CPU CI and leave CUDA NOT_RUN. Provider-state cleanup +verification and any later billing reconciliation are separate evidence; no +account balance, current VM state or credit expiry was verified by this plan edit. + +### Firebase upload, execution and collection + +Use Flutter integration tests as Android instrumentation or iOS XCTest, as +[Firebase documents](https://firebase.google.com/docs/test-lab/flutter/integration-testing-with-flutter). +Robo crawling is not the correctness harness. The maintained quick-core harness +has now executed through both wrappers, with Android app-file retrieval and iOS +XCTest attachment export verified. The broader feature packs remain planned. + +1. Freeze source/dependency/model/configuration hashes. Build and locally validate + supported options, signed iOS inputs and device-specific install requirements. + Use Release mode for comparable measurements where the integration runner + supports it; validate that path first. Keep Debug-only measurements in a + separate cohort if Release instrumentation is unavailable. +2. Refresh catalog and quota, choose an exact device/OS and one backend profile. + Submit an explicit one-device matrix with a 20-minute execution timeout, no retries + and video disabled by default. Persist a local run ID and submission state; + record matrix ID immediately. An uncertain submission is reconciled against + existing matrices before any retry, to avoid duplicate quota consumption. +3. Run each backend profile in a separate cloud execution/app process. Include + lifecycle reload tests within that profile. A crash must not prevent the + other backend profiles from producing their own results. +4. Download pinned public models inside the test or use a verified supported + fixture-transfer mechanism. No paid custom model bucket. Native Flutter uses + a ten-minute download deadline within the 18-minute integration test and + 20-minute cloud execution limits; CLI preparation retains five minutes. + Report received/expected bytes on deadline expiry, remove partial weights, + and distinguish network preparation failure from inference. +5. Write results incrementally to durable files; capture Android pullable app + artifacts through a verified Test Lab mechanism and iOS XCTest attachments. + Prove retrieval on each platform before relying on it. Keep short sequenced + JSONL summaries in logs as crash fallback, with byte limits and checksums for + any reconstructed fragments. Never parse a truncated line as valid JSON. +6. Collect JUnit, JSON, logs, native crash/tombstone or XCTest diagnostics and + manifest. Reconcile app-completed cases with wrapper/matrix status; a process + crash with zero JUnit cases is still ERROR, not a zero-failure success. +7. Wait for terminal matrix/execution states. On interruption, reconnect via + saved matrix IDs; cancel unnecessary outstanding tests and verify termination. + Export evidence promptly rather than depending on the cloud console forever. + +The Firebase adapter pushes the local verified APK pair using +`gcloud firebase test android run --type=instrumentation --app=... --test=...`, +or the locally built/signed iOS XCTest zip using +`gcloud firebase test ios run --type=xctest --test=...`. Both specify the exact +project/device, `--async`, `--timeout=20m`, `--num-flaky-test-attempts=0`, +`--no-record-video`, a run/attempt label and a unique `--results-dir` per matrix. +Omit `--results-bucket` to retain default Test Lab storage. APKs/XCTest inputs +carry the selected profile/manifest; prove any runtime configuration injection +before depending on it. See the official +[Android CLI](https://docs.cloud.google.com/sdk/gcloud/reference/firebase/test/android/run) +and [iOS CLI](https://docs.cloud.google.com/sdk/gcloud/reference/firebase/test/ios/run). + +A run label or results directory is correlation, not a provider idempotency key. +If the CLI loses its response before the matrix ID is saved, mark submission +UNKNOWN and block replacement submissions until the existing attempt is positively +identified in provider records/console. Do not infer that nothing was submitted +from an empty local journal or an unsuccessful lookup. A later REST adapter can +use a persisted create request ID, but the CLI path must first prove its recovery +behavior; it cannot claim automatic deduplication from labels alone. + +The 20-minute flag limits test execution, not provider queue/setup/cleanup time. +Apply a separate 45-minute controller deadline, request cancellation when it is +exceeded, and continue terminal-state reconciliation on reconnect. Capture the +matrix ID/result URI immediately, poll that same matrix and collect from its +returned location. Use the +[testMatrices cancellation API](https://firebase.google.com/docs/test-lab/reference/testing/rest/v1/projects.testMatrices/cancel) +when aborting; sending cancellation alone is not proof that execution has ended. +Android `--directories-to-pull` must name verified accessible output directories +within the supported roots, accounting for scoped storage. iOS uses verified +XCTest attachments. Retain the earlier incremental log fallback for native crashes. + +Firebase manages device allocation/install/run and device cleanup; we own +submission, cancellation, evidence retrieval and terminal-state checks. Do not +delete the test project or its default result bucket as per-run teardown. End +any separately opened Device Streaming session. Serialize our submissions within +the selected Spark quota or explicitly authorized Blaze budget, account for other project users, and +never upgrade billing or retry automatically when quota is exhausted. No VM is +created by this adapter. + +Keep the default Test Lab result storage for the Spark pilot; do not provision +paid custom Cloud Storage. Store downloaded evidence locally and sanitized small +reports as CI artifacts with bounded retention when that path is implemented. +Do not rely on Flutter per-test Firebase timings or video segmentation: Firebase +documents limitations, so measure durations inside the suite. + +## 10. Logs, metrics and result contract + +Write `manifest.json`, incremental `events.jsonl`, and bounded `diagnostics/` on +device. One host-side exporter validates those records and derives `results.json`, +`junit.xml`, `samples.csv` and `summary.html`; wrappers must not implement separate +aggregation rules. Firebase's own JUnit remains raw provider evidence and is +reconciled with the derived case report. Preserve samples so reports can be +recomputed without rerunning inference. Common provenance belongs in the manifest; +events reference stable case/model/configuration IDs. Reruns append attempts and +preserve previous failures; never export only the best attempt. The quick-core +exporters are implemented. A preparation failure before the suite manifest is +currently retained as raw evidence and an incomplete run; promoting it into a +structured preparation-error envelope is still a follow-up. It cannot establish +runtime provenance, passing cases or TPS. + +The manifest lists every mandatory expanded row and a stable ID; each has one +terminal result per attempt. Detect missing, duplicate or truncated records. +Missing child rows after a process crash become NOT_RUN with the parent ERROR; +missing rows in an apparently successful run make the report incomplete. The +summary shows selected/attempted/passed counts and missing mandatory IDs. It can +be green only when every selected obligation passed (including explicitly +expected unsupported-operation guards); omitted feature packs remain uncovered. + +Required identity fields: source commit/dirty patch hash, package/Flutter/Dart +versions, native and Web release tags plus artifact checksums, build mode/ABI, +model/companion hashes, case/profile versions, effective config hash, requested +and resolved backend, CPU features, GPU/driver, device ID/OS build, RAM and page +size where measurable, browser/runtime capabilities, provider and cloud IDs. + +Per-case evidence includes monotonic start/end and phase timings; exact synthetic +messages; sanitized rendered prompt/template where observable; generated content, +thinking and tool deltas; token counts with provenance; finish/cancel reason; +expected predicate; actual result; exception type/code/stack; backend initialization +and fallback evidence; output hash; warm-up/sample index; resource availability. +Capture native INFO diagnostics around load/failure, and bounded logs for normal +runs. During timing, buffer bounded text/counters in memory, then serialize after +the stopwatch stops. Keep only small start/phase/crash breadcrumbs on the hot +path; record native logging level and flush overhead so logging changes cannot +masquerade as a speedup. Unknown values are `null` with a reason, never invented +zeroes. Redact credentials, signed URLs, account/device serials and private paths. + +| Metric | Definition / interpretation | +| --- | --- | +| Model preparation ms | Download and checksum separately; excluded from inference TPS | +| Public load ms | `loadModel` entry to return; lazy native initialization may remain | +| Cold first-response ms | From entry to the first `loadModel` on a new engine/process to first visible output of its first request; includes load and lazy initialization, excludes prior download/hash time. Record OS file-cache state as unknown unless controlled. | +| TTFA ms | Generation call to first nonempty public content delta; separate first-any-event/thinking times | +| Native TTFT ms | Runtime first-token timing only if exposed; do not relabel TTFA as TTFT | +| End-to-end output TPS | Authoritative generated visible-output token count / stream wall seconds, when available; include first-output latency | +| Estimated wall TPS | Retokenized visible output / stream wall seconds if authoritative count unavailable; label estimated, use null without tokenizer | +| Native prefill/decode TPS | Native token counters / respective native phase seconds; record whether counters are per request or differenced | +| Post-first-token TPS | `(tokens - 1) / (last-token time - first-token time)` only with actual token timestamps; batched Dart chunks are insufficient | +| Memory | Process RSS/peak and GPU allocation where available; provider OS profiler vs in-process sample distinguished | +| Reliability | Cases attempted/completed, crashes/timeouts, backend fallbacks, correctness failures, coverage NOT_RUN counts | +| ASR/TTS | ASR WER plus real-time factor = elapsed seconds from first audio feed to final transcript / input audio seconds. TTS real-time factor = elapsed seconds from synthesis request to final PCM sample / output audio seconds; excludes playback. Record streaming feed pacing; paced live ASR is a different cohort from unpaced file ASR. | + +One warm-up plus **three measured repetitions** per benchmark profile; report +median/min/max and individual dots. No p95 from three observations. End-of-sequence +may produce fewer than 32 tokens: record actual count and completion cause, never +divide by the requested maximum. Thinking/tool tokens and visible tokens are +different populations and must be named explicitly. + +Continue independent timing cases after semantic assertion failures if the +process is healthy. Retain correctness FAIL and tag their timings +`correctness_failed`; exclude them from passing performance baselines by default. +Crashes/timeouts have missing timings, not zero TPS. Measurement failure does not +erase correctness evidence already written. + +Statuses: `PASS`, `FAIL` (an assertion failed), `ERROR` (crash, timeout, harness, +download or infrastructure failure, with a precise reason), `UNSUPPORTED` +(expected contract limitation), `NOT_RUN` (quota, unavailable hardware, missing +fixture or deliberately out of selected scope). A tested unsupported-operation +guard itself can PASS while the positive feature row remains UNSUPPORTED. +Known issues remain failed/error rows linked to their issue; no green XFAIL mask. + +Illustrative record shape, using the observed pilot arithmetic failure; timing +values are deliberately absent: + +```json +{ + "schema_version": 1, + "case_id": "C04.arithmetic", + "device_id": "SC-51E", + "runtime": "litert_lm", + "requested_backend": "cpu", + "model_id": "chat-litert", + "expected": {"trimmed_regex": "^4[.!]?$"}, + "actual": {"content": "2"}, + "status": "FAIL", + "reason": "semantic_oracle_mismatch", + "metrics": {"decode_tps": null, "wall_tps": null}, + "metrics_unavailable_reason": "pilot_assertion_stopped_before_benchmark", + "issue": "https://github.com/leehack/llamadart/issues/509" +} +``` + +## 11. Report and graph design + +The first report is a portable offline HTML summary with embedded validated data, +status tables, per-sample TPS/latency plots and artifact links. Generate CSV/JSON +from the same records. Add interactive filters, historical trends and quota +visuals after the exporter and physical-device evidence are reliable. No hosted +database, paid dashboard, or GCP service is needed. The views below describe the +complete report design, not six blockers to the first working harness. + +| View | Visual / interaction | Reading rule | +| --- | --- | --- | +| Coverage | Device × runtime/backend heatmap; model/feature filters; labels and icons in addition to colors | PASS green, FAIL red, ERROR orange, UNSUPPORTED patterned gray, NOT_RUN empty gray; click opens exact evidence | +| Throughput | Separate small panels for native decode TPS and end-to-end/estimated wall TPS; three dots with median and min/max whiskers | Same model/config/build cohort only; show sample count and failed-output badge; never combine the two TPS definitions | +| Latency | Download, load, cold first response, warm TTFA and total stream duration, separately labelled | No stacking overlapping timings; missing values shown as unavailable | +| Trend | Per-device/model/backend median versus commit, with sample ranges | Compare identical OS/driver/runtime configuration or start a new cohort; annotate pin changes | +| Failures | Exact expectation and output, phase, exception/crash summary, issue link, native/backend evidence | Expose known failures and missing evidence before aggregate pass percentage | +| Cost/coverage | Execution count used/planned and remaining known quota, deferred rows | Show incomplete coverage plainly; no estimated spend presented as billed cost | + +The pilot's GGUF medians illustrate why separate charts are necessary: S24 CPU +estimated wall TPS 396.7 vs native decode 475.0; iPhone CPU 674.1 vs 747.5; +iPhone Metal 443.4 vs 5264.0. These are tiny-model diagnostic observations, not +a device ranking. Native compute timing excludes work included in the public +stream. Build modes also differ. Do not put these values in a comparable-platform +leaderboard or infer LiteRT TPS from them. + +Performance is initially informational. Establish repeatable per-device/model +Firebase cohorts, recording OS, driver, memory and available thermal information, +before setting thresholds; matching catalog IDs alone do not ensure equal device +conditions. A candidate alert is a >20% median slowdown with matching provenance, +but three lab samples alone cannot prove a regression: confirm in another +quota-approved run, or optionally locally. Correctness/crashes are +blocking independently of speed. No throughput threshold hides a known failure. + +## 12. Implementation sequence and completion criteria + +1. **Implementation authorized:** use current merged pins; reconcile any later + release changes before qualification. + lock reference-qualified core fixtures and resolve model-dependent oracles. + Reuse the three pilot issues to investigate failures separately from harness work. +2. Implement the smallest shared suite, desktop runner and mobile wrapper, plus + immutable manifests and incremental JSON. Integrate existing matrix/E2E + discovery. Validate the shared suite on the Mac and test provider failure paths + locally before consuming cloud quota; no personal phone or tablet is required. +3. Add separate Android/iOS backend profiles, local preflight, artifact retrieval + and terminal-state reconciliation. Test the Firebase adapter with fake + provider responses for uncertain submission, duplicate invocation, quota + exhaustion, cancellation and missing evidence before any cloud use. Use the + first approved cloud runs to prove crash recovery and result export, not just + a happy-path screenshot. +4. Produce offline reports and CI build artifacts; validate Web asset staging and + one tiny WASM model using the maintained Web E2E path. Keep sign/upload/run + actions separate from automatic artifact building. +5. Implement the GCE adapter and exercise its failure paths with fake + provider responses before VM use: uncertain creation, interrupted upload, + duplicate run invocation, timeout/cancellation, missing evidence, permission + failure during cleanup, a prematurely stopped VM and a mismatched resource + owner. Verify that unresolved cleanup blocks another VM. A separately + authorized short GCE run must prove upload, execution, result retrieval and + resource deletion before broad CUDA coverage; unavailable credit defers this + optional lane without blocking the Mac/CI/Firebase harness. +6. **Next mobile milestone:** preserve the demonstrated Android/iOS submission, + result retrieval and S24 NPU execution. The system-content boundary is now + covered by the suite branch correction, and the public replay matches canonical native `Cedar17` + rather than repeated tokens. Keep its strict capitalization failure and the + native combined-prompt failure explicit. Repeated Mac CPU public/native + controls reproduce both failure patterns. The original Gemma reference now + passes all four variants in three repetitions; the CPU tokenizer bytes and + rendered prompt/token IDs match after accounting for BOS. Separate converted + weights/quantization from LiteRT runtime execution next, alongside Qwen + arithmetic and GPU work. Keep those findings separate from NPU attribution. + Qualify the implemented Unicode generation case, compatible S24 Gemma CPU + and paired-report controls. Gated CPU + weights require verified private model transfer before a Firebase run; never + package an access token or substitute a signed URL in its model lock. + Qualify S24 then Pixel 10 NPU only after model/library + preflight and each native reference succeed. Complete the selected Firebase + rotation including targeted iPad checks, then older-device CPU full/compact + and feature packs as quota permits. + Fill every supported-platform row with exact + PASS/FAIL/ERROR/UNSUPPORTED/NOT_RUN evidence; never equate selected rotation + completion with whole-release qualification. + +The initial harness is usable when its selected bundle runs without the repository, +checks dependencies/models, exercises quick-core public APIs, records proven +backend use, survives independent assertion failures, retains useful crash +evidence, exports valid JSON/JUnit/CSV/basic HTML, and terminates bounded cloud +work. Subsequent milestones expand model/features/platforms; the initial milestone +does not qualify unrun rows. Known product failures remain visible. Full release +readiness still requires the repository's +existing review, platform matrix, affected-family and release gates. + +The [current readiness table](cross_platform_validation.md#current-readiness) +records the implementation boundary. Report validation now derives required +cases, expanded configuration and accelerator obligations from the profile; +the journal cannot declare its own exemptions. The next suite acceptance step is +exact-head CI build and portable execution evidence, followed by the missing +critical feature packs. Original-model diagnostics remain private local evidence, +not an extra heavyweight dependency in the default core or CI. + +## 13. Remaining work checklist (updated 2026-09-19) + +GitHub tracker: [#514](https://github.com/leehack/llamadart/issues/514). + +This is the remaining scope from the full plan, not a claim that all rows belong +in the first PR. The initial PR delivers the quick core, reports, portable build +and cloud adapters, NPU diagnostics and the discovered system-message correction. +Its CI and independent review must finish before merge readiness. Device/model +failures remain visible and are investigated separately from harness completion. + +| ID | Remaining work | Completion evidence | +| --- | --- | --- | +| R01 | Qualify the PR and bundle workflow on Linux x64, Windows x64 and macOS; build Android APK/test APK, Web and iOS inputs | Exact-head CI green, extracted bundles executable outside a checkout, checksums/manifests retained; independent high-risk review before ready. iOS physical signing remains on the Mac. | +| R02 | Complete C05 thinking/budgets, C07 tool auto/required/none and continuation, C10 stop-marker semantics, C12 guard/recovery subcases, C02 separate Unicode generation and C11 tool-bearing parity | C09's second cycle and C11 native text/thinking parity plus LiteRT Web option rejection are implemented. Catalog 4 now implements Unicode generation, thinking on/off, tool choice/result continuation, stop markers and unloaded-engine guards. Historical catalogs retain their original NOT_RUN records; current exact-model qualification is separate. C11 tool-bearing fixtures, NPU deterministic sampling and GGUF Web worker controls remain unqualified; do not substitute text-only evidence. | +| R03 | Core focused selection and versioned metadata implemented; extend the catalog as future model/media packs land | Schema 2 binds case/feature versions, resolved prompts/tools/predicates and fixture hashes; omitted cases have explicit reasons. `tiny-gguf-lifecycle` and `tiny-gguf-batching` are runnable focused examples. Catalog version 4 imports version-1/2/3 reports against their original definitions. Future pack media hashes and reference qualifications remain with R04/R05. | +| R04 | Add the eleven targeted packs in section 6 | Structured output; state/prompt reuse; embeddings; vision; audio understanding; ASR; TTS; LoRA; speculative decoding; runtime controls; app/device/browser lifecycle. Reuse existing registered tests and keep large models opt-in. | +| R05 | Lock and reference-qualify pack models/media | Primary Qwen3.5 and Gemma 4 GGUF/native/Web bundles; dedicated Qwen3-ASR, Moonshine and Qwen3-TTS exceptions; additional embedding/adaptor/draft fixtures only for their targeted packs; exact revisions/hashes/access and memory limits. Current quick/NPU model locks do not qualify these candidates. | +| R06 | Finish Firebase core device rotation | Isolated GGUF CPU/GPU and LiteRT CPU/GPU on S24, Tab P12, iPhone 16 Pro and SE 3; targeted iPad 10 GPU runs. Existing pilots are partial evidence, not a completed rotation. A05s full/compact, iPhone 8 and Pixel 5 remain later compatibility rows. | +| R07 | Complete S24 NPU qualification, then Tensor G5 | Resolve #513; add N02 Unicode generation/native tokenizer control, compatible S24 CPU Gemma with verified private model transfer, and paired native/public comparison. Require coherent N03 outputs and N04 lifecycle evidence; retain hybrid/unknown placement limits. Pixel 10 needs installed-app vendor-kit/SoC/probe and native/public/CPU runs; a compiled dispatch library is not execution proof. | +| R08 | Fill remaining platform/packaging rows | Android arm64 virtual 4K/16K and separate backcompat, Android x64 emulator, Apple simulators, Linux arm64, Windows arm64, macOS x64 as available; full/compact and lower-ISA physical coverage. Record unavailable hardware explicitly. | +| R09 | Qualify browser and GPU evidence paths | LiteRT GPU adapters with actual driver/delegate proof; Chrome WASM/WebGPU, Safari and Firefox capability rows; a genuine LiteRT Web model bundle and negative native-only contracts. Native Firebase XCTest does not qualify iPadOS Safari. | +| R10 | Exercise the GCE lifecycle and desktop CUDA runs | Historical Linux/Windows bootstrap runs executed GGUF CPU/CUDA/Vulkan and verified cleanup. Still qualify the maintained GCE adapter and reproducible image end to end; bootstrap evidence does not establish that contract or current-head model qualification. Recheck current credit before provisioning. LiteRT desktop GPU is Vulkan/D3D12, not CUDA. No available credit means NOT_RUN, not personal charges. | +| R11 | Complete the required evidence envelope and missing measurements | Model hash/size evidence, portable desktop payload verification, override rejection and collected identity binding are implemented. Preparation stage/byte progress before a model manifest is implemented. Remaining: explicit provenance/availability for artifact and companion hashes, device memory/page size, cold first response, prefill/native TTFT and first-thinking timing where observable. Keep unsupported counters null; local speech reports already record WER/real-time factor, but portable/mobile speech import and quality qualification remain open. | +| R12 | Add aggregate and historical reporting after core evidence is stable | Paired run comparison, device/backend coverage heatmap, comparable-cohort filters, trend and quota views. Existing per-run JSON/JUnit/CSV/HTML and three-sample TPS are usable; a dashboard or performance threshold is not required for the first PR. | + +Prioritize R01, then bounded R02/R03 work. R06/R07 use only freshly verified free +Firebase allowance or covered credit; never dispatch the entire rotation at once. +R10 is optional while credit is unavailable. R04/R05 are change-focused feature +coverage, not every-model-by-every-device permutations. R12 visual polish comes +after the mandatory evidence, not before correctness. + + +The [current Qwen reference diagnosis](cross_platform_validation.md#current-qwen-tool-and-history-reference-2026-09-19) +separates implemented infrastructure from model conformance: C07 passes after +merged runtime fixes, while the original C06 exact-case failure also reproduces +in matched upstream execution. Keep that failure and all unrun device rows +visible; reference agreement does not waive an expected-output predicate. diff --git a/doc/testing_matrix.md b/doc/testing_matrix.md index 144adbbc9..6da902b9c 100644 --- a/doc/testing_matrix.md +++ b/doc/testing_matrix.md @@ -4,6 +4,13 @@ This repository uses a layered test matrix so contributors can validate the essential runtime, model, feature, and platform paths without forcing every pull request to run large local models or device-only checks. +The proposed [cross-platform validation plan](cross_platform_validation_plan.md) +details reusable test apps, model/backend cases, metrics, a free Firebase +physical-device rotation, and VM/Firebase upload, execution and cleanup. It is +planning work; implementation is deferred until +the current release additions are complete, and its proposed rows are not yet +implemented by the runners below. + Use the matrix for every non-trivial PR: ```bash @@ -407,6 +414,10 @@ When an agent creates or updates a PR: rewinds. Consult `doc/pr_branch_writer_inventory.md` for writer scope and the remaining GitHub-managed governance boundary. +The executable quick suite and provider commands are documented in the +[cross-platform validation runbook](cross_platform_validation.md). Discover the +model-free `validation-harness` and opt-in `validation-model-core` matrix rows. + ### LiteRT-LM lifecycle regression diff --git a/example/chat_app/android/app/build.gradle.kts b/example/chat_app/android/app/build.gradle.kts index 5d1de09c6..4de8d3cfb 100644 --- a/example/chat_app/android/app/build.gradle.kts +++ b/example/chat_app/android/app/build.gradle.kts @@ -9,6 +9,20 @@ android { namespace = "com.example.llamadart_chat_example" compileSdk = flutter.compileSdkVersion ndkVersion = "28.2.13676358" + // The maintained bundle builder provides an isolated, verified staging tree. + val npuStage = System.getenv("LLAMADART_VALIDATION_NPU_STAGE") + if (npuStage != null) { + sourceSets.getByName("main") { + assets.srcDir("$npuStage/assets") + jniLibs.srcDir("$npuStage/jniLibs") + manifest.srcFile("$npuStage/AndroidManifest.xml") + } + packaging.jniLibs.useLegacyPackaging = true + packaging.jniLibs.keepDebugSymbols += setOf( + "**/libLiteRtDispatch_*.so", "**/libLlamadartVendor_*.so", "**/libQnn*.so", + ) + androidResources.noCompress += "litertlm" + } compileOptions { sourceCompatibility = JavaVersion.VERSION_17 @@ -44,6 +58,7 @@ flutter { source = "../.." } +// Match the integration_test plugin bundled with the pinned Flutter SDK. dependencies { androidTestImplementation("androidx.test:runner:1.3.0") androidTestImplementation("androidx.test:rules:1.2.0") diff --git a/example/chat_app/android/app/src/main/kotlin/com/example/llamadart_chat_example/MainActivity.kt b/example/chat_app/android/app/src/main/kotlin/com/example/llamadart_chat_example/MainActivity.kt index 52f015bd5..71b2bda6e 100644 --- a/example/chat_app/android/app/src/main/kotlin/com/example/llamadart_chat_example/MainActivity.kt +++ b/example/chat_app/android/app/src/main/kotlin/com/example/llamadart_chat_example/MainActivity.kt @@ -12,9 +12,11 @@ import java.util.concurrent.Executors class MainActivity : FlutterActivity() { private val clipboardExecutor = Executors.newSingleThreadExecutor() private var clipboardChannel: MethodChannel? = null + private var validationNpuHost: ValidationNpuHost? = null override fun configureFlutterEngine(flutterEngine: FlutterEngine) { super.configureFlutterEngine(flutterEngine) + validationNpuHost = ValidationNpuHost(this, flutterEngine) val channel = MethodChannel( flutterEngine.dartExecutor.binaryMessenger, CLIPBOARD_CHANNEL, @@ -53,6 +55,8 @@ class MainActivity : FlutterActivity() { } override fun onDestroy() { + validationNpuHost?.close() + validationNpuHost = null clipboardChannel?.setMethodCallHandler(null) clipboardChannel = null clipboardExecutor.shutdownNow() diff --git a/example/chat_app/android/app/src/main/kotlin/com/example/llamadart_chat_example/ValidationNpuHost.kt b/example/chat_app/android/app/src/main/kotlin/com/example/llamadart_chat_example/ValidationNpuHost.kt new file mode 100644 index 000000000..9ddc4d36c --- /dev/null +++ b/example/chat_app/android/app/src/main/kotlin/com/example/llamadart_chat_example/ValidationNpuHost.kt @@ -0,0 +1,130 @@ +package com.example.llamadart_chat_example + +import android.app.Activity +import android.os.Build +import android.system.Os +import io.flutter.embedding.engine.FlutterEngine +import io.flutter.plugin.common.MethodChannel +import org.json.JSONObject +import java.io.File +import java.security.MessageDigest +import java.util.concurrent.Executors + +/** Private validation entry point; normal app builds contain no NPU assets. */ +class ValidationNpuHost(private val activity: Activity, engine: FlutterEngine) { + private val executor = Executors.newSingleThreadExecutor() + private val channel = MethodChannel(engine.dartExecutor.binaryMessenger, "llamadart_validation/npu") + @Volatile private var cancelled = false + + init { + channel.setMethodCallHandler { call, result -> + if (call.method == "cancel") { + cancelled = true + result.success(null) + } else if (call.method == "prepare") { + cancelled = false + executor.execute { + try { + val value = prepare(call.argument("profile") ?: "") + activity.runOnUiThread { result.success(value) } + } catch (error: Exception) { + activity.runOnUiThread { + result.error("npu_preparation_failed", error.message, null) + } + } + } + } else result.notImplemented() + } + } + + fun close() { + cancelled = true + channel.setMethodCallHandler(null) + executor.shutdownNow() + } + + private fun assetJson(name: String): JSONObject = + activity.assets.open("llamadart_npu/$name").bufferedReader().use { JSONObject(it.readText()) } + + private fun hash(file: File): String { + val digest = MessageDigest.getInstance("SHA-256") + file.inputStream().use { input -> + val buffer = ByteArray(1024 * 1024) + while (true) { + check(!cancelled) { "NPU preparation cancelled" } + val count = input.read(buffer) + if (count < 0) break + digest.update(buffer, 0, count) + } + } + return digest.digest().joinToString("") { "%02x".format(it) } + } + + private fun prepare(profileId: String): Map { + check(Build.VERSION.SDK_INT >= 31) { "NPU requires Android API 31+" } + val profile = assetJson("profile.json") + check(profile.getString("id") == profileId) { "NPU compiled profile mismatch" } + val target = profile.getJSONObject("npu_target") + val soc = Build.SOC_MODEL + val aliases = target.getJSONArray("device_soc_models") + check((0 until aliases.length()).any { aliases.getString(it).equals(soc, true) }) { + "NPU SoC mismatch: observed $soc, expected ${target.getString("soc")}" } + check(Build.SUPPORTED_ABIS.contains("arm64-v8a")) { "NPU requires arm64-v8a" } + val directory = activity.applicationInfo.nativeLibraryDir + val kit = assetJson("npu-kit.json") + val libraries = kit.getJSONObject("libraries") + val required = target.getJSONObject("libraries") + check(libraries.length() == required.length()) { "NPU library inventory mismatch" } + for (name in required.keys()) { + check(name.matches(Regex("lib[A-Za-z0-9_]+\\.so"))) { "Invalid NPU library name" } + val file = File(directory, name) + val entry = libraries.getJSONObject(name) + check(file.isFile && file.length() == entry.getLong("bytes") && + hash(file) == entry.getString("sha256")) { "NPU library integrity failed: $name" } + val lock = required.getJSONObject(name) + if (lock.has("sha256")) check(entry.getString("sha256") == lock.getString("sha256")) { + "NPU audited library hash mismatch: $name" } + } + // Scope DSP lookup to the installed app first, retaining normal device + // firmware locations. This changes no device files or root permissions. + Os.setenv("ADSP_LIBRARY_PATH", "$directory;/vendor/lib/rfsa/adsp;/vendor/dsp", true) + val model = profile.getJSONObject("model") + val expected = model.getString("sha256") + val cache = File(activity.filesDir, "validation-npu/$expected") + cache.mkdirs() + val destination = File(cache, "model.litertlm") + val start = System.nanoTime() + if (!destination.isFile || destination.length() != model.getLong("bytes") || hash(destination) != expected) { + val temporary = File(cache, "model.partial") + try { + activity.assets.open("llamadart_npu/model.litertlm").use { input -> + temporary.outputStream().use { output -> + val buffer = ByteArray(1024 * 1024) + var total = 0L + while (true) { + check(!cancelled) { "NPU preparation cancelled" } + val count = input.read(buffer) + if (count < 0) break + total += count + check(total <= model.getLong("bytes")) { "NPU model exceeds locked size" } + output.write(buffer, 0, count) + } + } + } + check(temporary.length() == model.getLong("bytes") && hash(temporary) == expected) { + "NPU model integrity failed" } + check(temporary.renameTo(destination)) { "Cannot finalize staged NPU model" } + } finally { temporary.delete() } + } + check(!cancelled) { "NPU preparation cancelled" } + return mapOf( + "path" to destination.path, "dispatch_directory" to directory, + "soc_model" to soc, "soc_manufacturer" to Build.SOC_MANUFACTURER, + "device_model" to Build.MODEL, "android_api" to Build.VERSION.SDK_INT, + "abi" to "arm64-v8a", "verified" to true, + "sha256" to expected, "bytes" to destination.length(), + "total_ms" to (System.nanoTime() - start) / 1e6, + "kit" to kit.toString() + ) + } +} diff --git a/example/chat_app/android/gradle.properties b/example/chat_app/android/gradle.properties index d5da7278a..01636b9cf 100644 --- a/example/chat_app/android/gradle.properties +++ b/example/chat_app/android/gradle.properties @@ -1,6 +1,5 @@ org.gradle.jvmargs=-Xmx8G -XX:MaxMetaspaceSize=4G -XX:ReservedCodeCacheSize=512m -XX:+HeapDumpOnOutOfMemoryError android.useAndroidX=true -# This builtInKotlin flag was added automatically by Flutter migrator +# Keep the existing Kotlin/Gradle DSL while using Flutter 3.47.1. android.builtInKotlin=false -# This newDsl flag was added automatically by Flutter migrator android.newDsl=false diff --git a/example/chat_app/integration_test/validation_test.dart b/example/chat_app/integration_test/validation_test.dart new file mode 100644 index 000000000..739ac6215 --- /dev/null +++ b/example/chat_app/integration_test/validation_test.dart @@ -0,0 +1,32 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; + +import 'package:llamadart_chat_example/validation/controller.dart'; + +void main() { + final binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + const profile = String.fromEnvironment( + 'VALIDATION_PROFILE', + defaultValue: 'tiny-gguf-cpu', + ); + testWidgets('shared validation: $profile', (tester) async { + await tester.pumpWidget( + const MaterialApp( + home: Scaffold(body: Text('Running llamadart validation')), + ), + ); + final controller = ValidationController(); + addTearDown(controller.dispose); + await controller.run(profile); + binding.reportData = + controller.report?.toJson() ?? {'error': controller.error}; + expect(controller.error, isNull); + expect( + controller.report?.assertionsPassed, + isTrue, + reason: + 'Functional obligations failed; accelerator qualification remains separate in the report.', + ); + }, timeout: const Timeout(Duration(minutes: 18))); +} diff --git a/example/chat_app/ios/Runner.xcodeproj/project.pbxproj b/example/chat_app/ios/Runner.xcodeproj/project.pbxproj index bfa87d3ce..3d20b9f3c 100644 --- a/example/chat_app/ios/Runner.xcodeproj/project.pbxproj +++ b/example/chat_app/ios/Runner.xcodeproj/project.pbxproj @@ -8,7 +8,7 @@ /* Begin PBXBuildFile section */ 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */ = {isa = PBXBuildFile; fileRef = 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */; }; - 331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 331C807B294A618700263BE5 /* RunnerTests.swift */; }; + 331C808B294A63AB00263BE5 /* RunnerTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 331C807B294A618700263BE5 /* RunnerTests.m */; }; 3B3967161E833CAA004F5970 /* AppFrameworkInfo.plist in Resources */ = {isa = PBXBuildFile; fileRef = 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */; }; 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 74858FAE1ED2DC5600515810 /* AppDelegate.swift */; }; 78A318202AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage in Frameworks */ = {isa = PBXBuildFile; productRef = 78A3181F2AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage */; }; @@ -43,7 +43,7 @@ /* Begin PBXFileReference section */ 1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GeneratedPluginRegistrant.h; sourceTree = ""; }; 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GeneratedPluginRegistrant.m; sourceTree = ""; }; - 331C807B294A618700263BE5 /* RunnerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RunnerTests.swift; sourceTree = ""; }; + 331C807B294A618700263BE5 /* RunnerTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = RunnerTests.m; sourceTree = ""; }; 331C8081294A63A400263BE5 /* RunnerTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = RunnerTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = AppFrameworkInfo.plist; path = Flutter/AppFrameworkInfo.plist; sourceTree = ""; }; 74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "Runner-Bridging-Header.h"; sourceTree = ""; }; @@ -81,7 +81,7 @@ 331C8082294A63A400263BE5 /* RunnerTests */ = { isa = PBXGroup; children = ( - 331C807B294A618700263BE5 /* RunnerTests.swift */, + 331C807B294A618700263BE5 /* RunnerTests.m */, ); path = RunnerTests; sourceTree = ""; @@ -278,7 +278,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - 331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */, + 331C808B294A63AB00263BE5 /* RunnerTests.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/example/chat_app/ios/RunnerTests/RunnerTests.m b/example/chat_app/ios/RunnerTests/RunnerTests.m new file mode 100644 index 000000000..6751b7516 --- /dev/null +++ b/example/chat_app/ios/RunnerTests/RunnerTests.m @@ -0,0 +1,35 @@ +@import XCTest; +@import integration_test; + +INTEGRATION_TEST_IOS_RUNNER(RunnerTests) + +// Flutter executes the Dart suite before materializing its XCTest results. +// Keep the latest run's bounded evidence even when its assertions fail. +@implementation RunnerTests (ValidationAttachments) +- (void)tearDown { + NSFileManager *files = NSFileManager.defaultManager; + NSURL *documents = [files URLsForDirectory:NSDocumentDirectory + inDomains:NSUserDomainMask].firstObject; + NSURL *runs = [documents URLByAppendingPathComponent:@"validation/runs" isDirectory:YES]; + NSArray *directories = [files contentsOfDirectoryAtURL:runs + includingPropertiesForKeys:nil options:NSDirectoryEnumerationSkipsHiddenFiles error:nil]; + NSURL *latest = [[directories sortedArrayUsingComparator:^NSComparisonResult(NSURL *a, NSURL *b) { + return [a.lastPathComponent compare:b.lastPathComponent]; + }] lastObject]; + if (latest != nil) { + for (NSURL *url in [files contentsOfDirectoryAtURL:latest + includingPropertiesForKeys:@[NSURLFileSizeKey] options:NSDirectoryEnumerationSkipsHiddenFiles error:nil]) { + NSNumber *size = nil; + [url getResourceValue:&size forKey:NSURLFileSizeKey error:nil]; + if (size != nil && size.unsignedLongLongValue <= 2 * 1024 * 1024 && + [@[@"json", @"jsonl", @"csv", @"xml", @"html"] containsObject:url.pathExtension]) { + XCTAttachment *attachment = [XCTAttachment attachmentWithContentsOfFileAtURL:url]; + attachment.name = url.lastPathComponent; + attachment.lifetime = XCTAttachmentLifetimeKeepAlways; + [self addAttachment:attachment]; + } + } + } + [super tearDown]; +} +@end diff --git a/example/chat_app/ios/RunnerTests/RunnerTests.swift b/example/chat_app/ios/RunnerTests/RunnerTests.swift deleted file mode 100644 index 86a7c3b1b..000000000 --- a/example/chat_app/ios/RunnerTests/RunnerTests.swift +++ /dev/null @@ -1,12 +0,0 @@ -import Flutter -import UIKit -import XCTest - -class RunnerTests: XCTestCase { - - func testExample() { - // If you add code to the Runner application, consider adding tests here. - // See https://developer.apple.com/documentation/xctest for more information about using XCTest. - } - -} diff --git a/example/chat_app/lib/validation/controller.dart b/example/chat_app/lib/validation/controller.dart new file mode 100644 index 000000000..96d0151d8 --- /dev/null +++ b/example/chat_app/lib/validation/controller.dart @@ -0,0 +1,159 @@ +import 'dart:convert'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; + +import 'host.dart'; + +/// Coordinates the same suite for interactive and unattended Flutter runs. +class ValidationController extends ChangeNotifier { + /// Allows host injection in widget tests. + ValidationController({ValidationHost? host}) + : host = host ?? createValidationHost(); + + /// Native/browser storage adapter. + final ValidationHost host; + + /// Bounded current run progress. + final List> records = []; + + /// Whether a run is in progress. + bool running = false; + + /// Current case or preparation phase. + String phase = 'Ready'; + + /// Complete or partial report. + ValidationReport? report; + + /// Preparation/adapter failure description. + String? error; + ValidationRunner? _runner; + bool _cancelled = false; + bool _disposed = false; + void _notify() { + if (!_disposed) notifyListeners(); + } + + @override + void dispose() { + _disposed = true; + cancel(); + super.dispose(); + } + + /// Cancels inference and prevents the next case from starting. + void cancel() { + _cancelled = true; + host.cancelPreparation(); + _runner?.cancel(); + } + + /// Loads a checked-in profile and executes it once. + Future run(String profileId) async { + if (running) throw StateError('A validation run is already active'); + if (!RegExp(r'^[a-z][a-z0-9-]{0,63}$').hasMatch(profileId)) { + throw const FormatException('Invalid profile id'); + } + _cancelled = false; + running = true; + phase = 'Preparing model'; + error = null; + report = null; + records.clear(); + _notify(); + var started = false; + try { + final source = await rootBundle.loadString( + 'packages/llamadart_validation/assets/profiles/$profileId.json', + ); + final data = jsonDecode(source) as Map; + data['execution_path'] = const String.fromEnvironment( + 'VALIDATION_EXECUTION_PATH', + defaultValue: 'public_api', + ); + final profile = ValidationProfile.fromJson(data); + if (_cancelled) { + throw StateError('Run cancelled before model preparation'); + } + final runId = 'app-${DateTime.now().toUtc().microsecondsSinceEpoch}'; + await host.start(runId); + started = true; + if (_cancelled) { + throw StateError('Run cancelled before model preparation'); + } + final model = await host.prepare(profile); + if (_cancelled) throw StateError('Run cancelled during preparation'); + _runner = ValidationRunner( + profile: profile, + engine: host.createEngine(profile), + emit: (event) async { + await host.emit(event); + if (event['type'] == 'case' || event['type'] == 'case_start') { + phase = '${event['case_id']} ${event['status'] ?? 'running'}'; + if (event['type'] == 'case') records.add(event); + _notify(); + } + }, + ); + await _runner!.run( + model.path, + runId: runId, + preparation: model.evidence, + environment: { + 'platform': defaultTargetPlatform.name, + 'web': kIsWeb, + 'build_mode': kReleaseMode + ? 'release' + : kProfileMode + ? 'profile' + : 'debug', + 'source_commit': const String.fromEnvironment( + 'VALIDATION_COMMIT', + defaultValue: 'unknown', + ), + 'source_dirty': const bool.fromEnvironment( + 'VALIDATION_SOURCE_DIRTY', + defaultValue: true, + ), + 'hook_sha256': const String.fromEnvironment( + 'VALIDATION_HOOK_SHA256', + defaultValue: 'unknown', + ), + 'bridge_tag': const String.fromEnvironment( + 'VALIDATION_BRIDGE_TAG', + defaultValue: 'unknown', + ), + 'native_tag': const String.fromEnvironment( + 'VALIDATION_NATIVE_TAG', + defaultValue: 'unknown', + ), + 'litert_tag': const String.fromEnvironment( + 'VALIDATION_LITERT_TAG', + defaultValue: 'unknown', + ), + }, + ); + } catch (exception) { + error = redactDiagnostic('$exception'); + if (started) { + await host.emit({'type': 'preparation_error', 'message': error}); + } + } finally { + try { + if (started) report = await host.finish(); + } catch (exception) { + error = redactDiagnostic('Report persistence failed: $exception'); + } + running = false; + _runner = null; + phase = error != null + ? 'Error' + : report?.qualified == true + ? 'Qualified' + : 'Incomplete / failed'; + _notify(); + } + } +} diff --git a/example/chat_app/lib/validation/host.dart b/example/chat_app/lib/validation/host.dart new file mode 100644 index 000000000..664fafa13 --- /dev/null +++ b/example/chat_app/lib/validation/host.dart @@ -0,0 +1,37 @@ +import 'package:llamadart_validation/llamadart_validation.dart'; + +import 'host_native.dart' + if (dart.library.js_interop) 'host_web.dart' + as platform; + +/// Platform storage and model preparation for the shared validation runner. +abstract interface class ValidationHost { + /// Selects the prepared public API or direct native reference adapter. + ValidationEngine createEngine(ValidationProfile profile); + + /// Prepares and verifies the locked model before inference. + Future<({String path, Map evidence})> prepare( + ValidationProfile profile, + ); + + /// Cancels an in-progress download. + void cancelPreparation(); + + /// Opens a unique incremental result journal. + Future start(String runId); + + /// Persists a complete event and crash-recovery breadcrumb. + Future emit(Map event); + + /// Closes the journal and derives reports from it. + Future finish(); + + /// Exports one report to device storage or a browser download. + Future export(String name, String text); + + /// Describes where the host keeps result files. + String get outputLocation; +} + +/// Selects native filesystem or browser storage without importing IO on Web. +ValidationHost createValidationHost() => platform.createHost(); diff --git a/example/chat_app/lib/validation/host_native.dart b/example/chat_app/lib/validation/host_native.dart new file mode 100644 index 000000000..81bb7fe33 --- /dev/null +++ b/example/chat_app/lib/validation/host_native.dart @@ -0,0 +1,140 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:file_picker/file_picker.dart'; +import 'package:flutter/services.dart'; +import 'package:http/http.dart' as http; +import 'package:llamadart_validation/io.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:llamadart_validation/npu_io.dart'; +import 'package:path/path.dart' as p; +import 'package:path_provider/path_provider.dart'; + +import 'host.dart'; + +/// Creates the mobile/desktop Flutter filesystem host. +ValidationHost createHost() => _NativeHost(); + +class _NativeHost implements ValidationHost { + static const _npuChannel = MethodChannel('llamadart_validation/npu'); + AndroidNpuMonitor? _npu; + String? _probeLibrary; + @override + ValidationEngine createEngine(ValidationProfile profile) => + profile.nativeReference + ? NativeNpuReferenceEngine(_npu!, _probeLibrary!) + : PublicValidationEngine(npu: _npu); + FileValidationJournal? _journal; + Directory? _output; + http.Client? _client; + @override + void cancelPreparation() { + _client?.close(); + if (Platform.isAndroid) { + _npuChannel.invokeMethod('cancel').catchError((Object _) {}); + } + } + + @override + String get outputLocation => _output?.path ?? ''; + + @override + Future<({String path, Map evidence})> prepare( + ValidationProfile profile, + ) async { + _npu = null; + _probeLibrary = null; + if (profile.backend == 'npu') { + if (!Platform.isAndroid) { + throw UnsupportedError('NPU app validation requires Android'); + } + final value = Map.from( + (await _npuChannel + .invokeMapMethod('prepare', { + 'profile': profile.id, + }) + .timeout( + const Duration(minutes: 10), + onTimeout: () { + cancelPreparation(); + throw TimeoutException('NPU preparation exceeded ten minutes'); + }, + ))!, + ); + final kit = + jsonDecode(value.remove('kit') as String) as Map; + final directory = value.remove('dispatch_directory') as String; + final path = value.remove('path') as String; + final target = profile.data['npu_target'] as Map; + validateNpuDevice(profile, value); + if (value['verified'] != true || + value['sha256'] != profile.modelHash || + kit['target'] != target['soc'] || + kit['runtime_tag'] != + const String.fromEnvironment('VALIDATION_LITERT_TAG') || + kit['litert_revision'] != + '9fe5be45564c868408e6514c8aabb83e211a0911' || + kit['dispatch_header_sha256'] != + '11dd4d98bd084157ac987b1ee1951f3f96e2b3ca6b51a27c10e645686bf0e3ee') { + throw StateError( + 'Installed NPU host identity does not match the profile/runtime', + ); + } + _probeLibrary = (target['libraries'] as Map).keys + .cast() + .singleWhere((name) => name.startsWith('libLiteRtDispatch_')); + final identity = {'device': value, 'kit': kit}; + _npu = AndroidNpuMonitor(directory, _probeLibrary!, identity); + return (path: path, evidence: {...value, 'npu': identity}); + } + _client = http.Client(); + try { + return await prepareModel( + profile, + Directory( + p.join( + (await getApplicationSupportDirectory()).path, + 'validation', + 'models', + ), + ), + client: _client, + onProgress: _journal!.emitPreparation, + timeout: const Duration(minutes: 10), + ); + } finally { + _client?.close(); + _client = null; + } + } + + @override + Future start(String runId) async { + final base = Platform.isAndroid + ? await getExternalStorageDirectory() + : await getApplicationDocumentsDirectory(); + if (base == null) { + throw StateError('No validation output directory available'); + } + _output = Directory(p.join(base.path, 'validation', 'runs', runId)); + _journal = FileValidationJournal(_output!); + } + + @override + Future emit(Map event) => _journal!.emit(event); + + @override + Future finish() async { + _journal!.close(); + return writeReports(_output!); + } + + @override + Future export(String name, String text) async { + await FilePicker.platform.saveFile( + fileName: name, + bytes: Uint8List.fromList(utf8.encode(text)), + ); + } +} diff --git a/example/chat_app/lib/validation/host_web.dart b/example/chat_app/lib/validation/host_web.dart new file mode 100644 index 000000000..d8cc62e56 --- /dev/null +++ b/example/chat_app/lib/validation/host_web.dart @@ -0,0 +1,107 @@ +import 'dart:convert'; +import 'dart:async'; +import 'dart:typed_data'; +import 'dart:js_interop'; + +import 'package:crypto/crypto.dart'; +import 'package:http/http.dart' as http; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:web/web.dart' as web; + +import 'host.dart'; + +/// Creates the browser host, which exports its in-memory result journal. +ValidationHost createHost() => _WebHost(); + +class _WebHost implements ValidationHost { + @override + ValidationEngine createEngine(ValidationProfile profile) => + PublicValidationEngine(); + final _lines = []; + http.Client? _client; + @override + void cancelPreparation() => _client?.close(); + @override + String get outputLocation => 'Browser memory; export before closing the tab'; + + @override + Future<({String path, Map evidence})> prepare( + ValidationProfile profile, + ) async { + if (profile.runtime == 'litert') { + throw UnsupportedError( + 'Native LiteRT fixtures are not browser bundles. Supply a separately qualified Web profile.', + ); + } + final watch = Stopwatch()..start(); + final client = _client = http.Client(); + final timer = Timer(const Duration(minutes: 5), client.close); + try { + final response = await client.send( + http.Request('GET', Uri.parse(profile.model['url'] as String)), + ); + if (response.statusCode != 200) { + throw StateError('Model download HTTP mismatch'); + } + final bytes = BytesBuilder(copy: false); + await for (final chunk in response.stream) { + if (bytes.length + chunk.length > (profile.model['bytes'] as int)) { + throw StateError('Model size exceeded'); + } + bytes.add(chunk); + } + if (bytes.length != profile.model['bytes'] || + sha256.convert(bytes.takeBytes()).toString() != profile.modelHash) { + throw const FormatException('Model checksum/size mismatch'); + } + } finally { + timer.cancel(); + client.close(); + _client = null; + } + // The immutable URL retains the .gguf routing suffix. The bridge owns its + // URL cache; report that second transfer separately from native file reuse. + return ( + path: profile.model['url'] as String, + evidence: { + 'sha256': profile.modelHash, + 'bytes': profile.model['bytes'], + 'verified': true, + 'total_ms': watch.elapsedMilliseconds, + 'bridge_transfer': 'immutable URL; bridge may transfer bytes again', + }, + ); + } + + @override + Future start(String runId) async { + _lines.clear(); + } + + @override + Future emit(Map event) async { + final line = jsonEncode(event); + _lines.add(line); + if (line.length <= 32768) { + web.console.log('LLAMADART_VALIDATION $line'.toJS); + } + } + + @override + Future finish() async => + ValidationReport.parse(_lines.join('\n')); + + @override + Future export(String name, String text) async { + final blob = web.Blob( + [text.toJS].toJS, + web.BlobPropertyBag(type: 'application/octet-stream'), + ); + final url = web.URL.createObjectURL(blob); + final anchor = web.document.createElement('a') as web.HTMLAnchorElement + ..href = url + ..download = name; + anchor.click(); + Timer(const Duration(seconds: 1), () => web.URL.revokeObjectURL(url)); + } +} diff --git a/example/chat_app/lib/validation_main.dart b/example/chat_app/lib/validation_main.dart new file mode 100644 index 000000000..f460cea09 --- /dev/null +++ b/example/chat_app/lib/validation_main.dart @@ -0,0 +1,165 @@ +import 'dart:convert'; + +import 'package:flutter/material.dart'; + +import 'validation/controller.dart'; + +void main() => runApp(const ValidationApp()); + +/// Small interactive entry point for the shared cross-platform test suite. +class ValidationApp extends StatefulWidget { + /// Creates the QA app independently of the normal chat application. + const ValidationApp({super.key}); + + @override + State createState() => _ValidationAppState(); +} + +class _ValidationAppState extends State { + final _controller = ValidationController(); + static const _compiledProfile = String.fromEnvironment( + 'VALIDATION_PROFILE', + defaultValue: 'tiny-gguf-cpu', + ); + static const _executionPath = String.fromEnvironment( + 'VALIDATION_EXECUTION_PATH', + defaultValue: 'public_api', + ); + String _profile = _compiledProfile; + @override + void dispose() { + _controller.cancel(); + _controller.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) => MaterialApp( + title: 'llamadart validation', + theme: ThemeData( + colorSchemeSeed: const Color(0xff247a87), + useMaterial3: true, + ), + home: AnimatedBuilder( + animation: _controller, + builder: (context, _) => Scaffold( + appBar: AppBar(title: const Text('llamadart validation')), + body: Center( + child: ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 850), + child: ListView( + padding: const EdgeInsets.all(24), + children: [ + const Text( + 'Run the same package checks on this device.', + style: TextStyle(fontSize: 22), + ), + const SizedBox(height: 16), + Text( + _executionPath == 'native_c_api' + ? 'Direct native control · does not qualify the public Dart path' + : 'llamadart public API', + ), + DropdownButtonFormField( + initialValue: _profile, + decoration: const InputDecoration( + labelText: 'Model / backend profile', + ), + items: [ + for (final id + in _compiledProfile.startsWith('npu-') + ? [_compiledProfile] + : const [ + 'tiny-gguf-cpu', + 'tiny-gguf-lifecycle', + 'tiny-gguf-batching', + 'tiny-gguf-metal', + 'tiny-gguf-vulkan', + 'tiny-gguf-cuda', + 'chat-gguf-cpu', + 'chat-gguf-metal', + 'chat-gguf-vulkan', + 'chat-gguf-cuda', + 'chat-litert-cpu', + 'chat-litert-gpu', + ]) + DropdownMenuItem(value: id, child: Text(id)), + ], + onChanged: _controller.running + ? null + : (value) => setState(() => _profile = value!), + ), + const SizedBox(height: 16), + Wrap( + spacing: 12, + runSpacing: 12, + children: [ + FilledButton( + onPressed: _controller.running + ? null + : () => _controller.run(_profile), + child: const Text('Run tests'), + ), + OutlinedButton( + onPressed: _controller.running + ? _controller.cancel + : null, + child: const Text('Cancel'), + ), + OutlinedButton( + onPressed: _controller.report == null + ? null + : () => _controller.host.export( + 'summary.html', + _controller.report!.toHtml(), + ), + child: const Text('Export report'), + ), + OutlinedButton( + onPressed: _controller.report == null + ? null + : () => _controller.host.export( + 'results.json', + jsonEncode(_controller.report!.toJson()), + ), + child: const Text('Export JSON'), + ), + ], + ), + const SizedBox(height: 20), + if (_controller.running) const LinearProgressIndicator(), + Text( + _controller.phase, + style: Theme.of(context).textTheme.titleLarge, + ), + if (_controller.error != null) + SelectableText(_controller.error!), + if (_controller.report != null) ...[ + Text( + 'Assertions: ${_controller.report!.assertionsPassed ? 'passed' : 'failed or incomplete'}', + ), + Text( + 'Accelerator execution: ${_controller.report!.placement['required'] != true + ? 'not required' + : _controller.report!.acceleratorVerified + ? _controller.report!.placement['reason'] + : 'unverified; inspect native evidence'}', + ), + SelectableText(_controller.host.outputLocation), + ], + for (final record in _controller.records) + ListTile( + contentPadding: EdgeInsets.zero, + title: Text('${record['case_id']} · ${record['status']}'), + subtitle: Text( + '${record['reason'] ?? record['expected'] ?? ''}', + ), + ), + ], + ), + ), + ), + ), + ), + ); +} diff --git a/example/chat_app/pubspec.lock b/example/chat_app/pubspec.lock index 01c35a8bd..850836936 100644 --- a/example/chat_app/pubspec.lock +++ b/example/chat_app/pubspec.lock @@ -331,7 +331,7 @@ packages: source: hosted version: "1.0.1" http: - dependency: "direct dev" + dependency: "direct main" description: name: http sha256: "87721a4a50b19c7f1d49001e51409bddc46303966ce89a65af4f4e6004896412" @@ -428,6 +428,13 @@ packages: relative: true source: path version: "0.0.19" + llamadart_validation: + dependency: "direct main" + description: + path: "../../packages/llamadart_validation" + relative: true + source: path + version: "0.0.0" logging: dependency: transitive description: diff --git a/example/chat_app/pubspec.yaml b/example/chat_app/pubspec.yaml index 93a27483d..2daafbc3f 100644 --- a/example/chat_app/pubspec.yaml +++ b/example/chat_app/pubspec.yaml @@ -12,6 +12,9 @@ dependencies: sdk: flutter llamadart: path: ../.. # Use local version + llamadart_validation: + path: ../../packages/llamadart_validation + http: ^1.1.0 llamadart_llama_cpp_flutter: path: ../../packages/llamadart_llama_cpp_flutter llamadart_litert_lm_flutter: @@ -45,7 +48,6 @@ dev_dependencies: test: ^1.26.3 ffi: any - http: any flutter: uses-material-design: true diff --git a/example/chat_app/test/validation_app_test.dart b/example/chat_app/test/validation_app_test.dart new file mode 100644 index 000000000..db7da5461 --- /dev/null +++ b/example/chat_app/test/validation_app_test.dart @@ -0,0 +1,33 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:llamadart_chat_example/validation_main.dart'; + +void main() { + testWidgets('compiled validation profile is selectable without preparation', ( + tester, + ) async { + const profile = String.fromEnvironment( + 'VALIDATION_PROFILE', + defaultValue: 'tiny-gguf-cpu', + ); + await tester.pumpWidget(const ValidationApp()); + expect(tester.takeException(), isNull); + expect(find.text(profile), findsWidgets); + final field = tester.widget>( + find.byType(DropdownButtonFormField), + ); + expect(field.initialValue, profile); + const native = + String.fromEnvironment('VALIDATION_EXECUTION_PATH') == 'native_c_api'; + expect( + find.text( + native + ? 'Direct native control · does not qualify the public Dart path' + : 'llamadart public API', + ), + findsOneWidget, + ); + await tester.pumpWidget(const SizedBox()); + await tester.pump(); + }); +} diff --git a/example/chat_app/test/validation_controller_test.dart b/example/chat_app/test/validation_controller_test.dart new file mode 100644 index 000000000..4fb0104ef --- /dev/null +++ b/example/chat_app/test/validation_controller_test.dart @@ -0,0 +1,89 @@ +import 'dart:async'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:llamadart_chat_example/validation/controller.dart'; +import 'package:llamadart_chat_example/validation/host.dart'; + +class PreparationHost implements ValidationHost { + @override + ValidationEngine createEngine(ValidationProfile profile) => + throw StateError('Preparation did not finish'); + final started = Completer(); + final pending = Completer<({String path, Map evidence})>(); + bool cancelled = false; + bool finishFails = false; + @override + String get outputLocation => 'test'; + @override + Future start(String id) async {} + @override + Future<({String path, Map evidence})> prepare( + ValidationProfile profile, + ) { + started.complete(); + return pending.future; + } + + @override + void cancelPreparation() { + cancelled = true; + if (!pending.isCompleted) { + pending.completeError(StateError('cancelled download')); + } + } + + @override + Future emit(Map event) async {} + @override + Future finish() async { + if (finishFails) throw StateError('disk full'); + return ValidationReport.parse(''); + } + + @override + Future export(String name, String text) async {} +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + test( + 'cancelling model preparation restores idle and retains incomplete report', + () async { + final host = PreparationHost(); + final controller = ValidationController(host: host); + final run = controller.run('tiny-gguf-cpu'); + await host.started.future; + expect(controller.running, true); + controller.cancel(); + await run; + expect(host.cancelled, true); + expect(controller.running, false); + expect(controller.report?.qualified, false); + controller.dispose(); + }, + ); + test( + 'disposing during preparation cannot notify a disposed controller', + () async { + final host = PreparationHost(); + final controller = ValidationController(host: host); + final run = controller.run('tiny-gguf-cpu'); + await host.started.future; + controller.dispose(); + await run; + expect(controller.running, false); + }, + ); + test('failed report persistence still releases running state', () async { + final host = PreparationHost()..finishFails = true; + final controller = ValidationController(host: host); + final run = controller.run('tiny-gguf-cpu'); + await host.started.future; + controller.cancel(); + await run; + expect(controller.running, false); + expect(controller.error, contains('Report persistence failed')); + controller.dispose(); + }); +} diff --git a/lib/src/backends/litert_lm/litert_lm_service.dart b/lib/src/backends/litert_lm/litert_lm_service.dart index 05e7f782e..2dbc72359 100644 --- a/lib/src/backends/litert_lm/litert_lm_service.dart +++ b/lib/src/backends/litert_lm/litert_lm_service.dart @@ -1177,14 +1177,9 @@ class LiteRtLmService { seededMessages.add(_chatMessageToNativeJson(message)); } - final systemMessage = systemText.isEmpty - ? null - : jsonEncode({ - 'role': LlamaChatRole.system.name, - 'content': [ - {'type': 'text', 'text': systemText.join('\n')}, - ], - }); + // The runtime encodes this text as JSON content; the native conversation + // adds the system role itself. + final systemMessage = systemText.isEmpty ? null : systemText.join('\n'); return ( systemMessage: systemMessage, messages: seededMessages.isEmpty ? null : seededMessages, diff --git a/lib/src/backends/llama_cpp/llama_cpp_service.dart b/lib/src/backends/llama_cpp/llama_cpp_service.dart index 3423f75f8..9e6b54a04 100644 --- a/lib/src/backends/llama_cpp/llama_cpp_service.dart +++ b/lib/src/backends/llama_cpp/llama_cpp_service.dart @@ -1366,10 +1366,11 @@ class LlamaCppService { /// 1. Explicit environment override (`LLAMADART_NATIVE_LIB_DIR` or /// `LLAMADART_BACKEND_MODULE_DIR`) /// 2. Directory of resolved executable (if it looks like a native bundle) - /// 3. Current working directory (if it looks like a native bundle) - /// 4. Hook cache under `.dart_tool/llamadart/native_bundles`, including + /// 3. Standard CLI `bin/../lib` directory (if it looks like a native bundle) + /// 4. Current working directory (if it looks like a native bundle) + /// 5. Hook cache under `.dart_tool/llamadart/native_bundles`, including /// default, custom GitHub, and local archive cache namespaces. - /// 5. Directory of resolved executable (best-effort fallback) + /// 6. Directory of resolved executable (best-effort fallback) static String? resolveWindowsBackendModuleDirectory({ required String resolvedExecutablePath, required String currentDirectoryPath, @@ -1393,6 +1394,16 @@ class LlamaCppService { return executableDir; } + // `dart build cli` places executables in bin/ and native assets in lib/. + if (path.basename(executableDir).toLowerCase() == 'bin') { + final cliLibraryDir = path.normalize( + path.join(executableDir, '..', 'lib'), + ); + if (_containsWindowsNativeModules(cliLibraryDir)) { + return cliLibraryDir; + } + } + if (_containsWindowsNativeModules(currentDirectoryPath)) { return currentDirectoryPath; } diff --git a/packages/llamadart_validation/README.md b/packages/llamadart_validation/README.md new file mode 100644 index 000000000..4bc6a89e2 --- /dev/null +++ b/packages/llamadart_validation/README.md @@ -0,0 +1,95 @@ +# llamadart validation + +Private maintainer package (`publish_to: none`). It exercises the exported +llamadart API through the same core on desktop, Flutter mobile, and Web. +It is independent of the public package's implementation and exports. + +Start from the repository root: + +```sh +dart run tool/testing/run_local_e2e.dart --scenario validation-harness +dart run tool/testing/validation.dart local --profile tiny-gguf-cpu +dart run tool/testing/validation.dart local --profile tiny-gguf-lifecycle +dart run tool/testing/validation.dart build --target desktop --out .dart_tool/validation/bundles/desktop +``` + +See [the runbook](../../doc/cross_platform_validation.md) for models, bundles, +Firebase/GCE setup, run/recovery commands, evidence and cost boundaries. +See [the full plan](../../doc/cross_platform_validation_plan.md) for subsequent +platform and feature qualification. A quick run does not qualify a release. + +`assets/profiles/` locks each model URL/revision/SHA256/size and inference +configuration. `lib/llamadart_validation.dart` is the platform-neutral suite; conditional native +adapters and `lib/io.dart` handle filesystem/runtime checks. `bin/run.dart` and `bin/report.dart` are the +CLI entrypoints. Flutter uses the package's profile assets and shared runner. +Native wrappers persist JSONL per event; reports always derive from that journal. + +The two `npu-*` profiles are locked candidates. Inspect their local prerequisites +with `validation.dart npu-preflight`; the opt-in Android builder packages verified +local model/vendor inputs and the installed app checks its SoC and captures +per-generation execution proof. S24 NPU execution is verified but semantic +qualification still fails; Pixel 10 hardware execution remains NOT_RUN. + +Exit 0 means this selected run qualified, 1 means failed/incomplete. +Unknown/dirty source provenance, missing counters, unknown backend placement, skipped mandatory cases and missing +records remain incomplete. Imported model preparation must prove the locked +SHA256 and byte size. Desktop qualification additionally requires the portable +CLI's verified runtime inventory and bundle manifest hash; JIT remains diagnostic. +The CLI rejects runtime overrides, validates its environment and runtime payload, +and anchors native discovery to the bundle. Older desktop journals without this +proof preserve assertions but no longer qualify when reimported. `release` selection deliberately records unimplemented +feature packs as NOT_RUN until their fixtures and wrappers are qualified. +The reporter derives mandatory cases, effective settings and accelerator-proof +requirements from the validated profile. Self-declared inventory, rehashed +conflicting settings and per-record unsupported exemptions cannot waive them. + +Reports expose independent `summary.qualification_gaps`: assertion failures, +execution errors, unexecuted cases, missing accelerator evidence, incomplete +provenance, and journal/lifecycle problems. Multiple gaps can coexist. An +unverified accelerator does not establish model or platform incompatibility; +successful inference also does not prove GPU/NPU execution. Execution errors +require diagnosis against the retained native logs and host setup. The HTML +report shows these distinctions and the placement verifier's reason. These +diagnostics do not relax qualification requirements or replace case results. + +`C01.load` measures public load/readiness, which can precede native engine +initialization. In LiteRT, the first `C02.unicode` tokenization call can include +deferred engine creation. Its `tokenize_call_ms` is therefore not isolated +tokenizer latency. The runner records this timing scope, separate detokenization +latency, and the active public operation on errors/timeouts. A first-use timeout +does not establish a tokenizer defect. Case deadlines remain unchanged; provider +process deadlines can additionally include model preparation and host startup. + +Profiles select `quick`, `focused` or `release`. Focused profiles require a +nonempty, unique `focus_features` list; `tiny-gguf-lifecycle` adds the second +dispose/load/generate cycle to the quick CPU run. Core feature IDs are `text`, +`unicode`, `thinking`, `history`, `tools`, `streaming`, `batching`, `lifecycle`, `guards` and +`performance`. Unimplemented selected cases stay NOT_RUN. + +Journal schema 2 includes the versioned case/feature catalog, resolved synthetic +fixtures, explicit omissions and a catalog hash. Each terminal case carries its +case version and fixture hash. Defaults live in `lib/src/case_catalog.dart` and +compile into every host; model overrides remain in the locked JSON profiles. +The reporter validates metadata against the executable catalog and still imports +schema-1 journals with their original inventory. It does not invent missing +catalog provenance for those older runs. + +`tiny-gguf-batching` adds C11 text/thinking parity across default, 1-piece/1-byte, +and recovered default worker settings. Trial outputs/configuration/finish order +and metrics are retained; chunk counts may differ. LiteRT Web checks each native +option's typed rejection instead. NPU deterministic parity, GGUF Web controls and +tool-bearing fixtures remain unqualified. Catalog version 2 preserves imports of +version-1 journals against their original case/fixture definitions. + +Catalog 4 adds executable C02 Unicode generation (exact output, no trimming), +C05 thinking enabled/disabled (512 output-token bound, separated nonempty thinking +when enabled and none when disabled), and C07 tool choice (`auto`, `required`, +`none`, 128 output-token bound). C07 checks one weather call with exact city +arguments, public completion semantics, synthetic tool-result consumption and +default recovery. The actual tools, choices, prompts, messages and per-call +settings are logged; tool schemas are produced by the public ToolDefinition API. +Failures are strict model/runtime observations, not automatic diagnoses. +Thinking-budget controls and tool-bearing batching remain separate planned work. +Native C API controls do not execute these public feature cases. Catalogs 1–3 +retain their original fixtures and unimplemented obligations on import. No +existing device/model run is upgraded by this implementation. diff --git a/packages/llamadart_validation/analysis_options.yaml b/packages/llamadart_validation/analysis_options.yaml new file mode 100644 index 000000000..572dd239d --- /dev/null +++ b/packages/llamadart_validation/analysis_options.yaml @@ -0,0 +1 @@ +include: package:lints/recommended.yaml diff --git a/packages/llamadart_validation/assets/profiles/chat-gguf-cpu.json b/packages/llamadart_validation/assets/profiles/chat-gguf-cpu.json new file mode 100644 index 000000000..ede7812c4 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-gguf-cpu.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "id": "chat-gguf-cpu", + "runtime": "gguf", + "backend": "cpu", + "model": { + "kind": "chat", + "filename": "Qwen3.5-0.8B-Q4_0.gguf", + "revision": "8fea620810c4afa23dd6443f999a48574c1611a3", + "url": "https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/resolve/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q4_0.gguf", + "sha256": "57d1997790d1744fba5b40a7317df71ea5e2acee28c47e78f0cce39c0703f8cf", + "bytes": 563036064 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/chat-gguf-cuda.json b/packages/llamadart_validation/assets/profiles/chat-gguf-cuda.json new file mode 100644 index 000000000..95791ae37 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-gguf-cuda.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "id": "chat-gguf-cuda", + "runtime": "gguf", + "backend": "cuda", + "model": { + "kind": "chat", + "filename": "Qwen3.5-0.8B-Q4_0.gguf", + "revision": "8fea620810c4afa23dd6443f999a48574c1611a3", + "url": "https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/resolve/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q4_0.gguf", + "sha256": "57d1997790d1744fba5b40a7317df71ea5e2acee28c47e78f0cce39c0703f8cf", + "bytes": 563036064 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/chat-gguf-metal.json b/packages/llamadart_validation/assets/profiles/chat-gguf-metal.json new file mode 100644 index 000000000..31c999b6e --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-gguf-metal.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "id": "chat-gguf-metal", + "runtime": "gguf", + "backend": "metal", + "model": { + "kind": "chat", + "filename": "Qwen3.5-0.8B-Q4_0.gguf", + "revision": "8fea620810c4afa23dd6443f999a48574c1611a3", + "url": "https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/resolve/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q4_0.gguf", + "sha256": "57d1997790d1744fba5b40a7317df71ea5e2acee28c47e78f0cce39c0703f8cf", + "bytes": 563036064 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/chat-gguf-vulkan.json b/packages/llamadart_validation/assets/profiles/chat-gguf-vulkan.json new file mode 100644 index 000000000..88de6b177 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-gguf-vulkan.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "id": "chat-gguf-vulkan", + "runtime": "gguf", + "backend": "vulkan", + "model": { + "kind": "chat", + "filename": "Qwen3.5-0.8B-Q4_0.gguf", + "revision": "8fea620810c4afa23dd6443f999a48574c1611a3", + "url": "https://huggingface.co/ggml-org/Qwen3.5-0.8B-GGUF/resolve/8fea620810c4afa23dd6443f999a48574c1611a3/Qwen3.5-0.8B-Q4_0.gguf", + "sha256": "57d1997790d1744fba5b40a7317df71ea5e2acee28c47e78f0cce39c0703f8cf", + "bytes": 563036064 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/chat-litert-cpu.json b/packages/llamadart_validation/assets/profiles/chat-litert-cpu.json new file mode 100644 index 000000000..636a1c06c --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-litert-cpu.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "chat-litert-cpu", + "runtime": "litert", + "backend": "cpu", + "model": { + "id": "chat-litert", + "revision": "8414150f2e9dcc82449bcc9c5abc404b399a4d06", + "filename": "Qwen3-0.6B.litertlm", + "sha256": "555579ff2f4fd13379abe69c1c3ab5200f7338bc92471557f1d6614a6e5ab0b4", + "bytes": 614236160, + "kind": "chat", + "url": "https://huggingface.co/litert-community/Qwen3-0.6B/resolve/8414150f2e9dcc82449bcc9c5abc404b399a4d06/Qwen3-0.6B.litertlm" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/chat-litert-gpu.json b/packages/llamadart_validation/assets/profiles/chat-litert-gpu.json new file mode 100644 index 000000000..d5d9a995b --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/chat-litert-gpu.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "chat-litert-gpu", + "runtime": "litert", + "backend": "gpu", + "model": { + "id": "chat-litert", + "revision": "8414150f2e9dcc82449bcc9c5abc404b399a4d06", + "filename": "Qwen3-0.6B.litertlm", + "sha256": "555579ff2f4fd13379abe69c1c3ab5200f7338bc92471557f1d6614a6e5ab0b4", + "bytes": 614236160, + "kind": "chat", + "url": "https://huggingface.co/litert-community/Qwen3-0.6B/resolve/8414150f2e9dcc82449bcc9c5abc404b399a4d06/Qwen3-0.6B.litertlm" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/gemma3-litert-cpu.json b/packages/llamadart_validation/assets/profiles/gemma3-litert-cpu.json new file mode 100644 index 000000000..6010b07fe --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/gemma3-litert-cpu.json @@ -0,0 +1,33 @@ +{ + "schema_version": 1, + "id": "gemma3-litert-cpu", + "runtime": "litert", + "backend": "cpu", + "model": { + "id": "gemma3-1b-it-cpu-q4", + "revision": "a6306a4e292016480083b73b8dc6f3f939ae04c3", + "filename": "gemma3-1b-it-int4.litertlm", + "sha256": "1325ae366d31950f137c9c357b9fa89448b176d76998180c08ceaca78bba98be", + "bytes": 584417280, + "kind": "chat", + "access": "gated-local-staging", + "quantization": "q4_per_channel", + "url": "https://huggingface.co/litert-community/Gemma3-1B-IT/resolve/a6306a4e292016480083b73b8dc6f3f939ae04c3/gemma3-1b-it-int4.litertlm" + }, + "selection": "quick", + "context_size": 1280, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + }, + "enable_thinking": true, + "history_controls": true +} diff --git a/packages/llamadart_validation/assets/profiles/gemma4-gguf-cpu.json b/packages/llamadart_validation/assets/profiles/gemma4-gguf-cpu.json new file mode 100644 index 000000000..83f8970ac --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/gemma4-gguf-cpu.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "gemma4-gguf-cpu", + "runtime": "gguf", + "backend": "cpu", + "model": { + "kind": "chat", + "filename": "gemma-4-E2B-it-Q4_K_S.gguf", + "revision": "90f9618340396838ee7ff5b0ba2da27da62953d3", + "url": "https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/90f9618340396838ee7ff5b0ba2da27da62953d3/gemma-4-E2B-it-Q4_K_S.gguf", + "sha256": "0a2fac16f388b4839f075dedb681357aec3e73a96bd66b413e462b6853550c99", + "bytes": 3043932288 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + }, + "enable_thinking": false +} diff --git a/packages/llamadart_validation/assets/profiles/gemma4-gguf-cuda.json b/packages/llamadart_validation/assets/profiles/gemma4-gguf-cuda.json new file mode 100644 index 000000000..5303eedc6 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/gemma4-gguf-cuda.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "gemma4-gguf-cuda", + "runtime": "gguf", + "backend": "cuda", + "model": { + "kind": "chat", + "filename": "gemma-4-E2B-it-Q4_K_S.gguf", + "revision": "90f9618340396838ee7ff5b0ba2da27da62953d3", + "url": "https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/90f9618340396838ee7ff5b0ba2da27da62953d3/gemma-4-E2B-it-Q4_K_S.gguf", + "sha256": "0a2fac16f388b4839f075dedb681357aec3e73a96bd66b413e462b6853550c99", + "bytes": 3043932288 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + }, + "enable_thinking": false +} diff --git a/packages/llamadart_validation/assets/profiles/gemma4-gguf-metal.json b/packages/llamadart_validation/assets/profiles/gemma4-gguf-metal.json new file mode 100644 index 000000000..218704797 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/gemma4-gguf-metal.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "gemma4-gguf-metal", + "runtime": "gguf", + "backend": "metal", + "model": { + "kind": "chat", + "filename": "gemma-4-E2B-it-Q4_K_S.gguf", + "revision": "90f9618340396838ee7ff5b0ba2da27da62953d3", + "url": "https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/90f9618340396838ee7ff5b0ba2da27da62953d3/gemma-4-E2B-it-Q4_K_S.gguf", + "sha256": "0a2fac16f388b4839f075dedb681357aec3e73a96bd66b413e462b6853550c99", + "bytes": 3043932288 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + }, + "enable_thinking": false +} diff --git a/packages/llamadart_validation/assets/profiles/gemma4-gguf-vulkan.json b/packages/llamadart_validation/assets/profiles/gemma4-gguf-vulkan.json new file mode 100644 index 000000000..53338ed08 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/gemma4-gguf-vulkan.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "gemma4-gguf-vulkan", + "runtime": "gguf", + "backend": "vulkan", + "model": { + "kind": "chat", + "filename": "gemma-4-E2B-it-Q4_K_S.gguf", + "revision": "90f9618340396838ee7ff5b0ba2da27da62953d3", + "url": "https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/90f9618340396838ee7ff5b0ba2da27da62953d3/gemma-4-E2B-it-Q4_K_S.gguf", + "sha256": "0a2fac16f388b4839f075dedb681357aec3e73a96bd66b413e462b6853550c99", + "bytes": 3043932288 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + }, + "enable_thinking": false +} diff --git a/packages/llamadart_validation/assets/profiles/gemma4-litert-cpu.json b/packages/llamadart_validation/assets/profiles/gemma4-litert-cpu.json new file mode 100644 index 000000000..5684572a1 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/gemma4-litert-cpu.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "gemma4-litert-cpu", + "runtime": "litert", + "backend": "cpu", + "model": { + "kind": "chat", + "filename": "gemma-4-E2B-it.litertlm", + "revision": "6b78abd019e61a1ca4cbe3b212d2c9ce8ff38a94", + "url": "https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm/resolve/6b78abd019e61a1ca4cbe3b212d2c9ce8ff38a94/gemma-4-E2B-it.litertlm", + "sha256": "181938105e0eefd105961417e8da75903eacda102c4fce9ce90f50b97139a63c", + "bytes": 2588147712 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + }, + "enable_thinking": false +} diff --git a/packages/llamadart_validation/assets/profiles/gemma4-litert-gpu.json b/packages/llamadart_validation/assets/profiles/gemma4-litert-gpu.json new file mode 100644 index 000000000..666289014 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/gemma4-litert-gpu.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "gemma4-litert-gpu", + "runtime": "litert", + "backend": "gpu", + "model": { + "kind": "chat", + "filename": "gemma-4-E2B-it.litertlm", + "revision": "6b78abd019e61a1ca4cbe3b212d2c9ce8ff38a94", + "url": "https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm/resolve/6b78abd019e61a1ca4cbe3b212d2c9ce8ff38a94/gemma-4-E2B-it.litertlm", + "sha256": "181938105e0eefd105961417e8da75903eacda102c4fce9ce90f50b97139a63c", + "bytes": 2588147712 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + }, + "enable_thinking": false +} diff --git a/packages/llamadart_validation/assets/profiles/npu-qualcomm-sm8650.json b/packages/llamadart_validation/assets/profiles/npu-qualcomm-sm8650.json new file mode 100644 index 000000000..0e7ac387f --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/npu-qualcomm-sm8650.json @@ -0,0 +1,78 @@ +{ + "schema_version": 1, + "id": "npu-qualcomm-sm8650", + "runtime": "litert", + "backend": "npu", + "model": { + "id": "gemma3-1b-it-qualcomm-sm8650", + "revision": "a6306a4e292016480083b73b8dc6f3f939ae04c3", + "filename": "Gemma3-1B-IT_q4_ekv1280_sm8650.litertlm", + "sha256": "85d2ea5199802f913818d53897b3a304bcf983abb993393e6b1749fbdb005552", + "bytes": 690094080, + "kind": "chat", + "access": "gated-local-staging", + "quantization": "q4_per_channel", + "url": "https://huggingface.co/litert-community/Gemma3-1B-IT/resolve/a6306a4e292016480083b73b8dc6f3f939ae04c3/Gemma3-1B-IT_q4_ekv1280_sm8650.litertlm" + }, + "selection": "quick", + "context_size": 1280, + "threads": 4, + "max_tokens": 32, + "npu_target": { + "soc": "Qualcomm_SM8650", + "abi": "arm64-v8a", + "minimum_android_api": 31, + "qairt_version": "2.47.0.260601", + "libraries": { + "libLiteRtDispatch_Qualcomm.so": { + "elf_machine": 183, + "elf_class": 2, + "sha256": "d07d3324ad7617719fa613012168870892a221c11bbbd3795e1dcb64026cbcb5" + }, + "libQnnHtp.so": { + "elf_machine": 183, + "elf_class": 2, + "sha256": "c0488f2df87932a42ca0a563883e6fba190896bca439ad0fdaa2428358ab5092" + }, + "libQnnSystem.so": { + "elf_machine": 183, + "elf_class": 2, + "sha256": "077a8b20a53b216d006b85b58dd754a9e958e02f98e9c79d46619db6f8edfec9" + }, + "libQnnHtpPrepare.so": { + "elf_machine": 183, + "elf_class": 2, + "sha256": "9988ce10ffee6813ffd218df308478f44dbc758cc89aed9cb98cfebb030ca72e" + }, + "libQnnHtpV75Stub.so": { + "elf_machine": 183, + "elf_class": 2, + "sha256": "b126a79ebeabf09949656be42326546b03f3216ad671c9c0e4ac935cefa5a631" + }, + "libQnnHtpV75Skel.so": { + "elf_machine": 164, + "elf_class": 1, + "sha256": "607be3d7ec64df053f019438ad6eea59dd3eea5d5f985001709d52d5663479ba" + }, + "libLlamadartVendor_Qualcomm.so": { + "elf_machine": 183, + "elf_class": 2, + "sha256": "0b6c4d3c521047046689cb379eae8b57f1ef996ddd15fdb4090a081b94c4aa70" + } + }, + "device_soc_models": [ + "SM8650" + ], + "firebase_model": "SC-51E" + }, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/npu-tensor-g5.json b/packages/llamadart_validation/assets/profiles/npu-tensor-g5.json new file mode 100644 index 000000000..c15a3d381 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/npu-tensor-g5.json @@ -0,0 +1,53 @@ +{ + "schema_version": 1, + "id": "npu-tensor-g5", + "runtime": "litert", + "backend": "npu", + "model": { + "id": "gemma3-1b-it-tensor-g5", + "revision": "a6306a4e292016480083b73b8dc6f3f939ae04c3", + "filename": "Gemma3-1B-IT_q8_ekv1280_Google_Tensor_G5.litertlm", + "sha256": "1ed29548b302764ce32ebf03d7df8fff943218b76d14230e97ee4bd0224cd8d1", + "bytes": 1678542365, + "kind": "chat", + "access": "gated-local-staging", + "quantization": "q8_per_channel", + "url": "https://huggingface.co/litert-community/Gemma3-1B-IT/resolve/a6306a4e292016480083b73b8dc6f3f939ae04c3/Gemma3-1B-IT_q8_ekv1280_Google_Tensor_G5.litertlm" + }, + "selection": "quick", + "context_size": 1280, + "threads": 4, + "max_tokens": 32, + "npu_target": { + "soc": "Google_Tensor_G5", + "abi": "arm64-v8a", + "minimum_android_api": 31, + "qairt_version": null, + "libraries": { + "libLiteRtDispatch_GoogleTensor.so": { + "elf_machine": 183, + "elf_class": 2, + "sha256": "b8ff7daf8461265e64d782f267573487aa33e88eab551d04181f38ec84653798" + }, + "libLlamadartVendor_GoogleTensor.so": { + "elf_machine": 183, + "elf_class": 2, + "sha256": "71a3847419d37ea31bf93cf38e6cf4d61562a3f46aded42903fcf03873ef9467" + } + }, + "device_soc_models": [ + "Tensor G5" + ], + "firebase_model": "frankel" + }, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/qwen35-litert-cpu.json b/packages/llamadart_validation/assets/profiles/qwen35-litert-cpu.json new file mode 100644 index 000000000..fb86caaae --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/qwen35-litert-cpu.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "qwen35-litert-cpu", + "runtime": "litert", + "backend": "cpu", + "model": { + "kind": "chat", + "filename": "Qwen3.5-0.8B_int8.litertlm", + "revision": "c23b16e43ada6ead533b12593fe500bbe268014f", + "url": "https://huggingface.co/litert-community/Qwen3.5-0.8B/resolve/c23b16e43ada6ead533b12593fe500bbe268014f/Qwen3.5-0.8B_int8.litertlm", + "sha256": "684d4d34adf7176eb47f6026ff65c33d42584737254e5524a8d1ad62edc21b98", + "bytes": 963184864 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + }, + "enable_thinking": false +} diff --git a/packages/llamadart_validation/assets/profiles/qwen35-litert-gpu.json b/packages/llamadart_validation/assets/profiles/qwen35-litert-gpu.json new file mode 100644 index 000000000..7824cbc0d --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/qwen35-litert-gpu.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "id": "qwen35-litert-gpu", + "runtime": "litert", + "backend": "gpu", + "model": { + "kind": "chat", + "filename": "Qwen3.5-0.8B_int8.litertlm", + "revision": "c23b16e43ada6ead533b12593fe500bbe268014f", + "url": "https://huggingface.co/litert-community/Qwen3.5-0.8B/resolve/c23b16e43ada6ead533b12593fe500bbe268014f/Qwen3.5-0.8B_int8.litertlm", + "sha256": "684d4d34adf7176eb47f6026ff65c33d42584737254e5524a8d1ad62edc21b98", + "bytes": 963184864 + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + } + }, + "enable_thinking": false +} diff --git a/packages/llamadart_validation/assets/profiles/tiny-gguf-batching.json b/packages/llamadart_validation/assets/profiles/tiny-gguf-batching.json new file mode 100644 index 000000000..356120675 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/tiny-gguf-batching.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "id": "tiny-gguf-batching", + "runtime": "gguf", + "backend": "cpu", + "model": { + "id": "tiny-gguf", + "revision": "99dd1a73db5a37100bd4ae633f4cfce6560e1567", + "filename": "stories15M.gguf", + "sha256": "61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a", + "bytes": 98357920, + "kind": "raw", + "url": "https://huggingface.co/ggml-org/tiny-llamas/resolve/99dd1a73db5a37100bd4ae633f4cfce6560e1567/stories15M.gguf" + }, + "selection": "focused", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + }, + "unicode": { + "expected_prefix": " " + } + }, + "focus_features": [ + "batching" + ] +} diff --git a/packages/llamadart_validation/assets/profiles/tiny-gguf-cpu.json b/packages/llamadart_validation/assets/profiles/tiny-gguf-cpu.json new file mode 100644 index 000000000..0951983f6 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/tiny-gguf-cpu.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "tiny-gguf-cpu", + "runtime": "gguf", + "backend": "cpu", + "model": { + "id": "tiny-gguf", + "revision": "99dd1a73db5a37100bd4ae633f4cfce6560e1567", + "filename": "stories15M.gguf", + "sha256": "61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a", + "bytes": 98357920, + "kind": "raw", + "url": "https://huggingface.co/ggml-org/tiny-llamas/resolve/99dd1a73db5a37100bd4ae633f4cfce6560e1567/stories15M.gguf" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + }, + "unicode": { + "expected_prefix": " " + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/tiny-gguf-cuda.json b/packages/llamadart_validation/assets/profiles/tiny-gguf-cuda.json new file mode 100644 index 000000000..36d998b21 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/tiny-gguf-cuda.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "tiny-gguf-cuda", + "runtime": "gguf", + "backend": "cuda", + "model": { + "id": "tiny-gguf", + "revision": "99dd1a73db5a37100bd4ae633f4cfce6560e1567", + "filename": "stories15M.gguf", + "sha256": "61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a", + "bytes": 98357920, + "kind": "raw", + "url": "https://huggingface.co/ggml-org/tiny-llamas/resolve/99dd1a73db5a37100bd4ae633f4cfce6560e1567/stories15M.gguf" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + }, + "unicode": { + "expected_prefix": " " + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/tiny-gguf-lifecycle.json b/packages/llamadart_validation/assets/profiles/tiny-gguf-lifecycle.json new file mode 100644 index 000000000..e417697f9 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/tiny-gguf-lifecycle.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "id": "tiny-gguf-lifecycle", + "runtime": "gguf", + "backend": "cpu", + "model": { + "id": "tiny-gguf", + "revision": "99dd1a73db5a37100bd4ae633f4cfce6560e1567", + "filename": "stories15M.gguf", + "sha256": "61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a", + "bytes": 98357920, + "kind": "raw", + "url": "https://huggingface.co/ggml-org/tiny-llamas/resolve/99dd1a73db5a37100bd4ae633f4cfce6560e1567/stories15M.gguf" + }, + "selection": "focused", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + }, + "unicode": { + "expected_prefix": " " + } + }, + "focus_features": [ + "lifecycle" + ] +} diff --git a/packages/llamadart_validation/assets/profiles/tiny-gguf-metal.json b/packages/llamadart_validation/assets/profiles/tiny-gguf-metal.json new file mode 100644 index 000000000..14767c406 --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/tiny-gguf-metal.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "tiny-gguf-metal", + "runtime": "gguf", + "backend": "metal", + "model": { + "id": "tiny-gguf", + "revision": "99dd1a73db5a37100bd4ae633f4cfce6560e1567", + "filename": "stories15M.gguf", + "sha256": "61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a", + "bytes": 98357920, + "kind": "raw", + "url": "https://huggingface.co/ggml-org/tiny-llamas/resolve/99dd1a73db5a37100bd4ae633f4cfce6560e1567/stories15M.gguf" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + }, + "unicode": { + "expected_prefix": " " + } + } +} diff --git a/packages/llamadart_validation/assets/profiles/tiny-gguf-vulkan.json b/packages/llamadart_validation/assets/profiles/tiny-gguf-vulkan.json new file mode 100644 index 000000000..371d0e0ff --- /dev/null +++ b/packages/llamadart_validation/assets/profiles/tiny-gguf-vulkan.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "id": "tiny-gguf-vulkan", + "runtime": "gguf", + "backend": "vulkan", + "model": { + "id": "tiny-gguf", + "revision": "99dd1a73db5a37100bd4ae633f4cfce6560e1567", + "filename": "stories15M.gguf", + "sha256": "61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a", + "bytes": 98357920, + "kind": "raw", + "url": "https://huggingface.co/ggml-org/tiny-llamas/resolve/99dd1a73db5a37100bd4ae633f4cfce6560e1567/stories15M.gguf" + }, + "selection": "quick", + "context_size": 1024, + "threads": 4, + "max_tokens": 32, + "fixtures": { + "hello": { + "prompt": "Reply with one short sentence saying hello.", + "regex": "\\bhello\\b" + }, + "arithmetic": { + "prompt": "What is 2 + 2? Reply with only the number.", + "regex": "^4[.!]?$" + }, + "unicode": { + "expected_prefix": " " + } + } +} diff --git a/packages/llamadart_validation/assets/speech/README.md b/packages/llamadart_validation/assets/speech/README.md new file mode 100644 index 000000000..cb37603a0 --- /dev/null +++ b/packages/llamadart_validation/assets/speech/README.md @@ -0,0 +1,15 @@ +# Speech fixtures + +`jfk.wav` is the familiar excerpt from John F. Kennedy's January 20, 1961 +inaugural address, distributed as the `samples/jfk.wav` fixture in whisper.cpp. +The US federal government speech is public domain. Its exact bytes and reference +words are locked in `stt.json`; the reference ignores punctuation/case only. +No personal microphone recordings are included. + +The Qwen3-ASR and Qwen3-TTS model/projector locks match the maintained chat-app +catalog. Model files are downloaded separately, never included in the app or +repository. Model licenses remain with the linked upstream repositories. + +The fixture reference is a transcription oracle, not proof of exact-model +qualification. A failed transcription remains a failure; do not revise the +reference or threshold to match the output of a failing run. diff --git a/packages/llamadart_validation/assets/speech/jfk.wav b/packages/llamadart_validation/assets/speech/jfk.wav new file mode 100644 index 000000000..3184d372c Binary files /dev/null and b/packages/llamadart_validation/assets/speech/jfk.wav differ diff --git a/packages/llamadart_validation/assets/speech/litert-asr.json b/packages/llamadart_validation/assets/speech/litert-asr.json new file mode 100644 index 000000000..bbff92572 --- /dev/null +++ b/packages/llamadart_validation/assets/speech/litert-asr.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "pack": "litert-asr", + "preset": "moonshineTiny", + "backend": "cpu", + "model": { + "filename": "moonshine_tiny_5s_i8.tflite", + "sha256": "97abdeea122d579229091659c24c59d988c6419d453a200f6471241a53b9a9b9", + "bytes": 51936896, + "revision": "beb49ee5028b4fb21eb989bcbd2db30a433373db", + "url": "https://huggingface.co/litert-community/moonshine-tiny/resolve/beb49ee5028b4fb21eb989bcbd2db30a433373db/moonshine_tiny_5s_i8.tflite" + }, + "tokenizer": { + "filename": "moonshine_tokenizer.json", + "sha256": "6579793438bc4fbafffacf699169ff53e3769c5a0a0f5e71cdee8853e8130deb", + "bytes": 1985530, + "revision": "390624ed33d594443aa4aa221f5b9f283b545b5a", + "url": "https://huggingface.co/UsefulSensors/moonshine-tiny/resolve/390624ed33d594443aa4aa221f5b9f283b545b5a/tokenizer.json" + }, + "fixture": { + "filename": "jfk.wav", + "sha256": "59dfb9a4acb36fe2a2affc14bacbee2920ff435cb13cc314a08c13f66ba7860e", + "bytes": 352078, + "reference": "And so my fellow Americans ask not what your country can do for you ask what you can do for your country", + "wer_threshold": 0 + } +} diff --git a/packages/llamadart_validation/assets/speech/stt.json b/packages/llamadart_validation/assets/speech/stt.json new file mode 100644 index 000000000..4f4a812c7 --- /dev/null +++ b/packages/llamadart_validation/assets/speech/stt.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "pack": "stt", + "model": { + "kind": "raw", + "filename": "Qwen3-ASR-0.6B-Q8_0.gguf", + "bytes": 804749248, + "sha256": "bca259818b50ca7c4c05e9bdb35a5dc04fa039653a6d6f3f0f331f96f6aa1971", + "revision": "928ab958557df9aa2ef1c93e0e83c7ad0933fae2", + "url": "https://huggingface.co/ggml-org/Qwen3-ASR-0.6B-GGUF/resolve/928ab958557df9aa2ef1c93e0e83c7ad0933fae2/Qwen3-ASR-0.6B-Q8_0.gguf" + }, + "projector": { + "kind": "raw", + "filename": "mmproj-Qwen3-ASR-0.6B-Q8_0.gguf", + "bytes": 214392480, + "sha256": "41a342b5e4c514e968cb756de6cd1b7be39eff43c44c57a2ef5fc6522e36603d", + "revision": "928ab958557df9aa2ef1c93e0e83c7ad0933fae2", + "url": "https://huggingface.co/ggml-org/Qwen3-ASR-0.6B-GGUF/resolve/928ab958557df9aa2ef1c93e0e83c7ad0933fae2/mmproj-Qwen3-ASR-0.6B-Q8_0.gguf" + }, + "fixture": { + "filename": "jfk.wav", + "sha256": "59dfb9a4acb36fe2a2affc14bacbee2920ff435cb13cc314a08c13f66ba7860e", + "bytes": 352078, + "reference": "And so my fellow Americans ask not what your country can do for you ask what you can do for your country", + "wer_threshold": 0 + } +} diff --git a/packages/llamadart_validation/assets/speech/tts.json b/packages/llamadart_validation/assets/speech/tts.json new file mode 100644 index 000000000..d38341c35 --- /dev/null +++ b/packages/llamadart_validation/assets/speech/tts.json @@ -0,0 +1,20 @@ +{ + "schema_version": 1, + "pack": "tts", + "model": { + "kind": "raw", + "filename": "Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf", + "bytes": 1035965280, + "sha256": "8d18c94acb2addd042f97da63c98be144eafa76d0d9495177eab65130cf85129", + "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", + "url": "https://huggingface.co/ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF/resolve/ca27d74bc954b73dadab5b71ca265d87fc861a7c/Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf" + }, + "projector": { + "kind": "raw", + "filename": "mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf", + "bytes": 446422912, + "sha256": "6fd65188839bcd6ecc91b277ad471e22a0edfada4699a0fe82f1165c18cfcce2", + "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", + "url": "https://huggingface.co/ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF/resolve/ca27d74bc954b73dadab5b71ca265d87fc861a7c/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf" + } +} diff --git a/packages/llamadart_validation/bin/dedicated_speech.dart b/packages/llamadart_validation/bin/dedicated_speech.dart new file mode 100644 index 000000000..3d2a6b376 --- /dev/null +++ b/packages/llamadart_validation/bin/dedicated_speech.dart @@ -0,0 +1,94 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:llamadart/llamadart.dart'; +import 'package:llamadart_validation/io.dart'; +import 'package:llamadart_validation/src/runtime_environment.dart'; +import 'package:llamadart_validation/src/speech_runner.dart'; +import 'package:path/path.dart' as p; + +Future main(List args) async { + Directory? owned; + try { + final options = parseOptions(args, { + 'pack', + 'backend', + 'model', + 'tokenizer', + 'out', + 'cache', + }); + if (options['pack'] != 'litert-asr' || options['backend'] != 'cpu') { + throw ArgumentError('Dedicated speech requires litert-asr and cpu'); + } + requireValidationRuntimeEnvironment(); + final output = Directory( + options['out'] ?? (throw ArgumentError('--out required')), + ); + if (output.existsSync()) throw StateError('Output must be new'); + output.createSync(recursive: true); + File(p.join(output.path, '.speech-owner')).createSync(exclusive: true); + owned = output; + final assets = p.join( + File.fromUri(Platform.script).parent.parent.path, + 'assets', + 'speech', + ); + final lock = + jsonDecode(File(p.join(assets, 'litert-asr.json')).readAsStringSync()) + as Map; + for (final name in ['model', 'tokenizer']) { + final file = File( + options[name] ?? (throw ArgumentError('--$name required')), + ); + if (await file.length() != lock[name]['bytes'] || + (await sha256.bind(file.openRead()).first).toString() != + lock[name]['sha256']) { + throw FormatException('Locked $name mismatch'); + } + } + final fixture = lock['fixture'] as Map; + final wav = await File( + p.join(assets, fixture['filename'] as String), + ).readAsBytes(); + if (wav.length != fixture['bytes'] || + sha256.convert(wav).toString() != fixture['sha256']) { + throw const FormatException('Fixture mismatch'); + } + final result = await runSpeechValidation( + PublicDedicatedSpeechAdapter( + config: LiteRtLmAsrRuntimeConfig( + modelPath: options['model']!, + tokenizerPath: options['tokenizer']!, + modelPreset: LiteRtLmAsrModelPreset.moonshineTiny, + ), + wav: wav, + reference: fixture['reference'] as String, + ), + ); + result.addAll({ + 'pack': 'litert-asr', + 'backend': 'cpu', + 'model_lock': lock, + 'os': Platform.operatingSystem, + 'sample_rate_hz': 16000, + 'pcm_push_samples': 1600, + }); + await File( + p.join(output.path, 'speech-results.json'), + ).writeAsString('${const JsonEncoder.withIndent(' ').convert(result)}\n'); + stdout.writeln( + 'Speech report: ${p.join(output.path, 'speech-results.json')}', + ); + if (result['functional_pass'] != true) exitCode = 1; + } catch (error) { + stderr.writeln('Dedicated speech validation failed: ${error.runtimeType}'); + if (owned != null) { + await File(p.join(owned.path, 'failure.json')).writeAsString( + jsonEncode({'qualified': false, 'error_type': '${error.runtimeType}'}), + ); + } + exitCode = 1; + } +} diff --git a/packages/llamadart_validation/bin/report.dart b/packages/llamadart_validation/bin/report.dart new file mode 100644 index 000000000..4435469ef --- /dev/null +++ b/packages/llamadart_validation/bin/report.dart @@ -0,0 +1,23 @@ +import 'dart:io'; + +import 'package:llamadart_validation/io.dart'; + +void main(List args) { + if (args.length != 1 && !(args.length == 3 && args[1] == '--native-log')) { + stderr.writeln('Usage: report [--native-log ]'); + exitCode = 64; + return; + } + try { + exitCode = + writeReports( + Directory(args.first), + nativeLog: args.length == 3 ? File(args[2]).readAsStringSync() : null, + ).qualified + ? 0 + : 1; + } catch (error) { + stderr.writeln(error); + exitCode = 1; + } +} diff --git a/packages/llamadart_validation/bin/run.dart b/packages/llamadart_validation/bin/run.dart new file mode 100644 index 000000000..a250d231f --- /dev/null +++ b/packages/llamadart_validation/bin/run.dart @@ -0,0 +1,171 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:llamadart_validation/io.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:llamadart_validation/src/desktop_bundle.dart'; +import 'package:llamadart_validation/src/runtime_environment.dart'; +import 'package:path/path.dart' as p; + +Future main(List args) async { + try { + final options = parseOptions(args, { + 'help', + 'list', + 'assets', + 'profile', + 'profile-file', + 'model', + 'cache', + 'out', + 'run-id', + 'environment-file', + }); + final assets = + options['assets'] ?? + p.join(File.fromUri(Platform.script).parent.parent.path, 'assets'); + if (options.containsKey('help')) { + stdout.writeln( + 'llamadart-validate --profile [--model ] ' + '[--out ] [--cache ] [--assets ]\n' + '--list lists bundled profiles. --profile-file loads an explicit locked manifest.', + ); + return; + } + if (options.containsKey('list')) { + for (final file in Directory( + p.join(assets, 'profiles'), + ).listSync().whereType()) { + if (file.path.endsWith('.json')) { + stdout.writeln(p.basenameWithoutExtension(file.path)); + } + } + return; + } + requireValidationRuntimeEnvironment(); + final launchDirectory = Directory.current.path; + // Resolve caller paths before anchoring runtime discovery in the bundle. + for (final name in [ + 'assets', + 'profile-file', + 'model', + 'cache', + 'out', + 'environment-file', + ]) { + if (options[name] != null) options[name] = p.absolute(options[name]!); + } + final assetRoot = p.absolute(assets); + final executableRoot = File(Platform.resolvedExecutable).parent.parent; + final bundleManifest = File( + p.join(executableRoot.path, 'bundle-manifest.json'), + ); + final portable = bundleManifest.existsSync(); + final Map provenance; + if (portable) { + provenance = await verifyDesktopValidationBundle(executableRoot); + if (options['environment-file'] case final requested?) { + final supplied = File(requested).readAsStringSync(); + final bundled = File( + p.join(executableRoot.path, 'environment.json'), + ).readAsStringSync(); + if (supplied != bundled) { + throw const FormatException('Use the bundled desktop environment'); + } + } + // LiteRT searches CWD before executable-relative caches. The verified + // bundle must be first, regardless of where the user launched the CLI. + Directory.current = executableRoot; + } else { + if (const bool.fromEnvironment('dart.vm.product')) { + throw const FormatException('Portable validation bundle is missing'); + } + provenance = { + if (options['environment-file'] case final path?) + ...jsonDecode(File(path).readAsStringSync()) as Map, + 'runtime_payload_verified': false, + }; + } + final profileId = options['profile'] ?? 'tiny-gguf-cpu'; + if (!RegExp(r'^[a-z][a-z0-9-]{0,63}$').hasMatch(profileId)) { + throw const FormatException('Invalid profile id'); + } + final profile = ValidationProfile.fromJson( + jsonDecode( + File( + options['profile-file'] ?? + p.join(assetRoot, 'profiles', '$profileId.json'), + ).readAsStringSync(), + ) + as Map, + ); + final runId = + options['run-id'] ?? + 'local-${DateTime.now().toUtc().microsecondsSinceEpoch}'; + final directory = Directory( + options['out'] ?? + p.join(launchDirectory, '.dart_tool', 'validation', 'runs', runId), + ); + final journal = FileValidationJournal(directory); + try { + final prepared = await prepareModel( + profile, + Directory( + options['cache'] ?? + p.join( + launchDirectory, + '.dart_tool', + 'validation', + 'model-cache', + ), + ), + suppliedPath: options['model'], + onProgress: journal.emitPreparation, + ); + final environment = { + ...provenance, + 'os': Platform.operatingSystem, + 'os_version': Platform.operatingSystemVersion, + 'processors': Platform.numberOfProcessors, + 'dart': Platform.version, + 'build_mode': const bool.fromEnvironment('dart.vm.product') + ? 'release' + : 'jit', + }; + final runner = ValidationRunner( + profile: profile, + engine: PublicValidationEngine(), + emit: journal.emit, + ); + final interrupt = ProcessSignal.sigint.watch().listen( + (_) => runner.cancel(), + ); + try { + await runner.run( + prepared.path, + runId: runId, + environment: environment, + preparation: prepared.evidence, + ); + } finally { + await interrupt.cancel(); + } + } catch (error) { + await journal.emit({ + 'type': 'preparation_error', + 'message': redactDiagnostic('$error'), + }); + rethrow; + } finally { + journal.close(); + final report = writeReports(directory); + stdout.writeln( + 'REPORT ${p.join(directory.absolute.path, 'summary.html')}', + ); + if (!report.qualified) exitCode = 1; + } + } catch (error) { + stderr.writeln(redactDiagnostic('$error')); + exitCode = 1; + } +} diff --git a/packages/llamadart_validation/bin/speech.dart b/packages/llamadart_validation/bin/speech.dart new file mode 100644 index 000000000..d4115d7e5 --- /dev/null +++ b/packages/llamadart_validation/bin/speech.dart @@ -0,0 +1,149 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:llamadart/llamadart.dart'; +import 'package:llamadart_validation/io.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:llamadart_validation/src/runtime_environment.dart'; +import 'package:llamadart_validation/src/speech_runner.dart'; +import 'package:path/path.dart' as p; + +/// Runs locked GGUF speech packs; output is explicitly diagnostic qualification. +Future main(List args) async { + Directory? output; + var ownsOutput = false; + try { + final options = parseOptions(args, { + 'pack', + 'backend', + 'model', + 'projector', + 'out', + 'cache', + }); + final pack = options['pack']; + if (!['stt', 'tts'].contains(pack)) { + throw ArgumentError('--pack must be stt or tts'); + } + final backend = GpuBackend.values.byName(options['backend'] ?? 'cpu'); + if (!['cpu', 'metal', 'vulkan', 'cuda', 'opencl'].contains(backend.name)) { + throw ArgumentError('Select an explicit supported native backend'); + } + requireValidationRuntimeEnvironment(); + output = Directory( + options['out'] ?? (throw ArgumentError('--out is required')), + ); + if (output.existsSync()) throw StateError('Output must be a new directory'); + output.createSync(recursive: true); + File(p.join(output.path, '.speech-owner')).createSync(exclusive: true); + ownsOutput = true; + final assets = Directory( + p.join( + File.fromUri(Platform.script).parent.parent.path, + 'assets', + 'speech', + ), + ); + final lock = + jsonDecode(File(p.join(assets.path, '$pack.json')).readAsStringSync()) + as Map; + final prepared = evidence})>{}; + for (final name in ['model', 'projector']) { + final profile = ValidationProfile.fromJson({ + 'schema_version': 1, + 'id': 'speech-$pack-$name', + 'runtime': 'gguf', + 'backend': backend.name, + 'model': lock[name], + 'context_size': 4096, + 'max_tokens': 512, + }); + prepared[name] = await prepareModel( + profile, + Directory( + options['cache'] ?? + p.join( + Directory.current.path, + '.dart_tool', + 'validation', + 'model-cache', + ), + ), + suppliedPath: options[name], + ); + } + final fixture = pack == 'stt' + ? lock['fixture'] as Map + : null; + final audio = fixture == null + ? null + : await File( + p.join(assets.path, fixture['filename'] as String), + ).readAsBytes(); + if (audio != null && + (audio.length != fixture!['bytes'] || + sha256.convert(audio).toString() != fixture['sha256'])) { + throw const FormatException('Speech fixture lock mismatch'); + } + var outputIndex = 0; + final adapter = PublicSpeechValidationAdapter( + model: prepared['model']!.path, + projector: prepared['projector']!.path, + backend: backend, + pack: pack!, + audio: audio, + audioSeconds: audio == null ? null : speechFixtureSeconds(audio), + audioPath: fixture == null + ? null + : p.join(assets.path, fixture['filename'] as String), + reference: fixture?['reference'] as String?, + saveAudio: (bytes) async { + await File( + p.join(output!.path, 'speech-${outputIndex++}.wav'), + ).writeAsBytes(bytes); + }, + ); + final result = await runSpeechValidation( + adapter, + checkBytes: pack == 'stt', + ); + result.addAll({ + 'pack': pack, + 'requested_backend': backend.name, + 'accelerator_execution_verified': false, + 'runtime_observations': adapter.observedRuntime, + 'os': Platform.operatingSystem, + 'dart': Platform.version, + 'model_lock': lock, + 'model_lock_hash': jsonHash(lock), + 'preparation': { + for (final entry in prepared.entries) entry.key: entry.value.evidence, + }, + 'config': { + 'context_size': 4096, + 'stt_max_tokens': 512, + 'tts_seed': 1, + 'tts_max_frames': 384, + 'tts_text': 'Hello from llamadart. The answer is forty two.', + 'language': 'English', + }, + }); + await File( + p.join(output.path, 'speech-results.json'), + ).writeAsString('${const JsonEncoder.withIndent(' ').convert(result)}\n'); + stdout.writeln( + 'Speech report: ${p.join(output.path, 'speech-results.json')}', + ); + if (result['functional_pass'] != true) exitCode = 1; + } catch (error) { + // Do not expose URLs or caller paths through raw native error strings. + stderr.writeln('Speech validation failed: ${error.runtimeType}'); + if (ownsOutput && output != null && output.existsSync()) { + await File(p.join(output.path, 'failure.json')).writeAsString( + jsonEncode({'qualified': false, 'error_type': '${error.runtimeType}'}), + ); + } + exitCode = 1; + } +} diff --git a/packages/llamadart_validation/bin/voice.dart b/packages/llamadart_validation/bin/voice.dart new file mode 100644 index 000000000..f5f63fcd1 --- /dev/null +++ b/packages/llamadart_validation/bin/voice.dart @@ -0,0 +1,154 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:llamadart/llamadart.dart'; +import 'package:llamadart_validation/io.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:llamadart_validation/src/runtime_environment.dart'; +import 'package:llamadart_validation/src/speech_runner.dart'; +import 'package:llamadart_validation/src/voice_runner.dart'; +import 'package:path/path.dart' as p; + +Future main(List args) async { + Directory? ownedOutput; + try { + final options = parseOptions(args, { + 'out', + 'cache', + 'chat-model', + 'chat-profile', + }); + requireValidationRuntimeEnvironment(); + final output = Directory( + options['out'] ?? (throw ArgumentError('--out required')), + ); + if (output.existsSync()) throw StateError('Output must be new'); + output.createSync(recursive: true); + File(p.join(output.path, '.speech-owner')).createSync(exclusive: true); + ownedOutput = output; + final assets = p.join( + File.fromUri(Platform.script).parent.parent.path, + 'assets', + ); + final cache = Directory( + options['cache'] ?? + p.join( + Directory.current.path, + '.dart_tool', + 'validation', + 'model-cache', + ), + ); + final locks = >{}; + final prepared = evidence})>{}; + for (final pack in ['stt', 'tts']) { + final lock = locks[pack] = + jsonDecode( + File(p.join(assets, 'speech', '$pack.json')).readAsStringSync(), + ) + as Map; + for (final name in ['model', 'projector']) { + prepared['$pack-$name'] = await prepareModel( + ValidationProfile.fromJson({ + 'schema_version': 1, + 'id': 'voice-$pack-$name', + 'runtime': 'gguf', + 'backend': 'cpu', + 'model': lock[name], + }), + cache, + ); + } + } + final chatId = options['chat-profile'] ?? 'gemma4-gguf-cpu'; + if (![ + 'gemma4-gguf-cpu', + 'chat-gguf-cpu', + 'gemma4-litert-cpu', + 'qwen35-litert-cpu', + ].contains(chatId)) { + throw ArgumentError('Voice baseline requires a primary CPU chat profile'); + } + final profile = ValidationProfile.fromJson( + jsonDecode( + File(p.join(assets, 'profiles', '$chatId.json')).readAsStringSync(), + ) + as Map, + ); + prepared['chat'] = await prepareModel( + profile, + cache, + suppliedPath: options['chat-model'], + ); + final fixture = locks['stt']!['fixture'] as Map; + final path = p.join(assets, 'speech', fixture['filename'] as String); + final audio = await File(path).readAsBytes(); + if (audio.length != fixture['bytes'] || + sha256.convert(audio).toString() != fixture['sha256']) { + throw const FormatException('Voice fixture hash/size mismatch'); + } + final promptPrefix = 'Summarize this transcript in one short sentence: '; + final result = await runVoiceRoundTrip( + recognizer: PublicSpeechValidationAdapter( + model: prepared['stt-model']!.path, + projector: prepared['stt-projector']!.path, + backend: GpuBackend.cpu, + pack: 'stt', + audio: audio, + audioPath: path, + audioSeconds: speechFixtureSeconds(audio), + reference: fixture['reference'] as String, + saveAudio: (_) async {}, + ), + respond: (transcript) async { + final chat = PublicValidationEngine(); + try { + await chat.load(prepared['chat']!.path, profile); + final response = await chat.generate( + '$promptPrefix$transcript', + profile, + ); + return response['content'] as String; + } finally { + await chat.dispose(); + } + }, + synthesizer: (response) => PublicSpeechValidationAdapter( + model: prepared['tts-model']!.path, + projector: prepared['tts-projector']!.path, + backend: GpuBackend.cpu, + pack: 'tts', + text: response, + saveAudio: (bytes) async { + await File(p.join(output.path, 'response.wav')).writeAsBytes(bytes); + }, + ), + ); + result.addAll({ + 'os': Platform.operatingSystem, + 'requested_backend': 'cpu', + 'model_locks': locks, + 'chat_profile': profile.toJson(), + 'prompt_prefix': promptPrefix, + 'preparation': { + for (final entry in prepared.entries) entry.key: entry.value.evidence, + }, + }); + await File( + p.join(output.path, 'voice-results.json'), + ).writeAsString('${const JsonEncoder.withIndent(' ').convert(result)}\n'); + stdout.writeln( + 'Voice report: ${p.join(output.path, 'voice-results.json')}', + ); + if (result['functional_pass'] != true) exitCode = 1; + } catch (error) { + stderr.writeln('Voice validation failed: ${error.runtimeType}'); + if (ownedOutput != null) { + await File(p.join(ownedOutput.path, 'failure.json')).writeAsString( + jsonEncode({'qualified': false, 'error_type': '${error.runtimeType}'}), + ); + } + exitCode = 1; + } +} diff --git a/packages/llamadart_validation/lib/io.dart b/packages/llamadart_validation/lib/io.dart new file mode 100644 index 000000000..1f1aa9d34 --- /dev/null +++ b/packages/llamadart_validation/lib/io.dart @@ -0,0 +1,247 @@ +/// Native filesystem adapters; never imported by the shared/browser suite. +library; + +import 'dart:convert'; +import 'dart:async'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:http/http.dart' as http; +import 'package:path/path.dart' as p; + +import 'llamadart_validation.dart'; + +/// Verifies model bytes before reuse or load; download timing is separate. +Future<({String path, Map evidence})> prepareModel( + ValidationProfile profile, + Directory cache, { + String? suppliedPath, + Duration timeout = const Duration(minutes: 5), + http.Client? client, + Future Function(Map)? onProgress, +}) async { + profile.requireRunnable(); + final started = Stopwatch()..start(); + final target = suppliedPath == null + ? File(p.join(cache.path, profile.modelHash, profile.filename)) + : File(suppliedPath); + var hit = target.existsSync(); + Future progress(String stage, int bytes, String state) async { + await onProgress?.call({ + 'type': 'preparation_progress', + 'profile_id': profile.id, + 'model_sha256': profile.modelHash, + 'stage': stage, + 'state': state, + 'bytes': bytes, + 'expected_bytes': profile.model['bytes'], + 'elapsed_ms': started.elapsedMilliseconds, + }); + } + + Future valid(File file) async { + final length = await file.length(); + await progress('checksum', 0, 'started'); + if (length != profile.model['bytes']) { + await progress('checksum', length, 'rejected'); + return false; + } + var hashed = 0; + var lastReport = started.elapsedMilliseconds; + final hash = await sha256 + .bind( + file.openRead().asyncMap((chunk) async { + hashed += chunk.length; + if (started.elapsedMilliseconds - lastReport >= 10000) { + await progress('checksum', hashed, 'running'); + lastReport = started.elapsedMilliseconds; + } + return chunk; + }), + ) + .first; + final matches = hash.toString() == profile.modelHash; + await progress('checksum', hashed, matches ? 'verified' : 'rejected'); + return matches; + } + + final checksum = Stopwatch()..start(); + if (hit && !await valid(target)) { + if (suppliedPath != null) { + throw const FormatException('Supplied model hash/size mismatch'); + } + await target.delete(); + hit = false; + } + checksum.stop(); + var downloadMs = 0; + if (!hit) { + if (suppliedPath != null) { + throw const FileSystemException('Supplied model is missing'); + } + await target.parent.create(recursive: true); + final temporary = File( + '${target.path}.${DateTime.now().microsecondsSinceEpoch}.part', + ); + final transport = client ?? http.Client(); + var lastReport = started.elapsedMilliseconds; + var receivedBytes = 0; + var deadlineExpired = false; + final timer = Timer(timeout, () { + deadlineExpired = true; + transport.close(); + }); + IOSink? sink; + final watch = Stopwatch()..start(); + try { + await progress('download', 0, 'started'); + await (() async { + final response = await transport.send( + http.Request('GET', Uri.parse(profile.model['url'] as String)), + ); + if (response.statusCode != 200) { + throw HttpException('Model download HTTP ${response.statusCode}'); + } + sink = temporary.openWrite(); + await for (final chunk in response.stream) { + receivedBytes += chunk.length; + if (receivedBytes > (profile.model['bytes'] as int)) { + throw const FormatException( + 'Model download exceeded locked byte size', + ); + } + sink!.add(chunk); + if (started.elapsedMilliseconds - lastReport >= 10000) { + await progress('download', receivedBytes, 'running'); + lastReport = started.elapsedMilliseconds; + } + } + await sink!.flush(); + await sink!.close(); + sink = null; + })(); + if (deadlineExpired) throw TimeoutException('Model download deadline'); + timer.cancel(); + downloadMs = watch.elapsedMilliseconds; + await progress('download', receivedBytes, 'finished'); + checksum.start(); + if (!await valid(temporary)) { + throw const FormatException('Downloaded model hash/size mismatch'); + } + checksum.stop(); + await temporary.rename(target.path); + } catch (_) { + if (deadlineExpired) { + throw TimeoutException( + 'Model download deadline exceeded after receiving $receivedBytes ' + 'of ${profile.model['bytes']} bytes', + timeout, + ); + } + rethrow; + } finally { + timer.cancel(); + transport.close(); + await sink?.close(); + if (temporary.existsSync()) await temporary.delete(); + } + } + await progress('ready', profile.model['bytes'] as int, 'verified'); + return ( + path: target.absolute.path, + evidence: { + 'sha256': profile.modelHash, + 'bytes': profile.model['bytes'], + 'verified': true, + 'cache_hit': hit, + 'download_timeout_ms': timeout.inMilliseconds, + 'download_ms': downloadMs, + 'checksum_ms': checksum.elapsedMilliseconds, + 'total_ms': started.elapsedMilliseconds, + }, + ); +} + +/// Writes incremental records synchronously enough to survive a native crash. +class FileValidationJournal { + FileValidationJournal(this.directory) { + directory.createSync(recursive: true); + final file = File(p.join(directory.path, 'events.jsonl')); + if (file.existsSync()) { + throw StateError('Run output already exists; use a new run/attempt'); + } + _file = file.openSync(mode: FileMode.write); + } + final Directory directory; + late final RandomAccessFile _file; + + Future emit(Map event) async { + final line = jsonEncode(event); + _file.writeStringSync('$line\n'); + _file.flushSync(); + if (event['type'] == 'manifest') { + File( + p.join(directory.path, 'manifest.json'), + ).writeAsStringSync(line, flush: true); + } + // Complete, bounded records provide a provider-log fallback. + if (utf8.encode(line).length <= 32768) { + stdout.writeln('LLAMADART_VALIDATION $line'); + } + } + + /// Persists diagnostics separately from the manifest-first suite protocol. + Future emitPreparation(Map event) async { + final line = jsonEncode(event); + File( + p.join(directory.path, 'preparation.jsonl'), + ).writeAsStringSync('$line\n', mode: FileMode.append, flush: true); + stdout.writeln('LLAMADART_PREPARATION $line'); + } + + void close() => _file.closeSync(); +} + +/// Recomputes every report format from an existing run journal. +ValidationReport writeReports(Directory directory, {String? nativeLog}) { + final report = ValidationReport.parse( + File(p.join(directory.path, 'events.jsonl')).readAsStringSync(), + nativeLog: nativeLog, + ); + for (final entry in { + 'results.json': const JsonEncoder.withIndent(' ').convert(report.toJson()), + 'junit.xml': report.toJUnit(), + 'samples.csv': report.toCsv(), + 'summary.html': report.toHtml(), + }.entries) { + File(p.join(directory.path, entry.key)).writeAsStringSync(entry.value); + } + return report; +} + +/// Simple strict long-option parser shared by the private CLI entry points. +Map parseOptions(List args, Set allowed) { + final values = {}; + for (var i = 0; i < args.length; i++) { + final arg = args[i]; + if (!arg.startsWith('--')) { + throw FormatException('Expected option, got $arg'); + } + final parts = arg.substring(2).split('='); + final key = parts.first; + if (!allowed.contains(key) || values.containsKey(key)) { + throw FormatException('Unknown/duplicate option --$key'); + } + if (key == 'help' || key == 'list') { + values[key] = 'true'; + } else if (parts.length > 1) { + values[key] = parts.skip(1).join('='); + } else { + if (++i >= args.length || args[i].startsWith('--')) { + throw FormatException('Missing --$key value'); + } + values[key] = args[i]; + } + } + return values; +} diff --git a/packages/llamadart_validation/lib/llamadart_validation.dart b/packages/llamadart_validation/lib/llamadart_validation.dart new file mode 100644 index 000000000..383b79348 --- /dev/null +++ b/packages/llamadart_validation/lib/llamadart_validation.dart @@ -0,0 +1,8 @@ +/// Private, platform-neutral public-package validation and report contracts. +library; + +export 'src/case_catalog.dart'; +export 'src/manifest.dart'; +export 'src/npu_evidence.dart'; +export 'src/runner.dart'; +export 'src/report.dart'; diff --git a/packages/llamadart_validation/lib/npu_io.dart b/packages/llamadart_validation/lib/npu_io.dart new file mode 100644 index 000000000..d981eee55 --- /dev/null +++ b/packages/llamadart_validation/lib/npu_io.dart @@ -0,0 +1,5 @@ +/// Private native-only adapters for the installed Android validation app. +library; + +export 'src/npu_monitor_io.dart'; +export 'src/npu_reference_io.dart'; diff --git a/packages/llamadart_validation/lib/src/case_catalog.dart b/packages/llamadart_validation/lib/src/case_catalog.dart new file mode 100644 index 000000000..60619a1ec --- /dev/null +++ b/packages/llamadart_validation/lib/src/case_catalog.dart @@ -0,0 +1,240 @@ +/// Current reproducible catalog contract; older journals retain their version. +const int validationCatalogVersion = 4; + +/// Versioned core feature selectors. Optional model/media packs are separate. +const validationFeatures = { + 'text': 1, + 'unicode': 1, + 'thinking': 1, + 'history': 1, + 'tools': 1, + 'streaming': 1, + 'batching': 1, + 'lifecycle': 1, + 'guards': 1, + 'performance': 1, +}; + +/// Shared synthetic fixtures, compiled into desktop, mobile and Web runners. +/// Model-specific overrides stay in the hashed profile manifest. +const validationFixtures = >{ + 'unicode': {'input': 'Montréal 👋\n한글 café', 'expected_prefix': ''}, + 'unicode_generation': { + 'prompt': 'Reply with exactly: Montréal 👋', + 'expected': 'Montréal 👋', + 'qualification': + 'strict public output assertion; a failure does not identify its root cause', + }, + 'raw': {'prompt': 'Once upon a time'}, + 'hello': { + 'prompt': 'Reply with one short sentence saying hello.', + 'regex': r'\bhello\b', + }, + 'arithmetic': { + 'prompt': 'What is 2 + 2? Reply with only the number.', + 'regex': r'^4[.!]?$', + }, + 'history': { + 'system': 'Remember the secret code exactly.', + 'user': 'The secret code is cedar17.', + 'assistant': 'I will remember the code.', + 'prompt': 'What is the secret code? Reply with only the code.', + 'expected': 'cedar17', + }, + 'tools': { + 'prompt': 'Call get_weather for Montréal.', + 'tool': { + 'type': 'function', + 'function': { + 'name': 'get_weather', + 'description': 'Return the weather for a city.', + 'parameters': { + 'type': 'object', + 'properties': { + 'city': {'type': 'string'}, + }, + 'required': ['city'], + }, + }, + }, + 'expected_arguments': {'city': 'Montréal'}, + 'response': {'city': 'Montréal', 'temperature_celsius': 17}, + 'modes': ['auto', 'required', 'none'], + }, + 'cancel': { + 'chat_prompt': + 'Write a long story about a fox. Continue for at least 500 words.', + 'max_tokens': 256, + 'deadline_ms': 5000, + }, + 'batching': {'token_threshold': 1, 'byte_threshold': 1}, + 'stop': { + 'prompt': 'Reply with exactly: alpha cedar17 omega', + 'marker': 'cedar17', + }, + 'limit': {'max_tokens': 1, 'expected_native_decode_tokens': 1}, + 'benchmark': { + 'chat_prompt': 'List the numbers from one to twenty in English.', + 'warmups': 1, + 'samples': 3, + }, +}; + +/// Case identity and fixture dependencies. False implementation flags always +/// produce NOT_RUN, never an unsupported-platform exemption. +class ValidationCaseDefinition { + /// Declares a case without hiding unimplemented obligations. + const ValidationCaseDefinition( + this.id, + this.features, + this.fixtures, { + this.implemented = true, + this.version = 1, + }); + + /// Stable identifier in journals and reports. + final String id; + + /// Feature selectors that include this case. + final List features; + + /// Fixture keys included in the case's evidence hash. + final List fixtures; + + /// Whether the shared runner can execute this case. + final bool implemented; + + /// Semantic version of the current case contract. + final int version; + + /// Serializable metadata, independent of model-specific fixture overrides. + Map toJson() => { + 'id': id, + 'version': version, + 'features': features, + 'fixture_ids': fixtures, + 'implemented': implemented, + }; +} + +/// Baseline case definitions; model applicability is resolved by the profile. +const coreValidationCases = [ + ValidationCaseDefinition('C01.load', ['lifecycle'], []), + ValidationCaseDefinition('C02.unicode', ['unicode'], ['unicode']), + ValidationCaseDefinition('C03.raw', ['text'], ['raw']), + ValidationCaseDefinition('C04.hello', ['text'], ['hello']), + ValidationCaseDefinition('C04.arithmetic', ['text'], ['arithmetic']), + ValidationCaseDefinition('C06.history', ['history'], ['history']), + ValidationCaseDefinition( + 'C06.history.public_system_wire', + ['history'], + ['history'], + ), + ValidationCaseDefinition('C06.history.no_system', ['history'], ['history']), + ValidationCaseDefinition('C06.history.combined', ['history'], ['history']), + ValidationCaseDefinition( + 'C08.cancel', + ['streaming', 'lifecycle'], + ['cancel', 'raw', 'hello'], + ), + ValidationCaseDefinition('C09.reload', ['lifecycle'], ['raw', 'hello']), + ValidationCaseDefinition( + 'C10.limit', + ['streaming'], + ['limit', 'raw', 'hello'], + ), + ValidationCaseDefinition( + 'C12.recovery', + ['guards', 'lifecycle'], + ['raw', 'hello'], + ), + ValidationCaseDefinition('B01.warmup', ['performance'], ['benchmark', 'raw']), + ValidationCaseDefinition('B01.1', ['performance'], ['benchmark', 'raw']), + ValidationCaseDefinition('B01.2', ['performance'], ['benchmark', 'raw']), + ValidationCaseDefinition('B01.3', ['performance'], ['benchmark', 'raw']), +]; + +/// Extra obligations added by release or matching focused feature selections. +const extendedValidationCases = [ + ValidationCaseDefinition( + 'C05.thinking', + ['thinking'], + ['arithmetic'], + version: 2, + ), + ValidationCaseDefinition('C07.tools', ['tools'], ['tools'], version: 2), + ValidationCaseDefinition( + 'C10.stop', + ['streaming'], + ['stop', 'hello', 'raw'], + version: 2, + ), + ValidationCaseDefinition( + 'C11.batching', + ['streaming', 'batching'], + ['raw', 'hello', 'batching'], + version: 2, + ), + ValidationCaseDefinition( + 'C12.guards', + ['guards'], + ['hello', 'raw'], + version: 2, + ), + ValidationCaseDefinition( + 'C02.generate', + ['unicode'], + ['unicode_generation'], + version: 2, + ), + ValidationCaseDefinition( + 'C09.reload.second', + ['lifecycle'], + ['raw', 'hello'], + ), +]; + +/// Stable order for the complete selected/omitted inventory. +const validationCaseCatalog = [ + ...coreValidationCases, + ...extendedValidationCases, +]; + +/// Finds a declared case; an unknown ID is a programming error. +ValidationCaseDefinition validationCase( + String id, { + int catalogVersion = validationCatalogVersion, +}) { + if (![1, 2, 3, validationCatalogVersion].contains(catalogVersion)) { + throw const FormatException('Unsupported catalog version'); + } + if (catalogVersion < 4 && + ['C05.thinking', 'C07.tools', 'C02.generate'].contains(id)) { + final current = validationCaseCatalog.singleWhere( + (definition) => definition.id == id, + ); + return ValidationCaseDefinition( + id, + current.features, + current.fixtures, + implemented: false, + ); + } + if (catalogVersion < 3 && (id == 'C10.stop' || id == 'C12.guards')) { + return ValidationCaseDefinition( + id, + [id == 'C10.stop' ? 'streaming' : 'guards'], + [], + implemented: false, + ); + } + if (catalogVersion == 1 && id == 'C11.batching') { + return const ValidationCaseDefinition( + 'C11.batching', + ['streaming'], + ['raw', 'hello'], + implemented: false, + ); + } + return validationCaseCatalog.singleWhere((definition) => definition.id == id); +} diff --git a/packages/llamadart_validation/lib/src/desktop_bundle.dart b/packages/llamadart_validation/lib/src/desktop_bundle.dart new file mode 100644 index 000000000..b626e113a --- /dev/null +++ b/packages/llamadart_validation/lib/src/desktop_bundle.dart @@ -0,0 +1,144 @@ +import 'dart:convert'; +import 'dart:ffi'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +// This private artifact verifier shares the owning package's loader contract. +// Inference still runs exclusively through the public engine adapter. +// ignore: implementation_imports +import 'package:llamadart/src/backends/litert_lm/litert_lm_runtime.dart'; +// ignore: implementation_imports +import 'package:llamadart/src/hook/native_bundle_config.dart'; +import 'package:path/path.dart' as p; + +import 'runtime_environment.dart'; + +/// Checks the portable payload before inference. Mutable run output and model +/// caches may live beside it, but executable/library directories remain sealed. +/// This proves the supplied payload, not accelerator placement or code signing. +Future> verifyDesktopValidationBundle( + Directory directory, { + Map? environment, +}) async { + requireValidationRuntimeEnvironment(environment: environment, portable: true); + final root = Directory(directory.resolveSymbolicLinksSync()); + final manifestFile = File(p.join(root.path, 'bundle-manifest.json')); + final manifestBytes = manifestFile.readAsBytesSync(); + final manifest = jsonDecode(utf8.decode(manifestBytes)) as Map; + if (manifest['schema_version'] != 1 || manifest['target'] != 'desktop') { + throw const FormatException('Expected a desktop validation bundle'); + } + final files = Map.from(manifest['files'] as Map); + if (!files.containsKey('environment.json') || + !files.containsKey( + 'bin/llamadart-validate${Platform.isWindows ? '.exe' : ''}', + )) { + throw const FormatException('Incomplete desktop validation inventory'); + } + if (!files.keys.any((name) => describeNativeLibrary(name).isPrimary)) { + throw const FormatException('Desktop native code asset is missing'); + } + // Use the runtime's own required-file contract so missing cache members + // cannot trigger discovery in an unchecked ancestor directory. + final abi = Abi.current(); + final libraries = liteRtLmRequiredLibrariesForAbi(abi); + if (libraries.isNotEmpty) { + final candidates = liteRtLmCacheDirectoryCandidatesForAbi(abi); + final cache = candidates.singleWhere( + (candidate) => candidate.contains('/'), + ); + final layout = + '.dart_tool/llamadart/litert_lm/${manifest['litert_tag']}/$cache'; + if (manifest['litert_runtime_supported'] != true || + manifest['litert_runtime_layout'] != layout || + libraries.any((name) => !files.containsKey('$layout/$name'))) { + throw const FormatException('Incomplete pinned LiteRT runtime layout'); + } + for (final alternate in candidates.where( + (candidate) => candidate != cache, + )) { + if (Directory( + p.join( + root.path, + '.dart_tool/llamadart/litert_lm', + '${manifest['litert_tag']}', + alternate, + ), + ).existsSync()) { + throw const FormatException( + 'Alternate LiteRT cache can shadow bundled runtime', + ); + } + } + } + for (final entry in files.entries) { + final parts = entry.key.split('/'); + if (p.posix.isAbsolute(entry.key) || + entry.key.contains('\\') || + parts.any((part) => part.isEmpty || part == '.' || part == '..')) { + throw const FormatException('Unsafe desktop bundle member'); + } + var path = root.path; + for (final part in parts) { + path = p.join(path, part); + if (FileSystemEntity.typeSync(path, followLinks: false) == + FileSystemEntityType.link) { + throw const FormatException('Symlink in desktop bundle'); + } + } + final file = File(path); + final expected = entry.value as Map; + if (!file.existsSync() || + await file.length() != expected['bytes'] || + (await sha256.bind(file.openRead()).first).toString() != + expected['sha256']) { + throw FormatException('Desktop bundle checksum mismatch: ${entry.key}'); + } + } + // Ambient sidecars in loader search directories must not change resolution. + for (final name in [ + 'bin', + 'lib', + '.dart_tool/lib', + '.dart_tool/llamadart', + 'Frameworks', + ]) { + final sealed = Directory(p.join(root.path, name)); + if (!sealed.existsSync()) continue; + for (final entity in sealed.listSync(recursive: true, followLinks: false)) { + if (entity is Directory) continue; + final key = p + .relative(entity.path, from: root.path) + .replaceAll('\\', '/'); + if (entity is Link || !files.containsKey(key)) { + throw const FormatException('Unexpected desktop runtime member'); + } + } + } + for (final entity in root.listSync(followLinks: false)) { + final name = p.basename(entity.path); + if (RegExp(r'\.(so(\.[0-9]+)*|dylib|dll)$').hasMatch(name) && + (entity is Link || !files.containsKey(name))) { + throw const FormatException('Unexpected desktop runtime sidecar'); + } + } + final provenance = + jsonDecode(File(p.join(root.path, 'environment.json')).readAsStringSync()) + as Map; + final identity = Map.from(manifest) + ..remove('schema_version') + ..remove('files'); + if (provenance.length != identity.length || + provenance.entries.any((entry) => identity[entry.key] != entry.value)) { + throw const FormatException('Desktop environment disagrees with bundle'); + } + if (provenance['build_os'] != Platform.operatingSystem || + provenance['build_abi'] != abi.toString()) { + throw const FormatException('Desktop bundle targets another OS or ABI'); + } + return { + ...provenance, + 'runtime_payload_verified': true, + 'runtime_bundle_sha256': sha256.convert(manifestBytes).toString(), + }; +} diff --git a/packages/llamadart_validation/lib/src/manifest.dart b/packages/llamadart_validation/lib/src/manifest.dart new file mode 100644 index 000000000..871fe7ac8 --- /dev/null +++ b/packages/llamadart_validation/lib/src/manifest.dart @@ -0,0 +1,458 @@ +import 'dart:convert'; + +import 'package:crypto/crypto.dart'; +import 'package:llamadart/llamadart.dart'; + +import 'case_catalog.dart'; + +/// Canonical encoding for stable experiment identity. +String canonicalJson(Object? value) { + Object? ordered(Object? value) { + if (value is Map) { + final keys = value.keys.cast().toList()..sort(); + return {for (final key in keys) key: ordered(value[key])}; + } + if (value is List) return value.map(ordered).toList(); + return value; + } + + return jsonEncode(ordered(value)); +} + +/// SHA256 identity of a JSON-compatible value. +Object? freezeJson(Object? value) => value is Map + ? Map.unmodifiable( + value.map((key, item) => MapEntry(key as String, freezeJson(item))), + ) + : value is List + ? List.unmodifiable(value.map(freezeJson)) + : value; + +/// SHA256 identity of a JSON-compatible value. +String jsonHash(Object? value) => + sha256.convert(utf8.encode(canonicalJson(value))).toString(); + +/// Locked model and inference selection shared by every host adapter. +class ValidationProfile { + /// Validates an immutable model/profile manifest before use. + ValidationProfile.fromJson(Map input) + : data = freezeJson(jsonDecode(jsonEncode(input))) as Map { + if (data['schema_version'] != 1) { + throw const FormatException('Unsupported validation schema_version'); + } + if (!RegExp(r'^[a-z][a-z0-9-]{0,63}$').hasMatch(id)) { + throw const FormatException('Invalid profile id'); + } + if (!const ['gguf', 'litert'].contains(runtime)) { + throw const FormatException('runtime must be gguf or litert'); + } + final validBackends = runtime == 'litert' + ? ['cpu', 'gpu', 'npu', 'auto'] + : GpuBackend.values.map((value) => value.name).toList(); + if (!validBackends.contains(backend)) { + throw FormatException('Invalid $runtime backend: $backend'); + } + if (!RegExp(r'^[0-9a-f]{64}$').hasMatch(modelHash) || + !RegExp( + r'^[0-9a-f]{40}$', + ).hasMatch(model['revision'] as String? ?? '')) { + throw const FormatException( + 'Model requires SHA256 and immutable revision', + ); + } + final uri = Uri.parse(model['url'] as String); + if (uri.scheme != 'https' || + uri.userInfo.isNotEmpty || + uri.hasQuery || + !uri.path.contains('/${model['revision']}/')) { + throw const FormatException('Use a public, immutable HTTPS model URL'); + } + if (!filename.endsWith(runtime == 'gguf' ? '.gguf' : '.litertlm') || + !RegExp(r'^[a-zA-Z0-9_.-]+$').hasMatch(filename) || + (model['bytes'] as int) <= 0) { + throw const FormatException('Invalid model filename, format or size'); + } + if (!const ['raw', 'chat'].contains(model['kind'])) { + throw const FormatException('Model kind must be raw or chat'); + } + if (contextSize < 128 || + contextSize > 8192 || + maxTokens < 1 || + maxTokens > 512 || + threads < 1 || + threads > 32) { + throw const FormatException('Profile exceeds bounded core limits'); + } + if (!const ['quick', 'focused', 'release'].contains(selection)) { + throw const FormatException( + 'selection must be quick, focused or release', + ); + } + final focus = data['focus_features'] ?? const []; + if (focus is! List || + focus.any((feature) => !validationFeatures.containsKey(feature)) || + focus.toSet().length != focus.length || + (selection != 'focused' && data.containsKey('focus_features')) || + (selection == 'focused') != focus.isNotEmpty) { + throw const FormatException( + 'Only focused selection requires a nonempty unique focus_features list ' + 'of catalog feature IDs', + ); + } + final overrides = data['fixtures']; + if (overrides != null && + (overrides is! Map || + overrides.entries.any( + (entry) => + !validationFixtures.containsKey(entry.key) || + entry.value is! Map || + (entry.value as Map).entries.any( + (field) => + !validationFixtures[entry.key]!.containsKey( + field.key, + ) || + // Core overrides are synthetic text/predicate fixtures; + // tool/media schemas need their own qualified pack. + field.value is! String || + validationFixtures[entry.key]![field.key] is! String, + ), + ))) { + throw const FormatException('Invalid core fixture override'); + } + for (final key in ['enable_thinking', 'history_controls']) { + if (data.containsKey(key) && data[key] is! bool) { + throw FormatException('$key must be a boolean'); + } + } + if (data['history_controls'] == true && + (runtime != 'litert' || backend != 'cpu' || !isChat)) { + throw const FormatException( + 'Optional history controls require a LiteRT CPU chat profile', + ); + } + if (!const [ + 'public_api', + 'native_c_api', + ].contains(data['execution_path'] ?? 'public_api') || + (nativeReference && backend != 'npu')) { + throw const FormatException( + 'Native C API controls require an explicit NPU profile', + ); + } + if (nativeReference && !enableThinking) { + throw const FormatException( + 'Native C API controls currently require thinking enabled', + ); + } + } + + /// JSON definition; callers receive a defensive copy through [toJson]. + final Map data; + + /// Stable profile identifier. + String get id => data['id'] as String; + + /// Model format/runtime identity. + String get runtime => data['runtime'] as String; + + /// Explicit runtime selector. + String get backend => data['backend'] as String; + + /// Immutable model definition. + Map get model => data['model'] as Map; + + /// Expected bytes SHA256. + String get modelHash => model['sha256'] as String; + + /// Safe local model basename. + String get filename => model['filename'] as String; + + /// Whether semantic chat cases apply. + bool get isChat => model['kind'] == 'chat'; + + /// Context token budget. + int get contextSize => data['context_size'] as int? ?? 1024; + + /// Maximum generation length. + int get maxTokens => data['max_tokens'] as int? ?? 32; + + /// Native inference threads. + int get threads => data['threads'] as int? ?? 4; + + /// Catalog selection; release includes explicit uncovered obligations. + String get selection => data['selection'] as String? ?? 'quick'; + + /// Canonical feature set used only by focused runs. + List get focusFeatures => + List.from(data['focus_features'] as List? ?? const [])..sort(); + + /// Separate direct-C-API controls from public-package qualification. + bool get nativeReference => data['execution_path'] == 'native_c_api'; + + /// Explicit model setting; preserves existing NPU and Qwen pilot defaults. + bool get enableThinking => + data['enable_thinking'] as bool? ?? backend == 'npu'; + + /// Adds the same seeded, literal-system, no-system and combined diagnostics. + bool get historyControls => + nativeReference || data['history_controls'] == true; + + List get _quickCaseIds => [ + 'C01.load', + if (!nativeReference) ...['C02.unicode', 'C03.raw'], + if (isChat) ...['C04.hello', 'C04.arithmetic'], + if (isChat) 'C06.history', + if (historyControls && isChat) ...[ + 'C06.history.public_system_wire', + 'C06.history.no_system', + 'C06.history.combined', + ], + if (!nativeReference) 'C08.cancel', + 'C09.reload', + if (!nativeReference) ...['C10.limit', 'C12.recovery'], + 'B01.warmup', + 'B01.1', + 'B01.2', + 'B01.3', + ]; + + /// Original journal-v1 obligations, retained for existing report imports. + List get legacyCaseIds => [ + ..._quickCaseIds, + if (selection == 'release') ...[ + 'C05.thinking', + 'C07.tools', + 'C10.stop', + 'C11.batching', + 'C12.guards', + ], + ]; + + /// Expanded obligations; a focused run adds relevant cases to the quick core. + List get caseIds { + final quick = _quickCaseIds; + return [ + ...quick, + for (final definition in extendedValidationCases) + if (selection == 'release' || + (selection == 'focused' && + definition.features.any(focusFeatures.contains))) + definition.id, + ]; + } + + /// Resolved synthetic fixtures, including explicit model-specific overrides. + Map get fixtures { + final overrides = data['fixtures'] as Map? ?? const {}; + return { + for (final entry in validationFixtures.entries) + entry.key: {...entry.value, ...?overrides[entry.key] as Map?}, + }; + } + + Map _fixturesForVersion(int version) { + final resolved = fixtures; + if (version < 4) { + final tools = resolved['tools'] as Map; + final tool = tools['tool'] as Map; + final function = tool['function'] as Map; + resolved['tools'] = { + ...tools, + 'tool': { + ...tool, + 'function': { + ...function, + 'parameters': { + ...function['parameters'] as Map, + 'additionalProperties': false, + }, + }, + }, + }; + resolved['unicode_generation'] = { + ...resolved['unicode_generation'] as Map, + 'qualification': + (data['fixtures'] + as Map?)?['unicode_generation']?['qualification'] ?? + 'requires an exact-model reference before execution', + }; + } + return resolved; + } + + /// Text read by the runner and included in the replay catalog. + String fixtureText(String fixture, String field) => + (fixtures[fixture] as Map)[field] as String; + + /// The resolved fixtures whose identity is bound to one case record. + Map caseFixtures( + String id, { + int catalogVersion = validationCatalogVersion, + }) => { + for (final key in validationCase( + id, + catalogVersion: catalogVersion, + ).fixtures) + key: _fixturesForVersion(catalogVersion)[key], + }; + + /// Current selected/omitted inventory with reproducible fixture contents. + Map get catalog => + catalogForVersion(validationCatalogVersion); + + /// Reconstructs a supported historical catalog without inventing new evidence. + Map catalogForVersion(int version) { + validationCase('C11.batching', catalogVersion: version); + if (version == 1 && + (focusFeatures.contains('batching') || + (data['fixtures'] as Map?)?.containsKey('batching') == true)) { + throw const FormatException( + 'Batching selection requires catalog version 2', + ); + } + return { + 'version': version, + 'features': { + for (final entry in validationFeatures.entries) + if (version != 1 || entry.key != 'batching') entry.key: entry.value, + }, + 'selection': selection, + 'focus_features': focusFeatures, + 'fixtures': { + for (final entry in _fixturesForVersion(version).entries) + if ((version != 1 || entry.key != 'batching') && + (version >= 3 || entry.key != 'stop')) + entry.key: entry.value, + }, + 'cases': [ + for (final definition in validationCaseCatalog) + { + ...validationCase(definition.id, catalogVersion: version).toJson(), + 'selected': caseIds.contains(definition.id), + if (!caseIds.contains(definition.id)) + 'omission_reason': _omissionReason(definition.id), + }, + ], + }; + } + + String _omissionReason(String id) { + if (id.startsWith('C06.history.') && !historyControls) { + return 'history_controls_disabled'; + } + if (!isChat && (id.startsWith('C04.') || id.startsWith('C06.'))) { + return 'raw_model_has_no_chat_oracle'; + } + if (nativeReference && + const [ + 'C02.unicode', + 'C03.raw', + 'C08.cancel', + 'C10.limit', + 'C12.recovery', + ].contains(id)) { + return 'outside_native_reference_scope'; + } + return 'outside_selected_features'; + } + + /// Accelerator evidence is mandatory for an explicit accelerator selection. + bool get requiresAcceleratorProof => + !['cpu', 'auto', 'blas'].contains(backend); + + /// NPU candidates require the installed Android host to verify the kit first. + void requireRunnable({bool verifiedAndroidNpuHost = false}) { + if (backend == 'npu' && !verifiedAndroidNpuHost) { + throw LlamaUnsupportedException( + 'NPU validation needs installed-app vendor packaging, SoC checks and ' + 'per-generation execution proof. Use validation.dart npu-preflight ' + 'to inspect the locked inputs without downloading or loading a model.', + ); + } + } + + /// Configuration actually passed to the public model loader. + ModelParams get loadParams => ModelParams( + contextSize: contextSize, + gpuLayers: backend == 'cpu' ? 0 : ModelParams.maxGpuLayers, + preferredBackend: runtime == 'gguf' + ? GpuBackend.values.byName(backend) + : GpuBackend.cpu, + liteRtLmBackend: runtime == 'litert' + ? LiteRtLmBackendPreference.values.byName(backend) + : LiteRtLmBackendPreference.auto, + numberOfThreads: threads, + numberOfThreadsBatch: runtime == 'litert' ? 0 : threads, + ); + + /// Requested sampler; NPU retains compiled runtime defaults instead. + GenerationParams get generationParams => GenerationParams( + maxTokens: maxTokens, + temp: 0, + seed: 1, + topK: 40, + topP: 0.9, + minP: 0, + penalty: 1.1, + presencePenalty: 0, + reusePromptPrefix: false, + ); + + /// Expanded options and explicit defaults for replay and cohort matching. + Map get effectiveConfig => { + 'context_size': contextSize, + 'threads': threads, + 'batch_threads': runtime == 'litert' ? 0 : threads, + 'backend': backend, + 'gpu_layers_hint': loadParams.gpuLayers, + 'max_tokens': maxTokens, + 'sampling_application': backend == 'npu' + ? 'runtime_defaults_requested_sampler_not_applied' + : 'requested_sampler', + 'effective_npu_sampler': null, + 'temperature': 0, + 'seed': 1, + 'top_k': 40, + 'top_p': 0.9, + 'min_p': 0, + 'repeat_penalty': 1.1, + 'presence_penalty': 0, + 'stop_sequences': [], + 'enable_thinking': enableThinking, + 'execution_path': nativeReference ? 'native_c_api' : 'public_api', + 'reuse_prompt_prefix': false, + 'stream_batch_tokens': generationParams.streamBatchTokenThreshold, + 'stream_batch_bytes': generationParams.streamBatchByteThreshold, + 'speculative_decoding': false, + 'grammar': null, + 'thinking_budget': null, + 'activation_type': null, + 'prefill_chunk_size': null, + 'dispatch_dir': backend == 'npu' ? 'android.nativeLibraryDir' : null, + 'batch_size': 0, + 'micro_batch_size': 0, + 'flash_attention': 'auto', + 'cache_type_k': 'f16', + 'cache_type_v': 'f16', + 'kv_unified': null, + 'use_mmap': true, + 'use_mlock': false, + 'load_mtp': false, + 'split_mode': 'layer', + 'main_gpu': 0, + 'loras': [], + 'max_parallel_sequences': 1, + 'speculative_rollback_token_max': 0, + 'rope_frequency_base': null, + 'rope_frequency_scale': null, + 'prefer_memory64': null, + 'parallel_file_section_loading': null, + 'runtime_defaults': 'resolved by the recorded runtime artifact', + 'native_log_level': 'info', + 'cache_behavior': 'fresh request; runtime cache unknown', + }; + + /// Detached serialized definition. + Map toJson() => + jsonDecode(jsonEncode(data)) as Map; +} diff --git a/packages/llamadart_validation/lib/src/native_reference_request.dart b/packages/llamadart_validation/lib/src/native_reference_request.dart new file mode 100644 index 000000000..ffd31c133 --- /dev/null +++ b/packages/llamadart_validation/lib/src/native_reference_request.dart @@ -0,0 +1,53 @@ +import 'dart:convert'; + +import 'package:llamadart/llamadart.dart'; + +/// Text-only request for the pinned C API, independent of the public adapter. +/// The system setter accepts JSON content, not a role/content message object. +Map nativeReferenceRequest( + String prompt, { + List? history, +}) { + final messages = + history ?? + [LlamaChatMessage.fromText(role: LlamaChatRole.user, text: prompt)]; + if (messages.isEmpty || + messages.last.role != LlamaChatRole.user || + messages.last.content != prompt) { + throw ArgumentError('Native control requires the final user prompt'); + } + if (messages.any( + (message) => + message.parts.any((part) => part is! LlamaTextContent) || + ![ + LlamaChatRole.system, + LlamaChatRole.user, + LlamaChatRole.assistant, + ].contains(message.role), + )) { + throw UnsupportedError('Native control accepts text chat messages only'); + } + Map encode(LlamaChatMessage message) => { + 'role': message.role.name, + 'content': [ + {'type': 'text', 'text': message.content}, + ], + }; + final seed = messages.take(messages.length - 1); + final system = seed + .where((message) => message.role == LlamaChatRole.system) + .map((message) => message.content.trim()) + .where((text) => text.isNotEmpty) + .join('\n'); + final past = seed + .where((message) => message.role != LlamaChatRole.system) + .map(encode) + .toList(); + return { + 'system_message_json': system.isEmpty ? null : jsonEncode(system), + 'messages_json': past.isEmpty ? null : jsonEncode(past), + 'message_json': jsonEncode(encode(messages.last)), + 'extra_context_json': jsonEncode({'enable_thinking': true}), + 'enable_constrained_decoding': false, + }; +} diff --git a/packages/llamadart_validation/lib/src/npu_evidence.dart b/packages/llamadart_validation/lib/src/npu_evidence.dart new file mode 100644 index 000000000..08736c33d --- /dev/null +++ b/packages/llamadart_validation/lib/src/npu_evidence.dart @@ -0,0 +1,68 @@ +import 'manifest.dart'; + +/// Platform adapter for a hash-verified, same-process native dispatch probe. +abstract interface class NpuExecutionMonitor { + /// Installed, read-only Android native library directory. + String get dispatchDirectory; + + /// Non-secret vendor, device and library identities recorded with every run. + Map get identity; + + /// Schema, sync started/completed/failed, async submitted/failed, in-flight. + List snapshot(); +} + +/// Checks deltas, never confusing async acceptance or prior work with completion. +Map npuGenerationEvidence(List before, List after) { + final shape = + before.length == 7 && + after.length == 7 && + before[0] == 1 && + after[0] == 1 && + before.every((v) => v >= 0) && + after.every((v) => v >= 0); + final monotonic = + shape && + List.generate(5, (i) => after[i + 1] >= before[i + 1]).every((v) => v); + final delta = monotonic + ? [for (var i = 1; i < 6; i++) after[i] - before[i]] + : []; + final verified = + monotonic && + before[6] == 0 && + after[6] == 0 && + delta[0] > 0 && + delta[1] == delta[0] && + delta[2] == 0 && + delta[3] == 0 && + delta[4] == 0; + return { + 'schema_version': 1, + 'before': before, + 'after': after, + 'verified': verified, + 'synchronous_completed': monotonic ? delta[1] : null, + 'placement': verified + ? 'npu_participation_cpu_partitions_unknown' + : 'unverified', + 'reason': verified + ? 'Completed synchronous vendor calls during this generation; CPU partition coverage unknown' + : 'Missing, reset, failed, in-flight or async-only execution proof', + }; +} + +/// Rejects an incompatible installed-app device before native library loading. +void validateNpuDevice(ValidationProfile profile, Map device) { + final target = profile.data['npu_target'] as Map; + final aliases = target['device_soc_models'] as List; + if (device['abi'] != target['abi'] || + device['android_api'] is! int || + (device['android_api'] as int) < (target['minimum_android_api'] as int) || + !aliases.any( + (soc) => '$soc'.toLowerCase() == '${device['soc_model']}'.toLowerCase(), + )) { + throw StateError( + 'NPU device identity does not match ${target['soc']} / ${target['abi']}', + ); + } +} diff --git a/packages/llamadart_validation/lib/src/npu_monitor_io.dart b/packages/llamadart_validation/lib/src/npu_monitor_io.dart new file mode 100644 index 000000000..c6f7cb162 --- /dev/null +++ b/packages/llamadart_validation/lib/src/npu_monitor_io.dart @@ -0,0 +1,40 @@ +import 'dart:ffi'; +import 'dart:io'; + +import 'package:ffi/ffi.dart'; +import 'package:path/path.dart' as p; + +import 'npu_evidence.dart'; + +/// Reads the probe already packaged and hash-checked by the Android host. +class AndroidNpuMonitor implements NpuExecutionMonitor { + AndroidNpuMonitor(this.dispatchDirectory, String library, this.identity) { + if (!Platform.isAndroid || p.basename(library) != library) { + throw UnsupportedError('NPU probe requires an installed Android library'); + } + _snapshot = DynamicLibrary.open(p.join(dispatchDirectory, library)) + .lookupFunction< + Int32 Function(Pointer, Size), + int Function(Pointer, int) + >('LlamadartNpuProbeSnapshot'); + } + + @override + final String dispatchDirectory; + @override + final Map identity; + late final int Function(Pointer, int) _snapshot; + + @override + List snapshot() { + final buffer = calloc(7); + try { + if (_snapshot(buffer, 7) != 0) { + throw StateError('NPU probe snapshot failed'); + } + return List.generate(7, (i) => buffer[i]); + } finally { + calloc.free(buffer); + } + } +} diff --git a/packages/llamadart_validation/lib/src/npu_reference_io.dart b/packages/llamadart_validation/lib/src/npu_reference_io.dart new file mode 100644 index 000000000..256eefa19 --- /dev/null +++ b/packages/llamadart_validation/lib/src/npu_reference_io.dart @@ -0,0 +1,469 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:ffi'; +import 'dart:isolate'; + +import 'package:ffi/ffi.dart'; +import 'package:llamadart/llamadart.dart'; +import 'package:path/path.dart' as p; + +import 'manifest.dart'; +import 'native_reference_request.dart'; +import 'npu_evidence.dart'; +import 'npu_monitor_io.dart'; +import 'runner.dart'; + +/// Direct upstream C API control on a dedicated isolate. It intentionally +/// bypasses LlamaEngine, the public backend, its worker and conversation adapter. +class NativeNpuReferenceEngine implements ValidationEngine { + NativeNpuReferenceEngine(this.monitor, this.probeLibrary); + final NpuExecutionMonitor monitor; + final String probeLibrary; + ReceivePort? _receive; + Isolate? _isolate; + SendPort? _send; + int _id = 0; + final _pending = >>{}; + + Future _start() async { + if (_send != null) return; + final ready = Completer(); + _receive = ReceivePort(); + _receive!.listen((message) { + if (message is SendPort) { + ready.complete(message); + } else if (message is Map) { + final pending = _pending.remove(message['id']); + if (message['error'] != null) { + pending?.completeError(StateError(message['error'] as String)); + } else { + pending?.complete( + Map.from(message['result'] as Map), + ); + } + } + }); + _isolate = await Isolate.spawn(_worker, _receive!.sendPort); + _send = await ready.future; + } + + Future> _call( + String method, [ + Map args = const {}, + ]) async { + await _start(); + final id = _id++; + final pending = Completer>(); + _pending[id] = pending; + _send!.send({'id': id, 'method': method, 'args': args}); + return pending.future; + } + + @override + bool get isWeb => false; + + @override + Future load(String location, ValidationProfile profile) async { + await _call('load', { + 'model': location, + 'directory': monitor.dispatchDirectory, + 'probe': probeLibrary, + 'identity': monitor.identity, + 'context': profile.contextSize, + 'threads': profile.threads, + }); + } + + @override + Future> generate( + String prompt, + ValidationProfile profile, { + bool raw = false, + int? maxTokens, + int? streamBatchTokens, + int? streamBatchBytes, + bool cancelAfterFirst = false, + List? history, + List? stopSequences, + bool? enableThinking, + List? tools, + ToolChoice? toolChoice, + }) { + if (enableThinking != null || tools != null || toolChoice != null) { + throw UnsupportedError( + "Native control does not implement feature overrides", + ); + } + if (stopSequences != null) { + throw LlamaUnsupportedException( + 'Native reference has no public stop filter', + ); + } + if (streamBatchTokens != null || streamBatchBytes != null) { + throw LlamaUnsupportedException( + 'Direct native control has no public worker batching', + ); + } + if (raw || cancelAfterFirst) { + throw UnsupportedError( + 'Native reference only covers blocking text conversations', + ); + } + return _call('generate', { + 'prompt': prompt, + 'max_tokens': maxTokens ?? profile.maxTokens, + 'wire': nativeReferenceRequest(prompt, history: history), + if (history != null) 'messages': history.map((m) => m.toJson()).toList(), + }); + } + + @override + Future> diagnostics() async => { + 'backend_name': 'LiteRT-LM NPU direct C API', + 'backend_selector_only': true, + 'npu_identity': monitor.identity, + }; + @override + Future unload() async { + if (_send != null) await _call('unload'); + } + + @override + Future dispose() async { + await unload(); + _isolate?.kill(); + _receive?.close(); + _send = null; + _isolate = null; + _receive = null; + } + + @override + void cancel() {} // Cancellation is tested on the public path, not this control. + @override + Future> tokenize(String text) => + throw UnsupportedError('Not a native control case'); + @override + Future detokenize(List tokens) => + throw UnsupportedError('Not a native control case'); +} + +void _worker(SendPort parent) { + final receive = ReceivePort(); + _Capi? engine; + parent.send(receive.sendPort); + receive.listen((dynamic raw) { + final message = raw as Map; + try { + final args = Map.from(message['args'] as Map); + Map result = {}; + switch (message['method']) { + case 'load': + engine?.close(); + engine = null; + engine = _Capi(args); + case 'generate': + result = engine!.generate( + args['prompt'] as String, + args['max_tokens'] as int, + Map.from(args['wire'] as Map), + ); + if (args['messages'] != null) result['messages'] = args['messages']; + case 'unload': + engine?.close(); + engine = null; + default: + throw StateError('Unknown native reference command'); + } + parent.send({'id': message['id'], 'result': result}); + } catch (error) { + parent.send({'id': message['id'], 'error': redactDiagnostic('$error')}); + } + }); +} + +// These signatures come directly from the pinned upstream c/engine.h and +// c/conversation.h. No generated/public Dart backend bindings are imported. +class _Capi { + _Capi(Map args) + : library = DynamicLibrary.open( + p.join(args['directory'] as String, 'libLiteRtLm.so'), + ), + monitor = AndroidNpuMonitor( + args['directory'] as String, + args['probe'] as String, + Map.from(args['identity'] as Map), + ) { + final model = (args['model'] as String).toNativeUtf8(); + final backend = 'npu'.toNativeUtf8(); + final directory = (args['directory'] as String).toNativeUtf8(); + Pointer settings = nullptr; + try { + settings = library + .lookupFunction< + Pointer Function( + Pointer, + Pointer, + Pointer, + Pointer, + ), + Pointer Function( + Pointer, + Pointer, + Pointer, + Pointer, + ) + >( + 'litert_lm_engine_settings_create', + )(model, backend, nullptr, nullptr); + _required(settings, 'engine settings'); + setInt( + 'engine_settings_set_max_num_tokens', + settings, + args['context'] as int, + ); + setInt( + 'engine_settings_set_num_threads', + settings, + args['threads'] as int, + ); + library.lookupFunction< + Void Function(Pointer, Pointer), + void Function(Pointer, Pointer) + >('litert_lm_engine_settings_set_litert_dispatch_lib_dir')( + settings, + directory, + ); + drop('engine_settings_enable_benchmark', settings); + handle = createFrom('engine_create', settings); + _required(handle, 'engine'); + } finally { + if (settings != nullptr) drop('engine_settings_delete', settings); + calloc.free(model); + calloc.free(backend); + calloc.free(directory); + } + } + final DynamicLibrary library; + final AndroidNpuMonitor monitor; + Pointer handle = nullptr; + String _name(String suffix) => 'litert_lm_$suffix'; + Pointer create(String name) => library + .lookupFunction Function(), Pointer Function()>( + _name(name), + )(); + Pointer createFrom(String name, Pointer value) => + library.lookupFunction< + Pointer Function(Pointer), + Pointer Function(Pointer) + >(_name(name))(value); + void drop(String name, Pointer value) => + library.lookupFunction< + Void Function(Pointer), + void Function(Pointer) + >(_name(name))(value); + void setInt(String name, Pointer value, int number) => + library.lookupFunction< + Void Function(Pointer, Int32), + void Function(Pointer, int) + >(_name(name))(value, number); + void setPointer(String name, Pointer value, Pointer other) => + library.lookupFunction< + Void Function(Pointer, Pointer), + void Function(Pointer, Pointer) + >(_name(name))(value, other); + void setJson(String name, Pointer value, String? json) { + if (json == null) return; + final text = json.toNativeUtf8(); + try { + library.lookupFunction< + Void Function(Pointer, Pointer), + void Function(Pointer, Pointer) + >(_name(name))(value, text); + } finally { + calloc.free(text); + } + } + + double metric(String name, Pointer info) => + library.lookupFunction< + Double Function(Pointer), + double Function(Pointer) + >(_name(name))(info); + static void _required(Pointer value, String kind) { + if (value == nullptr) { + throw StateError( + 'Direct C API failed to create $kind; inspect native log', + ); + } + } + + void close() { + if (handle != nullptr) { + drop('engine_delete', handle); + handle = nullptr; + } + } + + Map generate( + String prompt, + int maxTokens, + Map wire, + ) { + Pointer session = nullptr; + Pointer config = nullptr; + Pointer optionalArgs = nullptr; + Pointer conversation = nullptr; + Pointer response = nullptr; + Pointer benchmark = nullptr; + final message = (wire['message_json'] as String).toNativeUtf8(); + try { + session = create('session_config_create'); + config = create('conversation_config_create'); + _required(session, 'session config'); + _required(config, 'conversation config'); + // The public NPU runtime intentionally leaves session sampling at the + // compiled model/runtime defaults. Apply the same per-request output cap. + optionalArgs = create('conversation_optional_args_create'); + _required(optionalArgs, 'optional args'); + setInt( + 'conversation_optional_args_set_max_output_tokens', + optionalArgs, + maxTokens, + ); + setPointer('conversation_config_set_session_config', config, session); + setJson( + 'conversation_config_set_system_message', + config, + wire['system_message_json'] as String?, + ); + setJson( + 'conversation_config_set_messages', + config, + wire['messages_json'] as String?, + ); + setJson( + 'conversation_config_set_extra_context', + config, + wire['extra_context_json'] as String, + ); + library.lookupFunction< + Void Function(Pointer, Bool), + void Function(Pointer, bool) + >('litert_lm_conversation_config_set_enable_constrained_decoding')( + config, + wire['enable_constrained_decoding'] as bool, + ); + final before = monitor.snapshot(); + final setupWatch = Stopwatch()..start(); + conversation = library + .lookupFunction< + Pointer Function(Pointer, Pointer), + Pointer Function(Pointer, Pointer) + >('litert_lm_conversation_create')(handle, config); + _required(conversation, 'conversation'); + setupWatch.stop(); + final afterSetup = monitor.snapshot(); + final watch = Stopwatch()..start(); + response = library + .lookupFunction< + Pointer Function( + Pointer, + Pointer, + Pointer, + Pointer, + ), + Pointer Function( + Pointer, + Pointer, + Pointer, + Pointer, + ) + >( + 'litert_lm_conversation_send_message', + )(conversation, message, nullptr, optionalArgs); + watch.stop(); + final after = monitor.snapshot(); + _required(response, 'response'); + final responseText = library + .lookupFunction< + Pointer Function(Pointer), + Pointer Function(Pointer) + >('litert_lm_json_response_get_string')(response); + _required(responseText.cast(), 'response text'); + final text = responseText.toDartString(); + if (text.length > 65536) { + throw StateError('Native reference exceeded output bound'); + } + final json = jsonDecode(text) as Map; + final content = json['content']; + final visible = content is String + ? content + : (content as List) + .whereType() + .where((part) => part['type'] == 'text') + .map((part) => part['text']) + .join(); + benchmark = createFrom('conversation_get_benchmark_info', conversation); + _required(benchmark, 'benchmark info'); + final turns = library + .lookupFunction< + Int32 Function(Pointer), + int Function(Pointer) + >('litert_lm_benchmark_info_get_num_decode_turns')(benchmark); + if (turns != 1) { + throw StateError('Expected one native reference decode turn'); + } + final tokens = library + .lookupFunction< + Int32 Function(Pointer, Int32), + int Function(Pointer, int) + >('litert_lm_benchmark_info_get_decode_token_count_at')(benchmark, 0); + final tps = library + .lookupFunction< + Double Function(Pointer, Int32), + double Function(Pointer, int) + >( + 'litert_lm_benchmark_info_get_decode_tokens_per_sec_at', + )(benchmark, 0); + final wall = watch.elapsedMicroseconds / 1000; + return { + 'prompt': prompt, + 'content': visible, + 'thinking': '', + 'native_response': json, + 'native_request': wire, + 'native_conversation_create_ms': setupWatch.elapsedMicroseconds / 1000, + 'npu_after_conversation_create': afterSetup, + 'max_tokens': maxTokens, + 'execution_path': 'native_c_api', + 'npu_execution': npuGenerationEvidence(before, after), + 'metrics': { + 'wall_ms': wall, + 'ttfa_ms': null, + 'native_ttft_ms': + metric('benchmark_info_get_time_to_first_token', benchmark) * + 1000, + 'native_decode_tokens': tokens, + 'native_decode_tps': tps.isFinite && tps > 0 ? tps : null, + 'native_decode_ms': tps.isFinite && tps > 0 + ? tokens * 1000 / tps + : null, + 'estimated_wall_tps': wall > 0 ? tokens * 1000 / wall : null, + 'token_count_source': 'native C API decode counter', + 'native_timing_source': 'native C API per-conversation benchmark', + }, + }; + } finally { + if (benchmark != nullptr) drop('benchmark_info_delete', benchmark); + if (response != nullptr) drop('json_response_delete', response); + if (conversation != nullptr) drop('conversation_delete', conversation); + if (config != nullptr) drop('conversation_config_delete', config); + if (session != nullptr) drop('session_config_delete', session); + if (optionalArgs != nullptr) { + drop('conversation_optional_args_delete', optionalArgs); + } + calloc.free(message); + } + } +} diff --git a/packages/llamadart_validation/lib/src/placement.dart b/packages/llamadart_validation/lib/src/placement.dart new file mode 100644 index 000000000..751beb711 --- /dev/null +++ b/packages/llamadart_validation/lib/src/placement.dart @@ -0,0 +1,339 @@ +import 'dart:convert'; + +import 'package:crypto/crypto.dart'; + +import 'manifest.dart'; +import 'npu_evidence.dart'; + +/// Conservative GGUF native-log evidence. A selector or device inventory alone +/// is insufficient: every expected load needs positive offload and compute allocation. +Map inspectPlacement( + Map manifest, + List> cases, + String? log, +) { + final profile = manifest['profile'] is Map + ? manifest['profile'] as Map + : const {}; + final backend = profile['backend']; + // The selector determines the obligation; a journal flag cannot waive it. + final required = !['cpu', 'auto', 'blas'].contains(backend); + if (!required) { + return {'required': false, 'verified': true, 'reason': 'not_required'}; + } + final result = { + 'required': true, + 'verified': false, + 'reason': 'native placement evidence unavailable', + if (log != null) 'log_sha256': sha256.convert(utf8.encode(log)).toString(), + }; + // Derive obligations from the executable profile, never from a producer's + // case inventory or the evidence records that happen to be present. + final List selected; + try { + final parsed = ValidationProfile.fromJson( + Map.from(profile), + ); + final schema = manifest['schema_version']; + if (schema != 1 && schema != 2 || + schema == 1 && parsed.selection == 'focused') { + return result; + } + final catalog = manifest['catalog'] as Map?; + final version = catalog?['version'] ?? 1; + selected = (schema == 1 ? parsed.legacyCaseIds : parsed.caseIds) + .where( + (id) => + !['C10.stop', 'C12.guards'].contains(id) || + (schema == 2 && + [3, 4].contains(version) && + !parsed.nativeReference), + ) + .toList(); + } catch (_) { + return result; + } + if (profile['runtime'] == 'litert' && backend == 'npu') { + return _inspectNpu(manifest, cases, selected, result); + } + if (profile['runtime'] != 'gguf' || + log == null || + !['cuda', 'metal', 'vulkan'].contains(backend)) { + return result; + } + if (backend == 'vulkan' && + RegExp( + r'llvmpipe|lavapipe|swiftshader|microsoft basic render driver|software rasterizer', + caseSensitive: false, + ).hasMatch(log)) { + return { + ...result, + 'reason': + 'software Vulkan device present; physical accelerator execution is unproven', + }; + } + final expectedIds = selected + .where( + (id) => [ + 'C01.load', + 'C09.reload', + 'C12.recovery', + 'C09.reload.second', + 'C12.guards', + ].contains(id), + ) + .toList(); + final loads = cases + .where((c) => expectedIds.contains(c['case_id']) && c['status'] == 'PASS') + .toList(); + final expectedLoads = expectedIds.length; + if (expectedLoads == 0 || + expectedIds.any( + (id) => loads.where((c) => c['case_id'] == id).length != 1, + ) || + loads.any((c) { + final name = (c['diagnostics'] as Map?)?['backend_name']; + return name is! String || + !name.toLowerCase().contains(backend as String); + })) { + return result; + } + final offloads = RegExp( + r'load_tensors: offloaded (\d+)/(\d+) layers to GPU', + ).allMatches(log).toList(); + final prefix = { + 'cuda': r'CUDA\d+', + 'metal': r'MTL\d+|Metal', + 'vulkan': r'Vulkan\d+', + }[backend]!; + final buffers = RegExp( + '(?:$prefix) compute buffer size =\\s*([0-9.]+) (?:MiB|MB)', + ).allMatches(log).toList(); + var deviceIdentityVerified = true; + if (backend == 'vulkan') { + final devices = >{}; + void addDevice(String index, String name, {bool discovery = false}) { + // Discovery includes capability columns and sometimes a vendor suffix; + // selected-model records contain just the same physical device name. + final unsafe = RegExp( + r'\b(cpu|software|virtual)\b', + caseSensitive: false, + ).hasMatch(name); + final identity = !discovery || unsafe + ? name.trim() + : name + .split('|') + .first + .trim() + .replaceFirst( + RegExp(r'\s+\((?:NVIDIA|AMD|Intel|Apple|Qualcomm)\)$'), + '', + ); + devices.putIfAbsent(index, () => {}).add(identity); + } + + for (final match in RegExp( + r'ggml_vulkan:\s*(\d+)\s*=\s*([^\r\n]+)', + ).allMatches(log)) { + addDevice(match[1]!, match[2]!, discovery: true); + } + for (final match in RegExp( + r'^llama_prepare_model_devices: using device Vulkan(\d+) ' + r'\(((?:[^()\r\n]|\([^()\r\n]*\))+)\)' + r'(?: \([0-9a-fA-F:.]+\))? - [0-9]+(?:\.[0-9]+)? MiB free\r?$', + multiLine: true, + ).allMatches(log)) { + addDevice(match[1]!, match[2]!); + } + final selectedDevices = RegExp( + r'Vulkan(\d+) compute buffer size', + ).allMatches(log).map((match) => match[1]!).toSet(); + final hardwareVendor = RegExp( + r'\b(NVIDIA|AMD|Radeon|Intel|Apple|Adreno|Mali|PowerVR|Immortalis|Xclipse|Qualcomm)\b', + caseSensitive: false, + ); + final softwareDevice = RegExp( + r'\b(cpu|software|virtual)\b', + caseSensitive: false, + ); + deviceIdentityVerified = + selectedDevices.isNotEmpty && + selectedDevices.every((id) { + final names = devices[id]; + return names != null && + names.length == 1 && + hardwareVendor.hasMatch(names.single) && + !softwareDevice.hasMatch(names.single); + }); + } + final positive = + deviceIdentityVerified && + offloads.length == expectedLoads && + offloads.every( + (m) => int.parse(m[1]!) > 0 && int.parse(m[1]!) <= int.parse(m[2]!), + ) && + buffers.length == expectedLoads && + buffers.every((m) => (double.tryParse(m[1]!) ?? 0) > 0); + return { + ...result, + 'verified': positive, + 'reason': positive + ? 'matching runtime diagnostics, positive native tensor offload and compute buffers for all $expectedLoads loads' + : !deviceIdentityVerified + ? 'Vulkan compute device identity is missing, contradictory or unrecognized as physical hardware' + : 'native placement records incomplete or contradictory', + 'expected_loads': expectedLoads, + 'offload_records': offloads.map((m) => m[0]).toList(), + 'compute_records': buffers.map((m) => m[0]).toList(), + }; +} + +Map _inspectNpu( + Map manifest, + List> cases, + List selected, + Map result, +) { + final profile = manifest['profile'] as Map; + final target = profile['npu_target'] as Map? ?? {}; + final identity = (manifest['preparation'] as Map?)?['npu'] as Map? ?? {}; + final device = identity['device'] as Map? ?? {}; + final kit = identity['kit'] as Map? ?? {}; + final libraries = kit['libraries'] as Map? ?? {}; + final locks = target['libraries'] as Map? ?? {}; + final aliases = target['device_soc_models'] as List? ?? []; + final identitiesMatch = + locks.isNotEmpty && + kit['target'] == target['soc'] && + kit['runtime_tag'] == (manifest['environment'] as Map?)?['litert_tag'] && + kit['schema_version'] == 1 && + kit['litert_revision'] == '9fe5be45564c868408e6514c8aabb83e211a0911' && + kit['dispatch_header_sha256'] == + '11dd4d98bd084157ac987b1ee1951f3f96e2b3ca6b51a27c10e645686bf0e3ee' && + device['verified'] == true && + device['abi'] == target['abi'] && + device['android_api'] is int && + target['minimum_android_api'] is int && + (device['android_api'] as int) >= + (target['minimum_android_api'] as int) && + aliases.any( + (soc) => '$soc'.toLowerCase() == '${device['soc_model']}'.toLowerCase(), + ) && + libraries.length == locks.length && + locks.entries.every((entry) { + final value = libraries[entry.key]; + final lock = entry.value as Map; + return value is Map && + value['bytes'] is int && + (value['bytes'] as int) > 0 && + RegExp(r'^[a-f0-9]{64}$').hasMatch('${value['sha256']}') && + (lock['sha256'] == null || value['sha256'] == lock['sha256']); + }); + final expected = selected.where( + (id) => [ + 'C03.raw', + 'C04.hello', + 'C04.arithmetic', + 'C06.history', + 'C06.history.public_system_wire', + 'C06.history.no_system', + 'C06.history.combined', + 'C08.cancel', + 'C09.reload', + 'C10.limit', + 'C12.recovery', + 'B01.warmup', + 'B01.1', + 'B01.2', + 'B01.3', + 'C09.reload.second', + 'C10.stop', + 'C12.guards', + if ((manifest['catalog'] as Map?)?['version'] == 4) ...[ + 'C02.generate', + 'C05.thinking', + 'C07.tools', + ], + ].contains(id), + ); + final records = {for (final record in cases) record['case_id']: record}; + var valid = identitiesMatch; + var proven = 0; + List? previous; + for (final id in expected) { + final record = records[id]; + if (record == null) { + valid = false; + continue; + } + final generations = switch (id) { + 'C08.cancel' => [ + record['uncancelled_control'], + record, + record['recovery'], + ], + 'C10.stop' => [record['control'], record['stopped'], record['recovery']], + 'C12.guards' => [record['recovery']], + 'C05.thinking' => [ + for (var i = 0; i < 2; i++) + record['trials'] is List && (record['trials'] as List).length > i + ? (record['trials'] as List)[i] + : null, + ], + 'C07.tools' => [ + for (var i = 0; i < 3; i++) ...[ + record['trials'] is List && (record['trials'] as List).length > i + ? (record['trials'] as List)[i] + : null, + if (i < 2) + record['trials'] is List && + (record['trials'] as List).length > i && + (record['trials'] as List)[i] is Map + ? ((record['trials'] as List)[i] as Map)['tool_result_followup'] + : null, + ], + record['recovery'], + ], + _ => [record], + }; + for (final generation in generations) { + final evidence = generation is Map ? generation['npu_execution'] : null; + final before = evidence is Map ? evidence['before'] : null; + final after = evidence is Map ? evidence['after'] : null; + if (before is! List || + after is! List || + before.any((v) => v is! int) || + after.any((v) => v is! int)) { + valid = false; + continue; + } + final start = before.cast(); + final end = after.cast(); + final checked = npuGenerationEvidence(start, end); + if (checked['verified'] != true || + (previous != null && + (start.length != 7 || + List.generate( + 5, + (i) => start[i + 1] < previous![i + 1], + ).any((v) => v)))) { + valid = false; + } else { + proven++; + } + if (end.length == 7) previous = end; + } + } + return { + ...result, + 'verified': valid, + 'reason': valid + ? 'Per-generation completed vendor calls; CPU partitions remain unknown' + : 'Missing, inconsistent or failed per-generation NPU evidence/device identity', + 'placement': valid + ? 'npu_participation_cpu_partitions_unknown' + : 'unverified', + 'proven_generations': proven, + 'native_reference_only': profile['execution_path'] == 'native_c_api', + }; +} diff --git a/packages/llamadart_validation/lib/src/report.dart b/packages/llamadart_validation/lib/src/report.dart new file mode 100644 index 000000000..26eb8d646 --- /dev/null +++ b/packages/llamadart_validation/lib/src/report.dart @@ -0,0 +1,462 @@ +import 'dart:convert'; +import 'dart:math' as math; + +import 'case_catalog.dart'; +import 'manifest.dart'; +import 'placement.dart'; + +/// Validates the event journal and produces all report formats from one source. +class ValidationReport { + ValidationReport._( + this.manifest, + this.cases, + this.problems, + this.finished, + this.cleanupPassed, + this.placement, + ); + + final Map manifest; + final List> cases; + final List problems; + final bool finished; + final bool cleanupPassed; + final Map placement; + + /// Truncated, duplicate and missing records cannot become passing reports. + factory ValidationReport.parse(String jsonl, {String? nativeLog}) { + final problems = []; + final events = >[]; + var lineNumber = 0; + for (final line in const LineSplitter().convert(jsonl)) { + lineNumber++; + if (line.trim().isEmpty) continue; + try { + events.add(jsonDecode(line) as Map); + } catch (_) { + problems.add('Invalid JSON event at line $lineNumber'); + } + } + final manifests = events.where((e) => e['type'] == 'manifest').toList(); + if (manifests.length != 1) problems.add('Expected exactly one manifest'); + final manifest = manifests.isEmpty ? {} : manifests.first; + final legacy = manifest['schema_version'] == 1; + final declaredCatalog = manifest['catalog']; + final catalogVersion = declaredCatalog is Map + ? declaredCatalog['version'] + : null; + if (!const [1, 2].contains(manifest['schema_version'])) { + problems.add('Unsupported result schema'); + } + if (legacy && + (manifest.containsKey('catalog') || + manifest.containsKey('catalog_hash') || + events.any( + (event) => + event.containsKey('case_version') || + event.containsKey('fixture_hash'), + ))) { + problems.add('Catalog metadata requires result schema 2'); + } + if (manifest['profile_hash'] != jsonHash(manifest['profile'])) { + problems.add('Profile hash does not match the manifest'); + } + if (manifest['config_hash'] != jsonHash(manifest['effective_config'])) { + problems.add('Effective configuration hash does not match'); + } + ValidationProfile? profile; + try { + profile = ValidationProfile.fromJson( + manifest['profile'] as Map, + ); + } catch (_) { + problems.add('Invalid validation profile'); + } + final inventory = manifest['case_ids']; + final declared = inventory is List + ? inventory.whereType().toList() + : []; + if (inventory is! List || declared.length != inventory.length) { + problems.add('Malformed case inventory'); + } + if (profile != null) { + if (legacy && profile.selection == 'focused') { + problems.add('Focused selection requires result schema 2'); + } + if (!legacy) { + if (manifest['catalog_hash'] != jsonHash(manifest['catalog'])) { + problems.add('Catalog hash does not match the manifest'); + } + try { + if (catalogVersion is! int || + canonicalJson(manifest['catalog']) != + canonicalJson(profile.catalogForVersion(catalogVersion))) { + problems.add('Catalog does not match the executable profile'); + } + } on FormatException catch (error) { + problems.add(error.message.toString()); + } + } + if (canonicalJson(inventory) != + canonicalJson(legacy ? profile.legacyCaseIds : profile.caseIds)) { + problems.add('Case inventory does not match the profile'); + } + if (canonicalJson(manifest['effective_config']) != + canonicalJson(profile.effectiveConfig)) { + problems.add('Effective configuration does not match the profile'); + } + if (manifest['accelerator_evidence_required'] != + profile.requiresAcceleratorProof) { + problems.add( + 'Accelerator evidence requirement does not match the profile', + ); + } + } + final expected = profile == null + ? declared + : legacy + ? profile.legacyCaseIds + : profile.caseIds; + var sequence = 0; + for (var index = 0; index < events.length; index++) { + final event = events[index]; + if (event['type'] == 'manifest') { + if (index != 0) problems.add('Manifest must be first'); + continue; + } + if (![ + 'case_start', + 'case', + 'cleanup', + 'run_end', + ].contains(event['type'])) { + problems.add('Unknown event type'); + } + if (event['sequence'] != sequence++) { + problems.add('Event sequence mismatch'); + } + if (event['type'] == 'run_end' && index != events.length - 1) { + problems.add('Records follow run end'); + } + } + if (expected.isEmpty || expected.toSet().length != expected.length) { + problems.add('Invalid mandatory case inventory'); + } + final records = >{}; + for (final event in events.where((e) => e['type'] == 'case')) { + final id = event['case_id']; + if (id is! String || !expected.contains(id)) { + problems.add('Unexpected case record: $id'); + continue; + } + if (records.containsKey(id)) { + problems.add('Duplicate terminal record: $id'); + } + if (!legacy && + profile != null && + const [1, 2, 3, 4].contains(catalogVersion)) { + if (event['case_version'] != + validationCase( + id, + catalogVersion: catalogVersion as int, + ).version || + event['fixture_hash'] != + jsonHash( + profile.caseFixtures(id, catalogVersion: catalogVersion), + )) { + problems.add('Case version or fixture identity mismatch: $id'); + } + } + if (!legacy && + const [1, 2, 3, 4].contains(catalogVersion) && + !validationCase( + id, + catalogVersion: catalogVersion as int, + ).implemented && + event['status'] != 'NOT_RUN') { + problems.add( + 'Unimplemented catalog case cannot claim an executed result: $id', + ); + } + if (!const [ + 'PASS', + 'FAIL', + 'ERROR', + 'NOT_RUN', + 'UNSUPPORTED', + ].contains(event['status'])) { + problems.add('Invalid terminal status: $id'); + } + records.putIfAbsent(id, () => event); + } + for (final id in expected) { + if (!records.containsKey(id)) { + problems.add('Missing mandatory case: $id'); + records[id] = { + 'type': 'case', + 'case_id': id, + 'status': 'NOT_RUN', + 'reason': 'missing terminal record after crash/interruption', + }; + } + } + final endings = events.where((e) => e['type'] == 'run_end').toList(); + final finished = + endings.length == 1 && endings.single['cancelled'] == false; + if (!finished) problems.add('Run did not complete normally'); + final cleanups = events.where((e) => e['type'] == 'cleanup').toList(); + final cleanup = cleanups.length == 1 && cleanups.single['status'] == 'PASS'; + if (!cleanup) problems.add('Engine cleanup did not complete'); + return ValidationReport._( + manifest, + [for (final id in expected) records[id]!], + problems, + finished, + cleanup, + inspectPlacement(manifest, [ + for (final id in expected) records[id]!, + ], nativeLog), + ); + } + + /// True only for complete mandatory functional obligations. + bool get assertionsPassed => + problems.isEmpty && + cases.isNotEmpty && + // The current catalog has no allowed unsupported exemptions. A producer + // cannot grant itself one through an expected_unsupported event field. + cases.every((e) => e['status'] == 'PASS'); + + /// Public selector values alone never qualify an accelerator. + bool get acceleratorVerified => placement['verified'] == true; + + /// Independent findings; missing evidence does not diagnose incompatibility. + List get qualificationGaps => [ + if (problems.isNotEmpty || !finished || !cleanupPassed) + 'journal_or_lifecycle_incomplete', + if (cases.any((c) => c['status'] == 'FAIL')) 'assertion_failure', + if (cases.any((c) => c['status'] == 'ERROR')) 'execution_error', + if (cases.any((c) => c['status'] == 'NOT_RUN')) 'cases_not_run', + if (cases.any((c) => c['status'] == 'UNSUPPORTED')) + 'unsupported_case_unqualified', + if (!acceleratorVerified) 'accelerator_evidence_missing', + if (provenanceProblems.isNotEmpty) 'provenance_incomplete', + ]; + + /// Missing or uncommitted build identity preserves results but cannot qualify. + List get provenanceProblems { + final environment = manifest['environment'] is Map + ? manifest['environment'] as Map + : const {}; + final runtime = manifest['profile'] is Map + ? (manifest['profile'] as Map)['runtime'] + : null; + final tag = environment[runtime == 'litert' ? 'litert_tag' : 'native_tag']; + final model = (manifest['profile'] as Map?)?['model'] as Map? ?? {}; + final preparation = manifest['preparation'] is Map + ? manifest['preparation'] as Map + : const {}; + final desktop = + environment['web'] != true && + ['macos', 'linux', 'windows'].contains( + '${environment['os'] ?? environment['platform']}'.toLowerCase(), + ); + return [ + if (preparation['verified'] != true || + preparation['sha256'] != model['sha256'] || + preparation['bytes'] != model['bytes']) + 'Verified model hash and byte size do not match the profile lock', + if (desktop && + (environment['runtime_payload_verified'] != true || + !RegExp( + r'^[a-f0-9]{64}$', + ).hasMatch('${environment['runtime_bundle_sha256']}'))) + 'Desktop runtime payload was not verified from a portable bundle', + if (!RegExp( + r'^[a-f0-9]{40}$', + ).hasMatch('${environment['source_commit']}')) + 'Source commit is missing or unknown', + if (environment['source_dirty'] != false) + 'Build source is dirty or its cleanliness is unknown', + if (!RegExp(r'^[a-f0-9]{64}$').hasMatch('${environment['hook_sha256']}')) + 'Native hook identity is missing or unknown', + if (tag is! String || tag.isEmpty || tag == 'unknown') + 'Runtime artifact pin is missing or unknown', + if (environment['web'] == true && + (environment['bridge_tag'] == null || + environment['bridge_tag'] == 'unknown')) + 'Web runtime artifact pin is missing or unknown', + ]; + } + + /// Overall qualification requires correctness, backend proof and provenance. + bool get qualified => + assertionsPassed && acceleratorVerified && provenanceProblems.isEmpty; + + Map toJson() => { + 'schema_version': manifest['schema_version'] == 2 ? 2 : 1, + 'manifest': manifest, + 'cases': cases, + 'summary': { + 'qualified': qualified, + 'qualification_gaps': qualificationGaps, + 'assertions_passed': assertionsPassed, + 'accelerator_verified': acceleratorVerified, + 'accelerator_evidence': placement, + 'provenance_complete': provenanceProblems.isEmpty, + 'provenance_problems': provenanceProblems, + 'expected': cases.length, + for (final status in ['PASS', 'FAIL', 'ERROR', 'NOT_RUN', 'UNSUPPORTED']) + status: cases.where((c) => c['status'] == status).length, + 'problems': problems, + }, + }; + + /// A failing integrity case preserves crash/collection failures in JUnit. + String toJUnit() { + final errors = + cases.where((c) => c['status'] == 'ERROR').length + (qualified ? 0 : 1); + final xml = StringBuffer( + '', + ); + for (final record in cases) { + xml.write( + '', + ); + final status = record['status']; + final content = _escape(jsonEncode(record)); + if (status == 'FAIL') { + xml.write('$content'); + } + if (status == 'ERROR') { + xml.write('$content'); + } + if (status == 'NOT_RUN' || status == 'UNSUPPORTED') { + xml.write(''); + } + xml.write('$content'); + } + if (!qualified) { + xml.write( + '${_escape([...problems, ...provenanceProblems, if (!assertionsPassed) 'Mandatory assertions incomplete or failed', if (!acceleratorVerified) 'Accelerator execution not verified'].join('; '))}', + ); + } + xml.write(''); + return xml.toString(); + } + + /// Measured repetitions only; warm-ups are excluded from comparisons. + List> get samples => cases + .where((c) => c['benchmark'] == true && c['warmup'] == false) + .toList(); + + String toCsv() { + const metrics = [ + 'wall_ms', + 'ttfa_ms', + 'estimated_wall_tps', + 'native_decode_tps', + ]; + String cell(Object? value) => + '"${(value ?? '').toString().replaceAll('"', '""')}"'; + return [ + ['run_id', 'case_id', 'status', ...metrics].join(','), + for (final sample in samples) + [ + manifest['run_id'], + sample['case_id'], + sample['status'], + for (final metric in metrics) + (sample['metrics'] as Map? ?? {})[metric], + ].map(cell).join(','), + ].join('\n'); + } + + /// Standalone escaped HTML with separate native and estimated throughput plots. + String toHtml() { + final profile = manifest['profile'] is Map + ? manifest['profile'] as Map + : const {}; + final catalog = + manifest['schema_version'] == 2 && manifest['catalog'] is Map + ? manifest['catalog'] as Map + : const {}; + final catalogCases = catalog['cases'] is List + ? catalog['cases'] as List + : const []; + final omitted = catalogCases + .whereType() + .where((c) => c['selected'] == false) + .toList(); + final omittedHtml = catalog.isEmpty + ? '' + : '
Unselected cases (${omitted.length})' + '

Outside this run; these cases are not passing or unsupported evidence.

' + '' + '${omitted.map((c) => '').join()}
CaseReason
${_escape(c['id'])}${_escape(c['omission_reason'])}
'; + final nativeReference = profile['execution_path'] == 'native_c_api'; + String chart(String metric, String title) { + final values = + samples + .map((s) => (s['metrics'] as Map? ?? {})[metric]) + .whereType() + .where((n) => n.isFinite && n >= 0) + .toList() + ..sort(); + final max = values.isEmpty ? 1 : math.max(1, values.last); + final stats = values.isEmpty + ? 'Unavailable' + : 'n=${values.length}; median ${((values[(values.length - 1) ~/ 2] + values[values.length ~/ 2]) / 2).toStringAsFixed(2)}; ' + 'min ${values.first.toStringAsFixed(2)}; max ${values.last.toStringAsFixed(2)}'; + return '

${_escape(title)}

$stats

${samples.map((s) { + final value = (s['metrics'] as Map? ?? {})[metric] as num?; + return '
${_escape(s['case_id'])} · ${_escape(s['status'])}' + '
' + '${value?.toStringAsFixed(2) ?? 'unavailable'}
'; + }).join()}
'; + } + + return '' + '' + 'llamadart validation

llamadart validation

' + '

${qualified ? 'QUALIFIED' : 'INCOMPLETE / FAILED'} · ${_escape(manifest['run_id'])}

' + '

Execution path: ${nativeReference ? 'direct native C API control (does not qualify the public Dart path)' : 'llamadart public API'}.

' + '

Selection: ${_escape(profile['selection'] ?? 'quick')}; ' + 'focused features: ${_escape(profile['focus_features'] ?? 'none')}; ' + 'catalog version: ${_escape(catalog['version'] ?? 'unavailable in legacy journal')}.

' + '

Functional assertions: ${assertionsPassed ? 'passed' : 'incomplete or failed'}. ' + 'Accelerator placement: ${placement['required'] != true + ? 'not required' + : acceleratorVerified + ? placement['placement'] == 'npu_participation_cpu_partitions_unknown' + ? 'NPU participation verified; CPU partition coverage unknown' + : 'verified native offload' + : 'unverified'}.

' + '

Qualification gaps: ${_escape(qualificationGaps.isEmpty ? 'none' : qualificationGaps.join(', '))}.

' + '

Unverified placement means accelerator execution has not been proven; ' + 'it does not by itself establish incompatibility. FAIL denotes an assertion ' + 'mismatch, ERROR an execution error, and NOT_RUN an unexecuted obligation. ' + 'Execution errors still require diagnosis to distinguish runtime, model and host setup.

' + '

Placement evidence: ${_escape(placement['reason'] ?? '')}.

' + '

${_escape([...problems, ...provenanceProblems].join('; '))}

' + '' + '${cases.map((c) => '').join()}
CaseStatusReason
${_escape(c['case_id'])}${_escape(c['status'])}${_escape(c['reason'] ?? '')}
' + '$omittedHtml' + '${chart('native_decode_tps', 'Native decode tokens/second')}' + '${chart('estimated_wall_tps', 'Estimated visible-output tokens/second')}' + '${chart('ttfa_ms', 'Time to first visible answer (ms)')}' + '${chart('native_ttft_ms', 'Native time to first token (ms)')}' + '

Each series represents one exact model/configuration/build. Failed outputs remain visible. ' + 'Retokenized output counts are estimates; stream chunks are not tokens.

' + '
Complete evidence
${_escape(const JsonEncoder.withIndent('  ').convert(toJson()))}
'; + } +} + +String _escape(Object? value) => const HtmlEscape( + HtmlEscapeMode.attribute, +).convert((value ?? '').toString()); diff --git a/packages/llamadart_validation/lib/src/runner.dart b/packages/llamadart_validation/lib/src/runner.dart new file mode 100644 index 000000000..bef8bc769 --- /dev/null +++ b/packages/llamadart_validation/lib/src/runner.dart @@ -0,0 +1,1157 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:llamadart/llamadart.dart'; + +import 'case_catalog.dart'; +import 'manifest.dart'; +import 'npu_evidence.dart'; +import 'runtime_environment.dart'; + +/// Sink implemented by file, browser and Firebase host adapters. +typedef ValidationEventSink = Future Function(Map event); + +/// Injectable public-API boundary for model-free harness regression tests. +abstract interface class ValidationEngine { + /// Whether the adapter executes in a browser, independent of profile input. + bool get isWeb; + Future load(String location, ValidationProfile profile); + Future unload(); + Future dispose(); + void cancel(); + Future> diagnostics(); + Future> tokenize(String text); + Future detokenize(List tokens); + Future> generate( + String prompt, + ValidationProfile profile, { + bool raw = false, + int? maxTokens, + int? streamBatchTokens, + int? streamBatchBytes, + bool cancelAfterFirst = false, + List? history, + List? stopSequences, + bool? enableThinking, + List? tools, + ToolChoice? toolChoice, + }); +} + +/// Runs inference through the exported llamadart API on every platform. +class PublicValidationEngine implements ValidationEngine { + /// Uses the public engine; a factory allows isolated adapter verification. + PublicValidationEngine({this.npu, LlamaEngine Function()? engineFactory}) + : _engineFactory = engineFactory ?? (() => LlamaEngine(LlamaBackend())); + final NpuExecutionMonitor? npu; + final LlamaEngine Function() _engineFactory; + late LlamaEngine _engine = _engineFactory(); + @override + bool get isWeb => const bool.fromEnvironment('dart.library.js_interop'); + bool _disposed = false; + + @override + Future load(String location, ValidationProfile profile) async { + requireValidationRuntimeEnvironment(); + profile.requireRunnable(verifiedAndroidNpuHost: npu != null); + if (_disposed) { + _engine = _engineFactory(); + _disposed = false; + } + await _engine.setLogLevel(LlamaLogLevel.info); + await _engine.loadModel( + location, + modelParams: profile.loadParams.copyWith( + liteRtLmDispatchLibDir: npu?.dispatchDirectory, + ), + ); + if (!_engine.isReady) { + throw StateError('loadModel returned without readiness'); + } + } + + @override + Future unload() => _engine.unloadModel(); + @override + Future dispose() async { + await _engine.dispose(); + _disposed = true; + } + + @override + void cancel() => _engine.cancelGeneration(); + @override + Future> tokenize(String text) => + _engine.tokenize(text, addSpecial: false); + @override + Future detokenize(List tokens) => _engine.detokenize(tokens); + + @override + Future> diagnostics() async => { + 'backend_name': await _engine.getBackendName(), + 'reported_gpu_layers': await _engine.getResolvedGpuLayers(), + 'context_size': await _engine.getContextSize(), + 'model_metadata': await _engine.getMetadata(), + if (npu != null) 'npu_identity': npu!.identity, + // These public values can contain selector hints. They are not placement proof. + 'accelerator_execution_verified': false, + 'accelerator_evidence_reason': + 'requires correlated native driver/offload diagnostics', + }; + + @override + Future> generate( + String prompt, + ValidationProfile profile, { + bool raw = false, + int? maxTokens, + int? streamBatchTokens, + int? streamBatchBytes, + bool cancelAfterFirst = false, + List? history, + List? stopSequences, + bool? enableThinking, + List? tools, + ToolChoice? toolChoice, + }) async { + final text = StringBuffer(); + final thinking = StringBuffer(); + final finish = []; + final toolDeltas = >[]; + var toolBytes = 0; + var ordered = true; + var chunks = 0; + int? firstUs; + int? cancelUs; + final params = profile.generationParams.copyWith( + maxTokens: maxTokens, + streamBatchTokenThreshold: streamBatchTokens, + streamBatchByteThreshold: streamBatchBytes, + stopSequences: stopSequences, + ); + final npuBefore = npu?.snapshot(); + final watch = Stopwatch()..start(); + void append(String content) { + if (content.isEmpty) return; + firstUs ??= watch.elapsedMicroseconds; + text.write(content); + if (text.length > 65536) { + cancel(); + throw StateError('Output exceeded the 64 KiB core limit'); + } + if (cancelAfterFirst && cancelUs == null) { + cancelUs = watch.elapsedMicroseconds; + cancel(); + } + } + + String? cancellationAbort; + try { + if (raw) { + await for (final delta in _engine.generate(prompt, params: params)) { + chunks++; + append(delta); + } + } else { + await for (final chunk in _engine.create( + history ?? + [ + LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: prompt, + ), + ], + params: params, + enableThinking: enableThinking ?? profile.enableThinking, + tools: tools, + toolChoice: toolChoice, + )) { + chunks++; + for (final choice in chunk.choices) { + if (choice.index != 0 || finish.isNotEmpty) ordered = false; + for (final tool + in choice.delta.toolCalls ?? []) { + final data = tool.toJson(); + toolBytes += utf8.encode(jsonEncode(data)).length; + if (toolBytes > 65536) { + cancel(); + throw StateError('Tool deltas exceeded the 64 KiB core limit'); + } + toolDeltas.add(data); + } + append(choice.delta.content ?? ''); + thinking.write(choice.delta.thinking ?? ''); + if (thinking.length > 65536) { + cancel(); + throw StateError('Thinking exceeded the 64 KiB core limit'); + } + if (choice.finishReason != null) finish.add(choice.finishReason!); + } + } + } + } on LlamaInferenceException catch (error) { + // The pinned WASM delegate surfaces cancellation as this explicit abort. + if (cancelUs != null && + error.message == 'Generation failed' && + '${error.details}' == 'AbortError: Generation was cancelled.') { + cancellationAbort = error.toString(); + } else { + rethrow; + } + } + watch.stop(); + final npuAfter = npu?.snapshot(); + // Tokenization and diagnostic reads occur after the timed region. + int? estimatedTokens; + try { + estimatedTokens = (await tokenize(text.toString())).length; + } catch (_) {} + BackendPerfContextData? perf; + try { + perf = await _engine.getPerformanceContext(); + } catch (_) {} + final wallMs = watch.elapsedMicroseconds / 1000; + final nativeMs = perf?.decodeMs ?? perf?.evalMs; + final nativeTokens = perf?.evalTokens; + return { + 'prompt': prompt, + if (history != null) 'messages': history.map((m) => m.toJson()).toList(), + 'max_tokens': params.maxTokens, + 'stop_sequences': params.stopSequences, + 'enable_thinking': enableThinking ?? profile.enableThinking, + 'tools': tools?.map((tool) => tool.toJson()).toList(), + 'tool_choice': toolChoice?.name, + if (npuBefore != null && npuAfter != null) + 'npu_execution': npuGenerationEvidence(npuBefore, npuAfter), + 'content': text.toString(), + 'thinking': thinking.toString(), + 'chunks': chunks, + 'finish_reasons': finish, + 'tool_call_deltas': toolDeltas, + 'stream_completed': cancellationAbort == null, + 'completion_order_valid': + ordered && (raw ? finish.isEmpty : finish.length == 1), + 'stream_batch_tokens': params.streamBatchTokenThreshold, + 'stream_batch_bytes': params.streamBatchByteThreshold, + 'cancel_requested': cancelUs != null, + 'cancel_abort_observed': cancellationAbort != null, + 'cancel_abort': ?cancellationAbort, + 'cancel_to_done_ms': cancelUs == null + ? null + : (watch.elapsedMicroseconds - cancelUs!) / 1000, + 'metrics': { + 'wall_ms': wallMs, + 'ttfa_ms': firstUs == null ? null : firstUs! / 1000, + 'native_ttft_ms': null, + 'estimated_output_tokens': estimatedTokens, + 'token_count_source': 'retokenized visible output; not stream chunks', + 'estimated_wall_tps': wallMs > 0 && estimatedTokens != null + ? estimatedTokens * 1000 / wallMs + : null, + 'native_decode_tokens': nativeTokens, + 'native_decode_ms': nativeMs, + 'native_decode_tps': + nativeMs != null && + nativeMs > 0 && + nativeTokens != null && + nativeTokens > 0 + ? nativeTokens * 1000 / nativeMs + : null, + 'native_timing_source': perf == null + ? null + : 'getPerformanceContext per-request counters', + 'native_prompt_tokens': perf?.promptEvalTokens, + 'native_prompt_ms': perf?.promptEvalMs, + 'missing_native_metrics_reason': perf == null + ? 'backend did not expose counters' + : null, + }, + }; + } +} + +/// Single model/profile run. Assertion failures do not suppress later cases. +class ValidationRunner { + ValidationRunner({ + required this.profile, + required this.engine, + required this.emit, + this.caseTimeout = const Duration(seconds: 60), + }); + + final ValidationProfile profile; + final ValidationEngine engine; + final ValidationEventSink emit; + final Duration caseTimeout; + var _sequence = 0; + var _cancelled = false; + bool _closed = false; + bool _poisoned = false; + bool _settled = true; + String? _operationPhase; + Map? _partialCaseEvidence; + + /// Request cancellation from the UI or host without marking success. + void cancel() { + _cancelled = true; + engine.cancel(); + } + + /// Expanded obligations shared with the independent report validator. + List get caseIds => profile.caseIds; + + /// Captures a run without pretending selector diagnostics prove GPU placement. + Future run( + String location, { + required String runId, + required Map environment, + Map preparation = const {}, + }) async { + try { + await emit({ + 'type': 'manifest', + 'schema_version': 2, + 'run_id': runId, + 'profile': profile.toJson(), + 'profile_hash': jsonHash(profile.toJson()), + 'effective_config': profile.effectiveConfig, + 'config_hash': jsonHash(profile.effectiveConfig), + 'environment': environment, + 'preparation': preparation, + 'case_ids': caseIds, + 'catalog': profile.catalog, + 'catalog_hash': jsonHash(profile.catalog), + 'started_at': DateTime.now().toUtc().toIso8601String(), + 'accelerator_evidence_required': profile.requiresAcceleratorProof, + }); + var usable = false; + var poisoned = false; + for (final id in caseIds) { + if (_cancelled || poisoned || (!usable && id != 'C01.load')) { + await _record(id, 'NOT_RUN', { + 'reason': _cancelled + ? 'cancelled' + : poisoned + ? 'prior_timeout' + : 'model_load_failed', + }); + continue; + } + if (!validationCase(id).implemented) { + await _record(id, 'NOT_RUN', { + 'reason': 'selected case or reference fixture not implemented', + }); + continue; + } + await emit({ + 'type': 'case_start', + 'case_id': id, + 'sequence': _sequence++, + }); + final watch = Stopwatch()..start(); + _operationPhase = id; + _partialCaseEvidence = null; + try { + _settled = false; + final pending = _runCase( + id, + location, + ).whenComplete(() => _settled = true); + final actual = await pending.timeout(caseTimeout); + if (id == 'C01.load') usable = true; + final status = actual.remove('status') as String? ?? 'PASS'; + await _record(id, status, { + ...actual, + 'elapsed_ms': watch.elapsedMicroseconds / 1000, + }); + } on TimeoutException { + poisoned = true; + _poisoned = true; + engine.cancel(); + await _record(id, 'ERROR', { + 'reason': 'case_timeout', + ...?_partialCaseEvidence, + 'timeout_ms': caseTimeout.inMilliseconds, + 'operation_phase': _operationPhase, + 'elapsed_ms': watch.elapsedMicroseconds / 1000, + }); + } catch (error) { + await _record(id, 'ERROR', { + 'reason': 'runtime_exception', + ...?_partialCaseEvidence, + 'operation_phase': _operationPhase, + 'error_type': error.runtimeType.toString(), + 'message': redactDiagnostic('$error'), + }); + } + } + } finally { + _closed = true; + try { + await engine.dispose().timeout(const Duration(seconds: 10)); + await emit({ + 'type': 'cleanup', + 'status': _settled ? 'PASS' : 'ERROR', + if (!_settled) + 'message': 'Timed-out backend operation has not settled', + 'sequence': _sequence++, + }); + } catch (error) { + await emit({ + 'type': 'cleanup', + 'status': 'ERROR', + 'sequence': _sequence++, + 'message': redactDiagnostic('$error'), + }); + } + await emit({ + 'type': 'run_end', + 'sequence': _sequence++, + 'cancelled': _cancelled, + }); + } + } + + Future _record(String id, String status, Map values) => + emit({ + 'type': 'case', + 'case_id': id, + 'case_version': validationCase(id).version, + 'fixture_hash': jsonHash(profile.caseFixtures(id)), + 'status': status, + 'sequence': _sequence++, + ...values, + }); + + String get _shortPrompt => profile.isChat + ? profile.fixtureText('hello', 'prompt') + : profile.fixtureText('raw', 'prompt'); + + Future _checked(Future Function() operation) async { + void check() { + if (_closed || _poisoned || _cancelled) { + throw StateError('Case no longer active'); + } + } + + check(); + final result = await operation(); + check(); + return result; + } + + Future> _batching() async { + if (profile.nativeReference || + profile.backend == 'npu' || + (engine.isWeb && profile.runtime != 'litert')) { + return { + 'status': 'NOT_RUN', + 'reason': profile.nativeReference + ? 'Direct native control bypasses public worker batching' + : profile.backend == 'npu' + ? 'NPU runtime-default sampling has no qualified deterministic parity control' + : 'GGUF browser worker batching is not a qualified native-worker control', + }; + } + final fixture = profile.fixtures['batching'] as Map; + final tokens = fixture['token_threshold'] as int; + final bytes = fixture['byte_threshold'] as int; + final defaults = profile.generationParams; + Future> generate({ + int? tokenThreshold, + int? byteThreshold, + }) => _checked( + () => engine.generate( + _shortPrompt, + profile, + raw: !profile.isChat, + streamBatchTokens: tokenThreshold, + streamBatchBytes: byteThreshold, + ), + ); + final control = await generate(); + if (engine.isWeb) { + final rejected = >[]; + for (final option in [ + 'streamBatchTokenThreshold', + 'streamBatchByteThreshold', + ]) { + try { + final output = await generate( + tokenThreshold: option == 'streamBatchTokenThreshold' + ? tokens + : null, + byteThreshold: option == 'streamBatchByteThreshold' ? bytes : null, + ); + rejected.add({'option': option, 'rejected': false, 'output': output}); + } on LlamaUnsupportedException catch (error) { + rejected.add({ + 'option': option, + 'rejected': error.toString().contains(option), + 'error_type': error.runtimeType.toString(), + 'message': redactDiagnostic(error.toString()), + }); + } + } + final recovery = await generate(); + return { + 'control': control, + 'rejected_options': rejected, + 'recovery': recovery, + 'coverage': 'litert_web_native_option_rejection_and_recovery', + 'status': + rejected.every((entry) => entry['rejected'] == true) && + _validBatchOutput(control) && + _validBatchOutput(recovery) + ? 'PASS' + : 'FAIL', + 'expected': + 'Each native batching option is rejected with a named typed error; default requests still complete', + }; + } + final batched = await generate( + tokenThreshold: tokens, + byteThreshold: bytes, + ); + final recovery = await generate(); + final outputs = [control, batched, recovery]; + final configMatches = + control['stream_batch_tokens'] == defaults.streamBatchTokenThreshold && + control['stream_batch_bytes'] == defaults.streamBatchByteThreshold && + batched['stream_batch_tokens'] == tokens && + batched['stream_batch_bytes'] == bytes && + recovery['stream_batch_tokens'] == defaults.streamBatchTokenThreshold && + recovery['stream_batch_bytes'] == defaults.streamBatchByteThreshold; + final equal = outputs.every( + (output) => + output['content'] == control['content'] && + output['thinking'] == control['thinking'] && + canonicalJson(output['finish_reasons']) == + canonicalJson(control['finish_reasons']), + ); + final tools = outputs.any( + (output) => + output['tool_call_deltas'] is List && + (output['tool_call_deltas'] as List).isNotEmpty, + ); + return { + 'control': control, + 'batched': batched, + 'recovery': recovery, + 'coverage': 'native_text_and_thinking_reconstruction', + 'configurations_verified': configMatches, + 'reconstruction_equal': equal, + 'status': tools + ? 'NOT_RUN' + : configMatches && equal && outputs.every(_validBatchOutput) + ? 'PASS' + : 'FAIL', + 'expected': + 'Same nonempty content, thinking and finish reasons with ordered completion; default configuration recovers; chunk count may differ', + if (tools) + 'reason': + 'Tool emissions require the separately qualified C07 tool fixture', + }; + } + + bool _validBatchOutput(Map output) => + output['content'] is String && + (output['content'] as String).trim().isNotEmpty && + output['thinking'] is String && + output['finish_reasons'] is List && + output['tool_call_deltas'] is List && + output['stream_completed'] == true && + output['completion_order_valid'] == true && + output['cancel_requested'] == false; + + Future> _short() => _checked( + () => engine.generate(_shortPrompt, profile, raw: !profile.isChat), + ); + + Map _nonempty(Map output) => { + ...output, + 'expected': 'nonempty finite output', + 'status': (output['content'] as String).trim().isNotEmpty ? 'PASS' : 'FAIL', + }; + + Future> _withDiagnostics( + Map output, + ) async { + final diagnostics = await _checked(() => engine.diagnostics()); + final name = (diagnostics['backend_name'] as String? ?? '').toLowerCase(); + final runtimeResolved = + name.startsWith('litert-lm') == (profile.runtime == 'litert'); + final metadata = diagnostics['model_metadata'] as Map? ?? {}; + final wasmCpu = + name == 'wasm (prototype bridge)' && + metadata['llamadart.webgpu.n_gpu_layers'] == '0' && + metadata['llamadart.webgpu.core_variant'] == 'wasm32'; + final cpuResolved = + (name.contains('cpu') || wasmCpu) && + !RegExp(r'cuda|metal|vulkan|\bgpu\b|\bnpu\b').hasMatch(name); + return { + ...output, + 'diagnostics': diagnostics, + 'status': !runtimeResolved || (profile.backend == 'cpu' && !cpuResolved) + ? 'FAIL' + : output['status'] as String? ?? 'PASS', + if (!runtimeResolved) + 'reason': 'Resolved runtime does not match model format', + if (runtimeResolved && profile.backend == 'cpu' && !cpuResolved) + 'reason': 'Explicit CPU profile did not resolve to CPU diagnostics', + }; + } + + Future> _tools() async { + final fixture = profile.fixtures['tools'] as Map; + final function = (fixture['tool'] as Map)['function'] as Map; + final tool = ToolDefinition( + name: function['name'] as String, + description: function['description'] as String, + parameters: [ToolParam.string('city', required: true)], + handler: (_) async => fixture['response'], + ); + if (canonicalJson(tool.toJson()) != canonicalJson(fixture['tool'])) { + throw StateError( + 'Tool fixture schema does not match the public tool definition', + ); + } + final trials = >[]; + _partialCaseEvidence = {'trials': trials}; + var passed = true; + for (final mode in [ + ToolChoice.auto, + ToolChoice.required, + ToolChoice.none, + ]) { + _operationPhase = 'tools.${mode.name}.generate'; + final output = await _checked( + () => engine.generate( + fixture['prompt'] as String, + profile, + tools: [tool], + toolChoice: mode, + enableThinking: false, + maxTokens: 128, + ), + ); + final deltas = output['tool_call_deltas'] as List; + final name = StringBuffer(); + final arguments = StringBuffer(); + String? callId; + var valid = true; + for (final delta in deltas.cast()) { + if (delta['index'] != 0) valid = false; + if (delta['id'] != null) { + if (callId != null && callId != delta['id']) valid = false; + callId = delta['id'] as String; + } + final fn = delta['function'] as Map?; + name.write(fn?['name'] ?? ''); + arguments.write(fn?['arguments'] ?? ''); + } + Object? decoded; + if (deltas.isNotEmpty) { + try { + decoded = jsonDecode(arguments.toString()); + } on FormatException { + valid = false; + } + } + final expectedFinish = mode == ToolChoice.none ? 'stop' : 'tool_calls'; + final modePassed = + canonicalJson(output['finish_reasons']) == + canonicalJson([expectedFinish]) && + output['tool_choice'] == mode.name && + output['enable_thinking'] == false && + canonicalJson(output['tools']) == canonicalJson([tool.toJson()]) && + output['stream_completed'] == true && + output['completion_order_valid'] == true && + (mode == ToolChoice.none + ? deltas.isEmpty && + (output['content'] as String).trim().isNotEmpty + : valid && + deltas.isNotEmpty && + name.toString() == tool.name && + canonicalJson(decoded) == + canonicalJson(fixture['expected_arguments'])); + passed = passed && modePassed; + final trial = { + ...output, + 'mode_passed': modePassed, + 'reconstructed_name': name.toString(), + 'reconstructed_arguments': decoded, + }; + trials.add(trial); + Map? followup; + if (mode != ToolChoice.none && modePassed) { + final response = await tool.invoke( + Map.from(decoded as Map), + ); + final prompt = + 'What is the temperature_celsius from the tool result? Reply with only the number.'; + _operationPhase = 'tools.${mode.name}.tool_result_followup'; + final answer = await _checked( + () => engine.generate( + prompt, + profile, + enableThinking: false, + tools: [tool], + toolChoice: ToolChoice.none, + history: [ + LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: fixture['prompt'] as String, + ), + LlamaChatMessage.withContent( + role: LlamaChatRole.assistant, + content: [ + LlamaToolCallContent( + id: callId, + name: tool.name, + arguments: Map.from(decoded as Map), + rawJson: arguments.toString(), + ), + ], + ), + LlamaChatMessage.withContent( + role: LlamaChatRole.tool, + content: [ + LlamaToolResultContent( + id: callId, + name: tool.name, + result: response, + ), + ], + ), + LlamaChatMessage.fromText(role: LlamaChatRole.user, text: prompt), + ], + ), + ); + passed = + passed && + (answer['content'] as String).trim() == + '${(fixture['response'] as Map)['temperature_celsius']}' && + (answer['tool_call_deltas'] as List).isEmpty && + answer['tool_choice'] == 'none' && + canonicalJson(answer['finish_reasons']) == + canonicalJson(['stop']) && + answer['stream_completed'] == true && + answer['completion_order_valid'] == true; + followup = answer; + } + if (followup != null) trial['tool_result_followup'] = followup; + } + _operationPhase = 'tools.recovery'; + final recovery = await _short(); + return { + 'trials': trials, + 'recovery': recovery, + 'status': + passed && + RegExp( + profile.fixtureText('hello', 'regex'), + caseSensitive: false, + ).hasMatch(recovery['content'] as String) && + recovery['stream_completed'] == true && + recovery['completion_order_valid'] == true && + recovery['tools'] == null && + recovery['tool_choice'] == null && + (recovery['tool_call_deltas'] as List).isEmpty && + (profile.enableThinking || recovery['thinking'] == '') && + recovery['enable_thinking'] == profile.enableThinking && + canonicalJson(recovery['finish_reasons']) == + canonicalJson(['stop']) + ? 'PASS' + : 'FAIL', + }; + } + + Future> _runCase(String id, String location) async { + if (profile.nativeReference && + ['C02.generate', 'C05.thinking', 'C07.tools'].contains(id)) { + return { + 'status': 'NOT_RUN', + 'reason': 'Requires public chat feature controls', + }; + } + switch (id) { + case 'C02.generate': + final output = await _checked( + () => engine.generate( + profile.fixtureText('unicode_generation', 'prompt'), + profile, + enableThinking: false, + ), + ); + final expected = profile.fixtureText('unicode_generation', 'expected'); + return { + ...output, + 'expected': expected, + 'status': + output['content'] == expected && + output['thinking'] == '' && + output['enable_thinking'] == false && + output['stream_completed'] == true && + output['completion_order_valid'] == true + ? 'PASS' + : 'FAIL', + }; + case 'C05.thinking': + final prompt = profile.fixtureText('arithmetic', 'prompt'); + final trials = >[]; + for (final enabled in [true, false]) { + trials.add( + await _checked( + () => engine.generate( + prompt, + profile, + enableThinking: enabled, + maxTokens: 512, + ), + ), + ); + } + final expected = RegExp(profile.fixtureText('arithmetic', 'regex')); + return { + 'trials': trials, + 'status': + trials.every( + (trial) => + expected.hasMatch( + (trial['content'] as String).trim(), + ) && + trial['stream_completed'] == true && + trial['completion_order_valid'] == true, + ) && + trials[0]['enable_thinking'] == true && + (trials[0]['thinking'] as String).trim().isNotEmpty && + trials[1]['enable_thinking'] == false && + trials[1]['thinking'] == '' + ? 'PASS' + : 'FAIL', + }; + case 'C07.tools': + return _tools(); + case 'C01.load': + _operationPhase = 'public_load'; + await _checked(() => engine.load(location, profile)); + return _withDiagnostics({ + 'load_scope': 'public_load_and_readiness', + 'native_initialization_proven': false, + 'initialization_note': + 'Public readiness does not prove eager native initialization; first use may include deferred initialization.', + }); + case 'C02.unicode': + final text = profile.fixtureText('unicode', 'input'); + _operationPhase = profile.runtime == 'litert' + ? 'tokenize_including_possible_deferred_initialization' + : 'tokenize'; + final tokenizeWatch = Stopwatch()..start(); + final tokens = await _checked(() => engine.tokenize(text)); + tokenizeWatch.stop(); + _operationPhase = 'detokenize'; + final detokenizeWatch = Stopwatch()..start(); + final decoded = await _checked(() => engine.detokenize(tokens)); + detokenizeWatch.stop(); + final prefix = profile.fixtureText('unicode', 'expected_prefix'); + final expected = '$prefix$text'; + return { + 'input': text, + 'tokens': tokens, + 'decoded': decoded, + 'expected': expected, + 'tokenizer_prefix': prefix, + 'tokenize_call_ms': tokenizeWatch.elapsedMicroseconds / 1000, + 'detokenize_call_ms': detokenizeWatch.elapsedMicroseconds / 1000, + 'tokenize_timing_scope': profile.runtime == 'litert' + ? 'public_call_including_possible_deferred_initialization' + : 'public_call', + 'status': decoded == expected ? 'PASS' : 'FAIL', + }; + case 'C03.raw': + return _nonempty( + await _checked( + () => engine.generate( + profile.fixtureText('raw', 'prompt'), + profile, + raw: true, + ), + ), + ); + case 'C04.hello': + case 'C04.arithmetic': + final arithmetic = id.endsWith('arithmetic'); + final fixture = arithmetic ? 'arithmetic' : 'hello'; + final prompt = profile.fixtureText(fixture, 'prompt'); + final expected = profile.fixtureText(fixture, 'regex'); + final output = await _checked(() => engine.generate(prompt, profile)); + final text = (output['content'] as String).trim(); + return { + ...output, + 'expected_regex': expected, + 'status': + RegExp(expected, caseSensitive: false).hasMatch(text) && + (output['thinking'] as String).isEmpty + ? 'PASS' + : 'FAIL', + }; + case 'C06.history': + case 'C06.history.public_system_wire': + case 'C06.history.no_system': + case 'C06.history.combined': + final prompt = profile.fixtureText('history', 'prompt'); + final system = profile.fixtureText('history', 'system'); + final literalSystem = id == 'C06.history.public_system_wire'; + final combined = id == 'C06.history.combined'; + final messages = [ + if (id != 'C06.history.no_system') + LlamaChatMessage.fromText( + role: LlamaChatRole.system, + text: literalSystem + ? jsonEncode({ + 'role': 'system', + 'content': [ + {'type': 'text', 'text': system}, + ], + }) + : system, + ), + LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: profile.fixtureText('history', 'user'), + ), + LlamaChatMessage.fromText( + role: LlamaChatRole.assistant, + text: profile.fixtureText('history', 'assistant'), + ), + LlamaChatMessage.fromText(role: LlamaChatRole.user, text: prompt), + ]; + final selectedPrompt = combined + ? messages.map((message) => message.content).join('\n') + : prompt; + final output = await _checked( + () => engine.generate( + selectedPrompt, + profile, + history: combined ? null : messages, + ), + ); + return { + ...output, + if (profile.historyControls) 'history_control': id, + 'expected': profile.fixtureText('history', 'expected'), + 'status': + (output['content'] as String).trim() == + profile.fixtureText('history', 'expected') + ? 'PASS' + : 'FAIL', + }; + case 'C08.cancel': + final prompt = profile.isChat + ? profile.fixtureText('cancel', 'chat_prompt') + : profile.fixtureText('raw', 'prompt'); + final control = await _checked( + () => engine.generate( + prompt, + profile, + raw: !profile.isChat, + maxTokens: (profile.fixtures['cancel'] as Map)['max_tokens'] as int, + ), + ); + final output = await _checked( + () => engine.generate( + prompt, + profile, + raw: !profile.isChat, + maxTokens: (profile.fixtures['cancel'] as Map)['max_tokens'] as int, + cancelAfterFirst: true, + ), + ); + final recovery = await _short(); + final duration = output['cancel_to_done_ms'] as num?; + final fullTokens = + (control['metrics'] as Map?)?['native_decode_tokens'] as num?; + final cancelledTokens = + (output['metrics'] as Map?)?['native_decode_tokens'] as num?; + final prefixMatches = + (output['content'] as String).isNotEmpty && + (control['content'] as String).startsWith( + output['content'] as String, + ); + final nativeInterruption = + fullTokens != null && + cancelledTokens != null && + cancelledTokens > 0 && + cancelledTokens < fullTokens; + final interrupted = + prefixMatches && + (nativeInterruption || output['cancel_abort_observed'] == true); + + return { + ...output, + 'recovery': recovery, + 'uncancelled_control': control, + 'interruption_observed': interrupted, + 'status': output['cancel_requested'] != true || !interrupted + ? 'NOT_RUN' + : duration != null && + duration <= + (profile.fixtures['cancel'] as Map)['deadline_ms'] && + (recovery['content'] as String).trim().isNotEmpty + ? 'PASS' + : 'FAIL', + 'reason': + 'requires fewer native decoded tokens or an explicit cancellation abort, matching the seeded control prefix, prompt recovery and bounded cancellation latency', + }; + case 'C09.reload': + case 'C09.reload.second': + await _checked(() => engine.dispose()); + await _checked(() => engine.load(location, profile)); + return _withDiagnostics(_nonempty(await _short())); + case 'C10.limit': + final output = await _checked( + () => engine.generate( + _shortPrompt, + profile, + raw: !profile.isChat, + maxTokens: (profile.fixtures['limit'] as Map)['max_tokens'] as int, + ), + ); + final count = + (output['metrics'] as Map)['native_decode_tokens'] as num?; + return { + ...output, + 'expected': 'at most one native decoded token', + 'status': count == null + ? 'NOT_RUN' + : count == + (profile.fixtures['limit'] + as Map)['expected_native_decode_tokens'] + ? 'PASS' + : 'FAIL', + if (count == null) + 'reason': 'native token counter unavailable; chunks are not tokens', + }; + case 'C11.batching': + return _batching(); + case 'C10.stop': + if (profile.nativeReference || !profile.isChat) { + return { + 'status': 'NOT_RUN', + 'reason': 'Requires public chat generation', + }; + } + final prompt = profile.fixtureText('stop', 'prompt'); + final marker = profile.fixtureText('stop', 'marker'); + final control = await _checked(() => engine.generate(prompt, profile)); + final text = control['content'] as String; + final index = text.indexOf(marker); + final stopped = await _checked( + () => engine.generate(prompt, profile, stopSequences: [marker]), + ); + final recovery = await _short(); + return { + 'control': control, + 'stopped': stopped, + 'recovery': recovery, + 'stop_marker': marker, + 'expected_prefix': index < 0 ? null : text.substring(0, index), + 'status': + index > 0 && + marker.isNotEmpty && + stopped['content'] == text.substring(0, index) && + stopped['stream_completed'] == true && + stopped['completion_order_valid'] == true && + canonicalJson(stopped['stop_sequences']) == + canonicalJson([marker]) && + (recovery['content'] as String).trim().isNotEmpty + ? 'PASS' + : 'FAIL', + }; + case 'C12.guards': + if (profile.nativeReference) { + return { + 'status': 'NOT_RUN', + 'reason': 'Native control bypasses public readiness guards', + }; + } + await _checked(() => engine.unload()); + String? rejected; + try { + await _short(); + } on LlamaContextException catch (error) { + rejected = error.runtimeType.toString(); + } + await _checked(() => engine.load(location, profile)); + final recovery = await _short(); + return _withDiagnostics({ + 'rejected_error_type': rejected, + 'recovery': recovery, + 'expected': + 'typed unloaded-engine rejection followed by valid generation', + 'status': + rejected != null && + (recovery['content'] as String).trim().isNotEmpty + ? 'PASS' + : 'FAIL', + }); + case 'C12.recovery': + await _checked(() => engine.unload()); + String? errorType; + try { + await _checked(() => engine.load('$location.missing', profile)); + } on LlamaException catch (error) { + errorType = error.runtimeType.toString(); + } + await _checked(() => engine.unload()); + await _checked(() => engine.load(location, profile)); + final output = await _short(); + return _withDiagnostics({ + ...output, + 'rejected_error_type': errorType, + 'status': + errorType != null && + (output['content'] as String).trim().isNotEmpty + ? 'PASS' + : 'FAIL', + 'expected': 'typed missing-model error and valid recovery', + }); + case 'B01.warmup': + case 'B01.1': + case 'B01.2': + case 'B01.3': + final prompt = profile.isChat + ? profile.fixtureText('benchmark', 'chat_prompt') + : profile.fixtureText('raw', 'prompt'); + return { + ..._nonempty( + await _checked( + () => engine.generate(prompt, profile, raw: !profile.isChat), + ), + ), + 'benchmark': true, + 'warmup': id == 'B01.warmup', + 'cohort': profile.isChat + ? 'short-generation' + : 'tiny-packaging-diagnostic', + }; + default: + throw StateError('No implementation for catalog case $id'); + } + } +} + +/// Bounds and removes common credential/path material from diagnostic exports. +String redactDiagnostic(String value) { + final redacted = value + .replaceAll( + RegExp(r'bearer\s+[^\s]+', caseSensitive: false), + 'Bearer [redacted]', + ) + .replaceAll(RegExp(r'https?://[^\s]+\?[^\s]+'), '[signed URL redacted]') + .replaceAll(RegExp(r'/(?:Users|home)/[^\s/]+'), '/[user]'); + return redacted.length <= 8192 + ? redacted + : '${redacted.substring(0, 8192)}[truncated]'; +} diff --git a/packages/llamadart_validation/lib/src/runtime_environment.dart b/packages/llamadart_validation/lib/src/runtime_environment.dart new file mode 100644 index 000000000..b78e879b3 --- /dev/null +++ b/packages/llamadart_validation/lib/src/runtime_environment.dart @@ -0,0 +1,41 @@ +import 'runtime_environment_stub.dart' + if (dart.library.io) 'runtime_environment_io.dart'; + +/// Validation must use the pinned runtime, without ambient library overrides. +/// Values are deliberately omitted from diagnostics because paths can be secret. +/// Dart tooling sets library search paths for JIT native assets; only portable +/// qualification forbids those paths. JIT results cannot qualify a desktop row. +void requireValidationRuntimeEnvironment({ + Map? environment, + bool portable = false, +}) { + final values = environment ?? runtimeEnvironment(); + const overrides = { + 'LLAMADART_LITERT_LM_LIB_DIR', + 'LLAMADART_NATIVE_LIB_DIR', + 'LLAMADART_BACKEND_MODULE_DIR', + 'LLAMADART_ALLOW_LEGACY_LOCAL_BUNDLES', + 'GGML_BACKEND_PATH', + 'WEBGPU_BRIDGE_ASSETS_TAG', + 'WEBGPU_BRIDGE_ASSETS_REPO', + 'LD_PRELOAD', + 'DYLD_INSERT_LIBRARIES', + }; + final active = + values.keys + .where( + (key) => + (overrides.contains(key) || + (portable && + (key.startsWith('DYLD_') || + key == 'LD_LIBRARY_PATH'))) && + values[key]!.isNotEmpty, + ) + .toList() + ..sort(); + if (active.isNotEmpty) { + throw StateError( + 'Validation requires pinned runtime assets; unset ${active.join(', ')}', + ); + } +} diff --git a/packages/llamadart_validation/lib/src/runtime_environment_io.dart b/packages/llamadart_validation/lib/src/runtime_environment_io.dart new file mode 100644 index 000000000..232b87515 --- /dev/null +++ b/packages/llamadart_validation/lib/src/runtime_environment_io.dart @@ -0,0 +1,3 @@ +import 'dart:io'; + +Map runtimeEnvironment() => Platform.environment; diff --git a/packages/llamadart_validation/lib/src/runtime_environment_stub.dart b/packages/llamadart_validation/lib/src/runtime_environment_stub.dart new file mode 100644 index 000000000..2ec04dddb --- /dev/null +++ b/packages/llamadart_validation/lib/src/runtime_environment_stub.dart @@ -0,0 +1 @@ +Map runtimeEnvironment() => const {}; diff --git a/packages/llamadart_validation/lib/src/speech_runner.dart b/packages/llamadart_validation/lib/src/speech_runner.dart new file mode 100644 index 000000000..81eff0716 --- /dev/null +++ b/packages/llamadart_validation/lib/src/speech_runner.dart @@ -0,0 +1,522 @@ +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:llamadart/llamadart.dart'; + +import 'runner.dart' show redactDiagnostic; + +/// Word edit distance divided by reference words; insertions can exceed 1.0. +/// Normalization ignores case and ASCII punctuation, preserving Unicode words. +double speechWordErrorRate(String reference, String actual) { + List words(String text) => text + .toLowerCase() + .replaceAll(RegExp(r'''[.,!?:;"—–]'''), ' ') + .trim() + .split(RegExp(r'\s+')) + .where((word) => word.isNotEmpty) + .toList(); + final expected = words(reference); + final received = words(actual); + if (expected.isEmpty) throw ArgumentError('Reference must contain words'); + var previous = List.generate(received.length + 1, (index) => index); + for (var i = 1; i <= expected.length; i++) { + final current = [i]; + for (var j = 1; j <= received.length; j++) { + final costs = [ + previous[j] + 1, + current[j - 1] + 1, + previous[j - 1] + (expected[i - 1] == received[j - 1] ? 0 : 1), + ]..sort(); + current.add(costs.first); + } + previous = current; + } + return previous.last / expected.length; +} + +/// Validates generated audio, returning measurements without claiming quality. +Map inspectSpeechAudio(TextToSpeechResult result) { + if (result.sampleRateHz != 24000 || + result.channelCount != 1 || + result.samples.isEmpty || + result.samples.any((sample) => !sample.isFinite) || + result.samples.every((sample) => sample == 0) || + result.truncated) { + throw StateError('Invalid, silent, nonfinite or truncated TTS output'); + } + final seconds = result.samples.length / result.sampleRateHz; + return { + 'sample_rate_hz': result.sampleRateHz, + 'channels': result.channelCount, + 'samples': result.samples.length, + 'audio_seconds': seconds, + 'truncated': result.truncated, + 'listening_check': 'NOT_RUN', + 'intelligibility': 'UNVERIFIED', + }; +} + +/// Speech-only public API adapter; independent of chat-model generation. +abstract interface class SpeechValidationAdapter { + Future load(); + Future dispose(); + Future> execute({ + bool cancel = false, + bool invalid = false, + bool bytesInput = false, + }); +} + +/// Public GGUF speech adapter used by the portable speech runner. +class PublicSpeechValidationAdapter implements SpeechValidationAdapter { + PublicSpeechValidationAdapter({ + required this.model, + required this.projector, + required this.backend, + required this.pack, + required this.saveAudio, + this.audio, + this.audioSeconds, + this.audioPath, + this.reference, + this.text = 'Hello from llamadart. The answer is forty two.', + LlamaEngine Function()? createEngine, + }) : _createEngine = createEngine ?? (() => LlamaEngine(LlamaBackend())); + + final String model; + final String projector; + final GpuBackend backend; + final String pack; + final Uint8List? audio; + final double? audioSeconds; + final String? audioPath; + final String? reference; + final String text; + final Future Function(Uint8List) saveAudio; + final LlamaEngine Function() _createEngine; + LlamaEngine? _engine; + + /// Public diagnostics are selector hints, not accelerator execution proof. + Map observedRuntime = {}; + + @override + Future load() async { + if (!['stt', 'tts'].contains(pack)) { + throw ArgumentError('Unknown speech pack'); + } + if (pack == 'stt' && + (audio == null || + reference == null || + audioSeconds == null || + !audioSeconds!.isFinite || + audioSeconds! <= 0)) { + throw ArgumentError( + 'STT requires audio, transcript and measured duration', + ); + } + final engine = _engine = _createEngine(); + await engine.setLogLevel(LlamaLogLevel.info); + await engine.loadModel( + model, + modelParams: ModelParams( + contextSize: 4096, + preferredBackend: backend, + gpuLayers: backend == GpuBackend.cpu ? 0 : 99, + ), + ); + await engine.loadMultimodalProjector(projector); + observedRuntime = { + 'backend_name': await engine.getBackendName(), + 'resolved_gpu_layers': await engine.getResolvedGpuLayers(), + }; + } + + @override + Future dispose() async { + final engine = _engine; + _engine = null; + await engine?.dispose(); + } + + @override + Future> execute({ + bool cancel = false, + bool invalid = false, + bool bytesInput = false, + }) async { + final engine = _engine ?? (throw StateError('Speech engine is not loaded')); + final watch = Stopwatch()..start(); + if (pack == 'stt') { + final recognizer = SpeechToTextEngine( + engine, + modelProfile: SpeechToTextModelProfile.qwen3Asr, + ); + final capability = await recognizer.capabilities; + if (!capability.isSupported) { + throw StateError(capability.unsupportedReason ?? 'Speech unsupported'); + } + final task = await recognizer.transcribe( + SpeechToTextRequest( + audio: !invalid && !bytesInput && audioPath != null + ? SpeechAudioFileInput(audioPath!) + : SpeechAudioBytesInput( + invalid ? Uint8List(0) : audio!, + format: const SpeechAudioFormat(encoding: 'wav'), + ), + maxOutputTokens: 512, + ), + ); + if (cancel) task.cancel(); + final events = await task.events.toList(); + final completion = await task.done; + if (cancel) { + if (completion.state != SpeechToTextCompletionState.cancelled || + events.whereType().isNotEmpty) { + throw StateError('STT cancellation emitted a final result'); + } + return {'cancelled': true}; + } + if (completion.state != SpeechToTextCompletionState.completed || + events.whereType().length != 1) { + throw StateError('STT did not emit exactly one completed result'); + } + final transcript = completion.result!.text; + final wer = speechWordErrorRate(reference!, transcript); + return { + 'transcript': transcript, + 'reference': reference, + 'wer': wer, + 'predicate_passed': wer == 0, + 'elapsed_ms': watch.elapsedMicroseconds / 1000, + 'audio_seconds': audioSeconds, + 'real_time_factor': watch.elapsedMicroseconds / 1e6 / audioSeconds!, + 'first_partial_ms': null, + 'streaming_input': false, + }; + } + final synthesizer = TextToSpeechEngine( + engine, + modelProfile: TextToSpeechModelProfile.qwen3Tts, + ); + final capability = await synthesizer.capabilities; + if (!capability.isSupported) { + throw StateError(capability.unsupportedReason ?? 'Speech unsupported'); + } + final task = await synthesizer.synthesize( + TextToSpeechRequest( + text: invalid ? '' : text, + language: 'English', + maxFrames: 384, + seed: 1, + ), + ); + if (cancel) task.cancel(); + double? firstAudioMs; + var finals = 0; + await for (final event in task.events) { + if (event is TextToSpeechFinalEvent) { + firstAudioMs ??= watch.elapsedMicroseconds / 1000; + finals++; + } + } + final completion = await task.done; + if (cancel) { + if (completion.state != TextToSpeechCompletionState.cancelled || + finals != 0) { + throw StateError('TTS cancellation emitted a final result'); + } + return {'cancelled': true}; + } + if (completion.state != TextToSpeechCompletionState.completed || + finals != 1) { + throw StateError('TTS did not emit exactly one completed result'); + } + watch.stop(); + final result = completion.result!; + final metrics = inspectSpeechAudio(result); + await saveAudio(result.toWavBytes()); + return { + ...metrics, + 'predicate_passed': true, + 'elapsed_ms': watch.elapsedMicroseconds / 1000, + 'first_playable_audio_ms': firstAudioMs, + 'real_time_factor': + watch.elapsedMicroseconds / + 1e6 / + (metrics['audio_seconds'] as double), + 'streaming_audio': false, + }; + } +} + +/// Executes bounded speech lifecycle checks; cleanup failures remain failures. +/// +/// The result deliberately cannot assert hardware or perceptual qualification. +Future> runSpeechValidation( + SpeechValidationAdapter adapter, { + bool checkBytes = false, +}) async { + final results = >[]; + Future check( + String id, + Future> Function() action, + ) async { + try { + final result = await action(); + results.add({ + 'id': id, + 'status': result['predicate_passed'] == false ? 'FAIL' : 'PASS', + ...result, + }); + } catch (error) { + // Error classes are safe diagnostics; raw errors may contain local paths. + results.add({ + 'id': id, + 'status': 'FAIL', + 'error_type': '${error.runtimeType}', + 'message': redactDiagnostic('$error'), + }); + } + } + + try { + await check('load', () async { + await adapter.load(); + return {}; + }); + if (results.last['status'] == 'PASS') { + await check('generate', () => adapter.execute()); + if (checkBytes) { + await check('bytes_input', () => adapter.execute(bytesInput: true)); + } + await check('cancel', () async { + final result = await adapter.execute(cancel: true); + if (result['cancelled'] != true) { + throw StateError('Cancellation not confirmed'); + } + return result; + }); + await check('after_cancel', () => adapter.execute()); + await check('invalid_input', () async { + try { + await adapter.execute(invalid: true); + } on ArgumentError { + return {'rejected': true}; + } on LlamaAudioFormatException { + return {'rejected': true}; + } on LlamaTextToSpeechException catch (error) { + // This exception also represents synthesis failures. Only the exact + // empty-text contract exercised by this case is an input rejection. + if (error.message != 'Text to synthesize must not be empty.') rethrow; + return {'rejected': true}; + } on LlamaUnsupportedException { + return {'rejected': true}; + } + throw StateError('Invalid input did not produce a typed rejection'); + }); + await check('after_invalid', () => adapter.execute()); + await check('reload', () async { + await adapter.dispose(); + await adapter.load(); + return await adapter.execute(); + }); + } + } finally { + await check('dispose', () async { + await adapter.dispose(); + return {}; + }); + } + return { + 'schema_version': 1, + 'kind': 'speech_validation', + 'functional_pass': + results.length == (checkBytes ? 9 : 8) && + results.every((row) => row['status'] == 'PASS'), + 'qualified': false, + 'qualification_reason': + 'Requires reference, platform/accelerator and perceptual evidence; see individual cases.', + 'checks': results, + }; +} + +/// Duration from validated mono 16 kHz PCM16 RIFF/WAVE fixture bytes. +double speechFixtureSeconds(Uint8List bytes) { + final data = ByteData.sublistView(bytes); + String tag(int start) => + String.fromCharCodes(bytes.sublist(start, start + 4)); + if (bytes.length < 44 || + bytes.length > 5000000 || + tag(0) != 'RIFF' || + tag(8) != 'WAVE' || + data.getUint32(4, Endian.little) + 8 != bytes.length) { + throw const FormatException('Invalid bounded RIFF/WAVE fixture'); + } + var formatSeen = false; + int? pcmBytes; + var offset = 12; + while (offset + 8 <= bytes.length) { + final size = data.getUint32(offset + 4, Endian.little); + final start = offset + 8; + if (start + size > bytes.length) { + throw const FormatException('Truncated WAV chunk'); + } + if (tag(offset) == 'fmt ') { + if (formatSeen || + size < 16 || + data.getUint16(start, Endian.little) != 1 || + data.getUint16(start + 2, Endian.little) != 1 || + data.getUint32(start + 4, Endian.little) != 16000 || + data.getUint32(start + 8, Endian.little) != 32000 || + data.getUint16(start + 12, Endian.little) != 2 || + data.getUint16(start + 14, Endian.little) != 16) { + throw const FormatException('Expected mono 16 kHz PCM16 WAV'); + } + formatSeen = true; + } + if (tag(offset) == 'data') { + if (pcmBytes != null || size == 0 || size.isOdd) { + throw const FormatException('Invalid PCM data'); + } + pcmBytes = size; + } + offset = start + size + (size.isOdd ? 1 : 0); + } + if (!formatSeen || pcmBytes == null || offset != bytes.length) { + throw const FormatException('Incomplete WAV fixture'); + } + return pcmBytes / 32000; +} + +/// Dedicated native CPU ASR: PCM streaming is separate from GGUF prompt ASR. +class PublicDedicatedSpeechAdapter implements SpeechValidationAdapter { + PublicDedicatedSpeechAdapter({ + required this.config, + required this.wav, + required this.reference, + SpeechToTextEngine Function(LiteRtLmAsrRuntimeConfig)? createRecognizer, + }) : _createRecognizer = createRecognizer ?? SpeechToTextEngine.liteRtLm; + final SpeechToTextEngine Function(LiteRtLmAsrRuntimeConfig) _createRecognizer; + final LiteRtLmAsrRuntimeConfig config; + final Uint8List wav; + final String reference; + SpeechToTextEngine? _recognizer; + SpeechToTextStreamingSession? _active; + late Float32List _pcm; + late double _seconds; + + @override + Future load() async { + _seconds = speechFixtureSeconds(wav); + final data = ByteData.sublistView(wav); + var offset = 12; + while (offset + 8 <= wav.length) { + final size = data.getUint32(offset + 4, Endian.little); + if (String.fromCharCodes(wav.sublist(offset, offset + 4)) == 'data') { + _pcm = Float32List(size ~/ 2); + for (var i = 0; i < _pcm.length; i++) { + _pcm[i] = data.getInt16(offset + 8 + i * 2, Endian.little) / 32768; + } + break; + } + offset += 8 + size + (size.isOdd ? 1 : 0); + } + _recognizer = _createRecognizer(config); + final capabilities = await _recognizer!.capabilities; + if (!capabilities.isSupported) { + throw StateError(capabilities.unsupportedReason ?? 'ASR unsupported'); + } + } + + @override + Future dispose() async { + await _active?.cancel(); + _active = null; + _recognizer = null; + } + + @override + Future> execute({ + bool cancel = false, + bool invalid = false, + bool bytesInput = false, + }) async { + final watch = Stopwatch()..start(); + final recognizer = _recognizer ?? (throw StateError('ASR is not loaded')); + if (invalid) { + final rejected = await recognizer.startStream( + format: const SpeechAudioFormat( + sampleRateHz: 8000, + channelCount: 1, + encoding: 'pcm-f32le', + ), + ); + await rejected.cancel(); + throw StateError('Unsupported PCM format was accepted'); + } + final session = _active = await recognizer.startStream(); + double? firstPartial; + var partials = 0; + var finals = 0; + final drained = Completer(); + Object? streamError; + final events = session.events.listen( + (event) { + if (event is SpeechToTextPartialEvent) { + firstPartial ??= watch.elapsedMicroseconds / 1000; + partials++; + } + if (event is SpeechToTextFinalEvent) finals++; + }, + onError: (Object error) { + streamError = error; + }, + onDone: drained.complete, + ); + try { + if (cancel) { + await session.cancel(); + } else { + for (var offset = 0; offset < _pcm.length; offset += 1600) { + final end = offset + 1600 < _pcm.length ? offset + 1600 : _pcm.length; + await session.addPcm(Float32List.sublistView(_pcm, offset, end)); + } + await session.finish(); + } + final completion = await session.done; + await drained.future; + if (streamError != null) throw streamError!; + if (cancel) { + if (completion.state != SpeechToTextCompletionState.cancelled || + finals != 0) { + throw StateError('ASR cancellation did not complete cleanly'); + } + return {'cancelled': true}; + } + if (completion.state != SpeechToTextCompletionState.completed || + completion.result == null || + finals != 1) { + throw StateError('ASR stream did not complete'); + } + final text = completion.result!.text; + final wer = speechWordErrorRate(reference, text); + return { + 'transcript': text, + 'reference': reference, + 'wer': wer, + 'predicate_passed': wer == 0, + 'first_partial_ms': firstPartial, + 'partial_events': partials, + 'elapsed_ms': watch.elapsedMicroseconds / 1000, + 'audio_seconds': _seconds, + 'real_time_factor': watch.elapsedMicroseconds / 1e6 / _seconds, + 'backend': 'cpu', + 'streaming_input': true, + }; + } finally { + await session.cancel(); + await events.cancel(); + _active = null; + } + } +} diff --git a/packages/llamadart_validation/lib/src/voice_runner.dart b/packages/llamadart_validation/lib/src/voice_runner.dart new file mode 100644 index 000000000..6893baca8 --- /dev/null +++ b/packages/llamadart_validation/lib/src/voice_runner.dart @@ -0,0 +1,56 @@ +import 'speech_runner.dart'; + +/// One file-input STT -> chat -> TTS operation through injected public adapters. +/// Physical microphone/speaker behavior requires its own device evidence. +Future> runVoiceRoundTrip({ + required SpeechValidationAdapter recognizer, + required Future Function(String transcript) respond, + required SpeechValidationAdapter Function(String response) synthesizer, +}) async { + SpeechValidationAdapter? speaker; + final watch = Stopwatch()..start(); + final result = { + 'schema_version': 1, + 'kind': 'voice_round_trip', + 'functional_pass': false, + 'qualified': false, + 'microphone': 'NOT_RUN', + 'speaker_playback': 'NOT_RUN', + 'listening_check': 'NOT_RUN', + }; + try { + await recognizer.load(); + final transcription = await recognizer.execute(); + result['stt'] = transcription; + if (transcription['predicate_passed'] != true) { + throw StateError('STT reference predicate failed'); + } + final transcript = transcription['transcript'] as String; + if (transcript.trim().isEmpty) throw StateError('Empty transcript'); + await recognizer.dispose(); + final response = await respond(transcript); + if (response.trim().isEmpty) throw StateError('Empty chat response'); + result['response'] = response; + speaker = synthesizer(response); + await speaker.load(); + final synthesis = await speaker.execute(); + result['tts'] = synthesis; + if (synthesis['predicate_passed'] != true) { + throw StateError('TTS output predicate failed'); + } + result['functional_pass'] = true; + } catch (error) { + result['error_type'] = '${error.runtimeType}'; + } finally { + for (final adapter in [recognizer, ?speaker]) { + try { + await adapter.dispose(); + } catch (error) { + result['functional_pass'] = false; + result['cleanup_error_type'] = '${error.runtimeType}'; + } + } + result['elapsed_ms'] = watch.elapsedMicroseconds / 1000; + } + return result; +} diff --git a/packages/llamadart_validation/pubspec.lock b/packages/llamadart_validation/pubspec.lock new file mode 100644 index 000000000..036e97340 --- /dev/null +++ b/packages/llamadart_validation/pubspec.lock @@ -0,0 +1,460 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: fdcd9f70f9eb80df3bc5ed0fa67280df2595fd481948df8a9ab082e6a40ad04b + url: "https://pub.dev" + source: hosted + version: "108.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: a51c769bff3b6dfbe9d60199b8606d808290702a296bef0c26a4ca391d414e46 + url: "https://pub.dev" + source: hosted + version: "14.4.0" + archive: + dependency: transitive + description: + name: archive + sha256: "6c5bcd986e06b94e3c40244af471750840a3d2341d1f9763a1100a14add517b4" + url: "https://pub.dev" + source: hosted + version: "4.3.0" + args: + dependency: transitive + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: e2eb0491ba5ddb6177742d2da23904574082139b07c1e33b8503b9f46f3e1a37 + url: "https://pub.dev" + source: hosted + version: "2.13.1" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + cli_config: + dependency: transitive + description: + name: cli_config + sha256: ac20a183a07002b700f0c25e61b7ee46b23c309d76ab7b7640a028f18e4d99ec + url: "https://pub.dev" + source: hosted + version: "0.2.0" + code_assets: + dependency: transitive + description: + name: code_assets + sha256: "828110d598123b5ea96c00c9f3c72105bf79f8ee36c20a39b26209ade421ec57" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + collection: + dependency: transitive + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + coverage: + dependency: transitive + description: + name: coverage + sha256: "956a3de0725ca232ad353565a8290d3357592bf4250f6f298a185e2d949c5d3d" + url: "https://pub.dev" + source: hosted + version: "1.15.1" + crypto: + dependency: "direct main" + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.dev" + source: hosted + version: "3.0.7" + dinja: + dependency: transitive + description: + name: dinja + sha256: "34d4e569ceb3d900ab061f16cae947223968cc63d2484a8a4710d5ecfa4ae3a2" + url: "https://pub.dev" + source: hosted + version: "1.0.0" + ffi: + dependency: "direct main" + description: + name: ffi + sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694 + url: "https://pub.dev" + source: hosted + version: "4.0.0" + glob: + dependency: transitive + description: + name: glob + sha256: "218aeb56050c714f62a3182775320dfa04602b55074873e24e31bbd39bda96fb" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + hooks: + dependency: transitive + description: + name: hooks + sha256: eaac480a35ec0814146c2c48d96aaa829e0e44a7662c88ae84c9edf4bc35651f + url: "https://pub.dev" + source: hosted + version: "2.2.0" + http: + dependency: "direct main" + description: + name: http + sha256: "87721a4a50b19c7f1d49001e51409bddc46303966ce89a65af4f4e6004896412" + url: "https://pub.dev" + source: hosted + version: "1.6.0" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: "2635216ca6a737e60de577ffa1a48a0bec76ca8a62917cfc1bb88c14c570646f" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + lints: + dependency: "direct dev" + description: + name: lints + sha256: "12f842a479589fea194fe5c5a3095abc7be0c1f2ddfa9a0e76aed1dbd26a87df" + url: "https://pub.dev" + source: hosted + version: "6.1.0" + llamadart: + dependency: "direct main" + description: + path: "../.." + relative: true + source: path + version: "0.8.23" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "31bd099b47c10cd1aeb55146a2d46ce0277630ecef3f7dae54ad7873f36696cd" + url: "https://pub.dev" + source: hosted + version: "0.12.20" + meta: + dependency: transitive + description: + name: meta + sha256: "307249ce4ff29d58a18e97f6345f539382eb9c9c29ecda628900f31de0443dd9" + url: "https://pub.dev" + source: hosted + version: "1.19.0" + mime: + dependency: transitive + description: + name: mime + sha256: bd47de35f07e27267e69c8c8b22edf9473bfee170a60d60fcc93730c5144b7f6 + url: "https://pub.dev" + source: hosted + version: "2.1.0" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + package_config: + dependency: transitive + description: + name: package_config + sha256: ffcf4cf3d6c0b74ac43708d9f56625506e8a68aa935abe9d267a7330f320eb5d + url: "https://pub.dev" + source: hosted + version: "3.0.0" + path: + dependency: "direct main" + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + pool: + dependency: transitive + description: + name: pool + sha256: "4177f68c237ea2128d1bee66ac17b2ce05ba3dbaafcbdd54c5d40a39d0b6b11c" + url: "https://pub.dev" + source: hosted + version: "1.5.3" + posix: + dependency: transitive + description: + name: posix + sha256: bc1bad54ad2b735816e31f8d4600cfde6c7839975085ddfbca48b6c9f7c4044e + url: "https://pub.dev" + source: hosted + version: "6.5.2" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "261236774e8b1d69cfc6b9eabbc96c40f25e7a2d6b171f3385d4f65d5734fb24" + url: "https://pub.dev" + source: hosted + version: "2.2.1" + record_use: + dependency: transitive + description: + name: record_use + sha256: "1cb8564af8d43b464294411db9217f5ec04891c6f22ee2c32d73ae05e88a6bd2" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: c87c3875f91262785dade62d135760c2c69cb217ac759485334c5857ad89f6e3 + url: "https://pub.dev" + source: hosted + version: "1.1.3" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: c0713a43e323c3302c2abe2a1cc89aa057a387101ebd280371d6a6c9fa68516b + url: "https://pub.dev" + source: hosted + version: "2.1.2" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "14c2945847669b44089bb1222f66873d7ff7103c58911917f2a63c5a62327898" + url: "https://pub.dev" + source: hosted + version: "0.10.14" + source_span: + dependency: transitive + description: + name: source_span + sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab" + url: "https://pub.dev" + source: hosted + version: "1.10.2" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "277654b3034d17ac6f9f1cb5595db011b1d5d41e8806866db28e0abaa101c490" + url: "https://pub.dev" + source: hosted + version: "1.12.2" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test: + dependency: "direct dev" + description: + name: test + sha256: de5d145b0afff7921e5e788a880f52d7e5f3ae24068a202f6fd3b58e4ba26323 + url: "https://pub.dev" + source: hosted + version: "1.32.0" + test_api: + dependency: transitive + description: + name: test_api + sha256: "0a10344e901e5b2e63819567951cb6a06673ed6b84f40462188ff5a0c41f371f" + url: "https://pub.dev" + source: hosted + version: "0.7.14" + test_core: + dependency: transitive + description: + name: test_core + sha256: "80f3fb49087454e07e7e07c67578cfdd156c8c3a5227d8b3f47c7b2d019c2e93" + url: "https://pub.dev" + source: hosted + version: "0.6.20" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "5f37239c4851efcef929cea7824e76df7f2f0970aef85d66bbc430afa40e72f0" + url: "https://pub.dev" + source: hosted + version: "15.3.0" + watcher: + dependency: transitive + description: + name: watcher + sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.dev" + source: hosted + version: "3.0.3" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + yaml: + dependency: transitive + description: + name: yaml + sha256: f67cdd8e07d3c6329146aaef1ba043542b3134c12489f553ca9a7435d1068aea + url: "https://pub.dev" + source: hosted + version: "3.1.4" +sdks: + dart: ">=3.11.0 <4.0.0" diff --git a/packages/llamadart_validation/pubspec.yaml b/packages/llamadart_validation/pubspec.yaml new file mode 100644 index 000000000..32a319dd2 --- /dev/null +++ b/packages/llamadart_validation/pubspec.yaml @@ -0,0 +1,30 @@ +name: llamadart_validation +description: Private cross-platform validation harness for the public llamadart API. +publish_to: none +environment: + sdk: ^3.10.7 +dependencies: + llamadart: + path: ../.. + crypto: ^3.0.0 + ffi: ^2.1.0 + http: ^1.1.0 + path: ^1.8.3 +dev_dependencies: + lints: ^6.1.0 + test: ^1.26.3 +flutter: + assets: + - assets/profiles/ +hooks: + user_defines: + llamadart: + llamadart_native_runtimes: + runtimes: [llama_cpp, litert_lm] + platforms: + ios-x86_64-sim: [llama_cpp] + windows-arm64: [llama_cpp] + llamadart_native_backends: + platforms: + linux-x64: [cpu, vulkan, cuda] + windows-x64: [cpu, vulkan, cuda] diff --git a/packages/llamadart_validation/schemas/event.schema.json b/packages/llamadart_validation/schemas/event.schema.json new file mode 100644 index 000000000..e20419a1e --- /dev/null +++ b/packages/llamadart_validation/schemas/event.schema.json @@ -0,0 +1,127 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "llamadart validation journal events v1 and v2", + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "enum": [ + "manifest", + "case_start", + "case", + "cleanup", + "run_end", + "preparation_error" + ] + }, + "sequence": { + "type": "integer", + "minimum": 0 + }, + "status": { + "enum": [ + "PASS", + "FAIL", + "ERROR", + "UNSUPPORTED", + "NOT_RUN" + ] + }, + "case_id": { + "type": "string" + }, + "schema_version": { + "enum": [ + 1, + 2 + ] + }, + "case_version": { + "type": "integer", + "minimum": 1 + }, + "fixture_hash": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "manifest" + } + } + }, + "then": { + "required": [ + "schema_version", + "run_id", + "profile", + "profile_hash", + "effective_config", + "config_hash", + "environment", + "preparation", + "case_ids" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "case" + } + } + }, + "then": { + "required": [ + "case_id", + "status", + "sequence" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "run_end" + } + } + }, + "then": { + "required": [ + "sequence", + "cancelled" + ] + } + }, + { + "if": { + "required": [ + "type", + "schema_version" + ], + "properties": { + "type": { + "const": "manifest" + }, + "schema_version": { + "const": 2 + } + } + }, + "then": { + "required": [ + "catalog", + "catalog_hash" + ] + } + } + ] +} diff --git a/packages/llamadart_validation/schemas/profile.schema.json b/packages/llamadart_validation/schemas/profile.schema.json new file mode 100644 index 000000000..8a61ff366 --- /dev/null +++ b/packages/llamadart_validation/schemas/profile.schema.json @@ -0,0 +1,148 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "llamadart validation profile v1", + "type": "object", + "required": [ + "schema_version", + "id", + "runtime", + "backend", + "model" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "id": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]{0,63}$" + }, + "runtime": { + "enum": [ + "gguf", + "litert" + ] + }, + "backend": { + "type": "string" + }, + "selection": { + "enum": [ + "quick", + "focused", + "release" + ] + }, + "execution_path": { + "enum": [ + "public_api", + "native_c_api" + ] + }, + "enable_thinking": { + "type": "boolean" + }, + "history_controls": { + "type": "boolean" + }, + "context_size": { + "type": "integer", + "minimum": 128, + "maximum": 8192 + }, + "threads": { + "type": "integer", + "minimum": 1, + "maximum": 32 + }, + "max_tokens": { + "type": "integer", + "minimum": 1, + "maximum": 512 + }, + "model": { + "type": "object", + "required": [ + "revision", + "filename", + "sha256", + "bytes", + "kind", + "url" + ], + "properties": { + "revision": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "filename": { + "type": "string", + "pattern": "^[a-zA-Z0-9_.-]+$" + }, + "bytes": { + "type": "integer", + "minimum": 1 + }, + "kind": { + "enum": [ + "raw", + "chat" + ] + }, + "url": { + "type": "string", + "pattern": "^https://" + } + } + }, + "focus_features": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": [ + "text", + "unicode", + "thinking", + "history", + "tools", + "streaming", + "batching", + "lifecycle", + "guards", + "performance" + ] + } + } + }, + "allOf": [ + { + "if": { + "required": [ + "selection" + ], + "properties": { + "selection": { + "const": "focused" + } + } + }, + "then": { + "required": [ + "focus_features" + ] + }, + "else": { + "not": { + "required": [ + "focus_features" + ] + } + } + } + ] +} diff --git a/packages/llamadart_validation/test/coverage_catalog_test.dart b/packages/llamadart_validation/test/coverage_catalog_test.dart new file mode 100644 index 000000000..baa884a32 --- /dev/null +++ b/packages/llamadart_validation/test/coverage_catalog_test.dart @@ -0,0 +1,146 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:test/test.dart'; + +import '../../../tool/testing/validation/coverage_catalog.dart'; + +void main() { + test('primary runnable rows bind the intended model and backend', () { + for (final row in validationCoverage().where( + (row) => row['profile'] != null && row['priority'] == 'primary', + )) { + final profile = jsonDecode( + File('assets/profiles/${row['profile']}.json').readAsStringSync(), + ); + expect(profile['runtime'], row['runtime']); + expect(profile['backend'], row['backend']); + expect( + (profile['model']['filename'] as String).toLowerCase(), + startsWith(row['model'] == 'gemma4-e2b' ? 'gemma-4' : 'qwen3.5'), + ); + expect(row['status'], 'NOT_RUN'); + } + }); + + test('coverage never fabricates qualification or duplicate identities', () { + final rows = validationCoverage(); + expect(rows.map((row) => row['id']).toSet(), hasLength(rows.length)); + for (final row in rows) { + expect(row['status'], isIn(['NOT_RUN', 'UNVERIFIED', 'UNSUPPORTED'])); + expect(row['qualification'], 'NO_EVIDENCE_IN_CATALOG'); + expect(row['reason'], isNotEmpty); + } + }); + + test('Gemma4 NPU candidates cannot resolve legacy executable profiles', () { + final rows = validationCoverage().where( + (row) => + row['backend'] == 'npu' && + row['platform'] == 'android-arm64' && + (row['model'] as String).startsWith('gemma4'), + ); + expect( + rows.map((row) => row['target_soc']), + unorderedEquals(['tensor-g5', 'qualcomm-sm8750', 'qualcomm-sm8650']), + ); + for (final row in rows) { + expect(row['status'], 'UNVERIFIED'); + expect(row['profile'], isNull); + } + for (final row in validationCoverage().where( + (row) => row['priority'] == 'legacy-control', + )) { + final profile = + jsonDecode( + File( + 'assets/profiles/${row['profile']}.json', + ).readAsStringSync(), + ) + as Map; + expect(profile['backend'], 'npu'); + expect(profile['model']['id'], startsWith('gemma3')); + } + }); + + test('speech cannot inherit chat GPU or NPU support', () { + for (final row in validationCoverage().where( + (row) => row['runtime'] == 'litert' && row['use_case'] != 'chat', + )) { + final cpuAsr = + row['backend'] == 'cpu' && + row['use_case'] == 'stt' && + !['web', 'windows-arm64'].contains(row['platform']); + expect(row['status'], cpuAsr ? 'NOT_RUN' : 'UNSUPPORTED'); + } + }); + + test('Apple desktop and browser NPU remain explicitly unsupported', () { + final rows = validationCoverage().where( + (row) => row['backend'] == 'npu' && row['platform'] != 'android-arm64', + ); + expect(rows, isNotEmpty); + for (final row in rows) { + expect(row['status'], 'UNSUPPORTED'); + } + }); + + test('CLI filters real coverage rows and rejects unknown selectors', () async { + // CI prepares only this private package. The root CLI must use its + // explicit package configuration, not depend on a prepared root checkout. + final isolated = Directory.systemTemp.createTempSync('coverage-cli-'); + addTearDown(() => isolated.deleteSync(recursive: true)); + Directory('${isolated.path}/tool/testing').createSync(recursive: true); + File( + '../../tool/testing/validation.dart', + ).copySync('${isolated.path}/tool/testing/validation.dart'); + Directory('${isolated.path}/tool/testing/validation').createSync(); + for (final source in Directory( + '../../tool/testing/validation', + ).listSync().whereType()) { + if (source.path.endsWith('.dart')) { + source.copySync( + '${isolated.path}/tool/testing/validation/${source.uri.pathSegments.last}', + ); + } + } + final library = Directory( + '${isolated.path}/packages/llamadart_validation/lib/src', + )..createSync(recursive: true); + for (final name in [ + 'runtime_environment.dart', + 'runtime_environment_io.dart', + 'runtime_environment_stub.dart', + ]) { + File('lib/src/$name').copySync('${library.path}/$name'); + } + final command = [ + '--packages=${File('.dart_tool/package_config.json').absolute.path}', + '${isolated.path}/tool/testing/validation.dart', + 'coverage', + ]; + final result = await Process.run(Platform.resolvedExecutable, [ + ...command, + '--platform', + 'android-arm64', + '--backend', + 'npu', + ]); + expect(result.exitCode, 0, reason: '${result.stderr}'); + final rows = (jsonDecode(result.stdout as String) as Map)['rows'] as List; + expect(rows, isNotEmpty); + expect( + rows.every( + (dynamic row) => + row['backend'] == 'npu' && row['platform'] == 'android-arm64', + ), + isTrue, + ); + final bad = await Process.run(Platform.resolvedExecutable, [ + ...command, + '--backend', + 'nonexistent', + ]); + expect(bad.exitCode, isNot(0)); + }); +} diff --git a/packages/llamadart_validation/test/dedicated_speech_test.dart b/packages/llamadart_validation/test/dedicated_speech_test.dart new file mode 100644 index 000000000..85f229f5d --- /dev/null +++ b/packages/llamadart_validation/test/dedicated_speech_test.dart @@ -0,0 +1,116 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:llamadart/llamadart.dart'; +import 'package:llamadart_validation/src/speech_runner.dart'; +import 'package:test/test.dart'; + +class Recognizer implements SpeechToTextEngine { + final session = Session(); + @override + Future get capabilities async => + const SpeechToTextCapabilities(isSupported: true); + @override + Future startStream({ + SpeechAudioFormat? format, + }) async { + if (format?.sampleRateHz == 8000) throw ArgumentError('unsupported'); + return session; + } + + @override + dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation); +} + +class Session implements SpeechToTextStreamingSession { + final controller = StreamController(); + final completion = Completer(); + int samples = 0; + int largestPush = 0; + bool omitFinal = false; + bool emitError = false; + bool cancelCalled = false; + @override + Stream get events => controller.stream; + @override + Future get done => completion.future; + @override + Future addPcm(Float32List pcm) async { + samples += pcm.length; + if (pcm.length > largestPush) largestPush = pcm.length; + } + + @override + Future finish() async { + const result = SpeechToTextResult(text: 'expected'); + completion.complete(SpeechToTextCompletion.completed(result)); + // Deliberately deliver after completion, testing stream draining. + scheduleMicrotask(() { + if (emitError) controller.addError(StateError('stream failure')); + if (!omitFinal) controller.add(const SpeechToTextFinalEvent(result)); + controller.close(); + }); + } + + @override + Future cancel() async { + cancelCalled = true; + if (!completion.isCompleted) { + completion.complete(const SpeechToTextCompletion.cancelled()); + await controller.close(); + } + } +} + +void main() { + PublicDedicatedSpeechAdapter adapter(Recognizer recognizer) => + PublicDedicatedSpeechAdapter( + config: const LiteRtLmAsrRuntimeConfig( + modelPath: 'fixture', + tokenizerPath: 'fixture', + modelPreset: LiteRtLmAsrModelPreset.moonshineTiny, + ), + wav: File('assets/speech/jfk.wav').readAsBytesSync(), + reference: 'expected', + createRecognizer: (_) => recognizer, + ); + test( + 'CPU adapter pushes bounded PCM and drains delayed final before passing', + () async { + final engine = Recognizer(); + final target = adapter(engine); + await target.load(); + final result = await target.execute(); + expect(result['predicate_passed'], true); + expect(engine.session.samples, 176000); + expect(engine.session.largestPush, 1600); + expect(engine.session.cancelCalled, true); + await target.dispose(); + }, + ); + test( + 'missing final and stream errors cannot be masked by completed future', + () async { + for (final error in [false, true]) { + final engine = Recognizer(); + engine.session.omitFinal = !error; + engine.session.emitError = error; + final target = adapter(engine); + await target.load(); + await expectLater(target.execute(), throwsStateError); + expect(engine.session.cancelCalled, true); + await target.dispose(); + } + }, + ); + test( + 'invalid sample-rate is exercised through public streaming contract', + () async { + final target = adapter(Recognizer()); + await target.load(); + await expectLater(target.execute(invalid: true), throwsArgumentError); + await target.dispose(); + }, + ); +} diff --git a/packages/llamadart_validation/test/desktop_bundle_test.dart b/packages/llamadart_validation/test/desktop_bundle_test.dart new file mode 100644 index 000000000..0317543cc --- /dev/null +++ b/packages/llamadart_validation/test/desktop_bundle_test.dart @@ -0,0 +1,215 @@ +import 'dart:convert'; +import 'dart:ffi'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:llamadart/src/backends/litert_lm/litert_lm_runtime.dart'; +import 'package:llamadart_validation/src/desktop_bundle.dart'; +import 'package:llamadart_validation/src/runtime_environment.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +void main() { + late Directory root; + late Map identity; + late String runtimePath; + void put(String name, String content) { + final file = File(p.join(root.path, name)); + file.parent.createSync(recursive: true); + file.writeAsStringSync(content); + } + + void seal() { + put('environment.json', jsonEncode(identity)); + final files = {}; + for (final file in root.listSync(recursive: true).whereType()) { + final name = p.relative(file.path, from: root.path).replaceAll('\\', '/'); + if (name == 'bundle-manifest.json') continue; + files[name] = { + 'sha256': sha256.convert(file.readAsBytesSync()).toString(), + 'bytes': file.lengthSync(), + }; + } + put( + 'bundle-manifest.json', + jsonEncode({'schema_version': 1, ...identity, 'files': files}), + ); + } + + setUp(() { + root = Directory.systemTemp.createTempSync('validation-bundle-test-'); + final abi = Abi.current(); + final libraries = liteRtLmRequiredLibrariesForAbi(abi); + runtimePath = libraries.isEmpty + ? '' + : '.dart_tool/llamadart/litert_lm/0.17.0-5/${liteRtLmCacheDirectoryCandidatesForAbi(abi).singleWhere((candidate) => candidate.contains('/'))}'; + identity = { + 'target': 'desktop', + 'build_os': Platform.operatingSystem, + 'build_abi': abi.toString(), + 'litert_tag': '0.17.0-5', + 'litert_runtime_supported': libraries.isNotEmpty, + if (libraries.isNotEmpty) 'litert_runtime_layout': runtimePath, + }; + put( + 'bin/llamadart-validate${Platform.isWindows ? '.exe' : ''}', + 'executable', + ); + put( + 'lib/${Platform.isMacOS + ? 'libllamadart.dylib' + : Platform.isWindows + ? 'llamadart.dll' + : 'libllamadart.so'}', + 'native', + ); + for (final library in libraries) { + put('$runtimePath/$library', library); + } + seal(); + }); + tearDown(() => root.deleteSync(recursive: true)); + + test( + 'verified payload tolerates results without weakening library inventory', + () async { + put('results/events.jsonl', 'journal'); + put('model-cache/weights.gguf', 'model'); + final value = await verifyDesktopValidationBundle( + root, + environment: const {}, + ); + expect(value['runtime_payload_verified'], true); + expect(value['runtime_bundle_sha256'], matches(r'^[a-f0-9]{64}$')); + // Preserve upstream platform-suffixed primary library filenames. + final native = + Directory(p.join(root.path, 'lib')).listSync().single as File; + native.renameSync( + p.join(native.parent.path, 'llamadart-windows-x64.dll'), + ); + seal(); + expect( + (await verifyDesktopValidationBundle( + root, + environment: const {}, + ))['runtime_payload_verified'], + true, + ); + }, + ); + test('modified runtime bytes fail before inference', () async { + final native = + Directory(p.join(root.path, 'lib')).listSync().single as File; + native.writeAsStringSync('changed'); + await expectLater( + verifyDesktopValidationBundle(root, environment: const {}), + throwsFormatException, + ); + }); + test('unlisted runtime sidecars fail before discovery', () async { + for (final name in [ + 'lib/extra.dll', + '.dart_tool/lib/extra.so', + 'Frameworks/extra.dylib', + 'extra.dll', + ]) { + put(name, 'unchecked'); + await expectLater( + verifyDesktopValidationBundle(root, environment: const {}), + throwsFormatException, + ); + File(p.join(root.path, name)).deleteSync(); + } + }); + test( + 'self-consistent incomplete LiteRT inventory cannot use ancestor cache', + () async { + if (runtimePath.isEmpty) return; + (Directory(p.join(root.path, runtimePath)).listSync().first as File) + .deleteSync(); + seal(); + await expectLater( + verifyDesktopValidationBundle(root, environment: const {}), + throwsFormatException, + ); + }, + ); + test('rehashed environment cannot contradict bundle provenance', () async { + put('environment.json', jsonEncode({...identity, 'litert_tag': 'other'})); + final file = File(p.join(root.path, 'bundle-manifest.json')); + final manifest = jsonDecode(file.readAsStringSync()) as Map; + final environment = File(p.join(root.path, 'environment.json')); + (manifest['files'] as Map)['environment.json'] = { + 'sha256': sha256.convert(environment.readAsBytesSync()).toString(), + 'bytes': environment.lengthSync(), + }; + file.writeAsStringSync(jsonEncode(manifest)); + await expectLater( + verifyDesktopValidationBundle(root, environment: const {}), + throwsFormatException, + ); + }); + test('runtime override diagnostics expose names only', () { + requireValidationRuntimeEnvironment( + environment: {'PATH': '/usual/path'}, + portable: true, + ); + for (final key in [ + 'LLAMADART_LITERT_LM_LIB_DIR', + 'LLAMADART_NATIVE_LIB_DIR', + 'LLAMADART_BACKEND_MODULE_DIR', + 'LD_PRELOAD', + 'DYLD_LIBRARY_PATH', + 'WEBGPU_BRIDGE_ASSETS_TAG', + ]) { + expect( + () => requireValidationRuntimeEnvironment( + environment: {key: '/secret/token'}, + portable: true, + ), + throwsA( + isA().having( + (error) => '$error', + 'diagnostic', + allOf(contains(key), isNot(contains('/secret/token'))), + ), + ), + ); + } + }); + test( + 'public engine refuses ambient override before creating native engine', + () async { + final probe = File(p.join(root.path, 'override_probe.dart')) + ..writeAsStringSync(''' +import 'dart:convert'; +import 'dart:io'; +import 'package:llamadart_validation/llamadart_validation.dart'; +Future main() async { + final profile = ValidationProfile.fromJson(jsonDecode(File('assets/profiles/chat-litert-cpu.json').readAsStringSync())); + final engine = PublicValidationEngine(engineFactory: () => throw StateError('ENGINE_WAS_CREATED')); + try { + await engine.load('missing-model', profile); + throw StateError('OVERRIDE_ACCEPTED'); + } on StateError catch (error) { + if (!error.message.toString().contains('unset LLAMADART_LITERT_LM_LIB_DIR')) rethrow; + stdout.write('override rejected before engine creation'); + } +} +'''); + final result = await Process.run( + Platform.resolvedExecutable, + [ + '--packages=${p.absolute('.dart_tool/package_config.json')}', + probe.path, + ], + environment: {'LLAMADART_LITERT_LM_LIB_DIR': '/unverified/runtime'}, + ); + expect(result.exitCode, 0, reason: '${result.stdout}\n${result.stderr}'); + expect( + result.stdout, + contains('override rejected before engine creation'), + ); + }, + ); +} diff --git a/packages/llamadart_validation/test/model_io_test.dart b/packages/llamadart_validation/test/model_io_test.dart new file mode 100644 index 000000000..ec0d1cfbf --- /dev/null +++ b/packages/llamadart_validation/test/model_io_test.dart @@ -0,0 +1,245 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; +import 'package:llamadart_validation/io.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +class BlockedClient extends http.BaseClient { + final response = Completer(); + bool closed = false; + @override + Future send(http.BaseRequest request) => + response.future; + @override + void close() { + closed = true; + if (!response.isCompleted) { + response.completeError(http.ClientException('cancelled')); + } + } +} + +class SlowModelClient extends http.BaseClient { + final body = StreamController>(); + bool closed = false; + + @override + Future send(http.BaseRequest request) async { + body.add([1, 2]); + return http.StreamedResponse(body.stream, 200); + } + + @override + void close() { + if (closed) return; + closed = true; + body.addError(http.ClientException('Connection closed by transport')); + unawaited(body.close()); + } +} + +void main() { + late Directory cache; + late ValidationProfile profile; + final bytes = utf8.encode('model fixture bytes'); + setUp(() { + cache = Directory.systemTemp.createTempSync('validation-model-test-'); + final json = + jsonDecode( + File('assets/profiles/tiny-gguf-cpu.json').readAsStringSync(), + ) + as Map; + json['model']['sha256'] = sha256.convert(bytes).toString(); + json['model']['bytes'] = bytes.length; + profile = ValidationProfile.fromJson(json); + }); + tearDown(() => cache.deleteSync(recursive: true)); + test('verified cache hit never downloads again', () async { + var calls = 0; + final first = await prepareModel( + profile, + cache, + client: MockClient((_) async { + calls++; + return http.Response.bytes(bytes, 200); + }), + timeout: const Duration(minutes: 10), + ); + expect(first.evidence['verified'], true); + expect(first.evidence['download_timeout_ms'], 600000); + final second = await prepareModel( + profile, + cache, + client: MockClient((_) async => throw StateError('unexpected network')), + ); + expect(second.evidence['cache_hit'], true); + expect(calls, 1); + }); + test( + 'oversized or wrong-hash download leaves no partial or cached model', + () async { + for (final content in [ + List.filled(bytes.length, 0), + [...bytes, 0], + ]) { + await expectLater( + prepareModel( + profile, + cache, + client: MockClient((_) async => http.Response.bytes(content, 200)), + ), + throwsFormatException, + ); + expect(cache.listSync(recursive: true).whereType(), isEmpty); + } + }, + ); + test( + 'preparation progress survives failure without claiming verification', + () async { + final events = >[]; + await expectLater( + prepareModel( + profile, + cache, + client: MockClient( + (_) async => + http.Response.bytes(List.filled(bytes.length, 0), 200), + ), + onProgress: (event) async => events.add(event), + ), + throwsFormatException, + ); + expect(events.map((e) => '${e['stage']}:${e['state']}'), [ + 'download:started', + 'download:finished', + 'checksum:started', + 'checksum:rejected', + ]); + expect(events.last['bytes'], bytes.length); + expect( + events.every((e) => e['model_sha256'] == profile.modelHash), + isTrue, + ); + expect(jsonEncode(events), isNot(contains(cache.path))); + expect(jsonEncode(events), isNot(contains(profile.model['url']))); + expect(cache.listSync(recursive: true).whereType(), isEmpty); + }, + ); + + test( + 'durable progress distinguishes download, hash and verified reuse', + () async { + final journal = FileValidationJournal( + Directory(p.join(cache.path, 'run')), + ); + await prepareModel( + profile, + cache, + client: MockClient((_) async => http.Response.bytes(bytes, 200)), + onProgress: journal.emitPreparation, + ); + await journal.emit({'type': 'manifest', 'test_marker': true}); + journal.close(); + final suiteEvents = File( + p.join(cache.path, 'run', 'events.jsonl'), + ).readAsLinesSync(); + expect(suiteEvents, hasLength(1)); + expect(jsonDecode(suiteEvents.single)['type'], 'manifest'); + final events = const LineSplitter() + .convert( + File( + p.join(cache.path, 'run', 'preparation.jsonl'), + ).readAsStringSync(), + ) + .map((line) => jsonDecode(line) as Map) + .toList(); + expect(events.map((e) => '${e['stage']}:${e['state']}'), [ + 'download:started', + 'download:finished', + 'checksum:started', + 'checksum:verified', + 'ready:verified', + ]); + expect(events.last['bytes'], bytes.length); + final reuse = >[]; + await prepareModel( + profile, + cache, + client: MockClient((_) async => throw StateError('no network')), + onProgress: (e) async => reuse.add(e), + ); + expect(reuse.map((e) => e['stage']), ['checksum', 'checksum', 'ready']); + // Progress alone cannot qualify an interrupted preparation as a suite pass. + expect( + ValidationReport.parse(events.map(jsonEncode).join('\n')).qualified, + isFalse, + ); + }, + ); + + test('invalid supplied file is retained for the user', () async { + final input = File(p.join(cache.path, 'user-model.gguf')) + ..writeAsStringSync('wrong'); + await expectLater( + prepareModel(profile, cache, suppliedPath: input.path), + throwsFormatException, + ); + expect(input.readAsStringSync(), 'wrong'); + }); + test( + 'deadline closes the active request and removes partial files', + () async { + final client = BlockedClient(); + await expectLater( + prepareModel( + profile, + cache, + client: client, + timeout: const Duration(milliseconds: 5), + ), + throwsA(isA()), + ); + expect(client.closed, true); + expect(cache.listSync(recursive: true).whereType(), isEmpty); + }, + ); + + test('download deadline retains byte progress in its diagnostic', () async { + final client = SlowModelClient(); + await expectLater( + prepareModel( + profile, + cache, + client: client, + timeout: const Duration(milliseconds: 50), + ), + throwsA( + isA().having( + (error) => error.message, + 'bounded progress diagnostic', + contains('receiving 2 of ${bytes.length} bytes'), + ), + ), + ); + expect(client.closed, isTrue); + expect(cache.listSync(recursive: true).whereType(), isEmpty); + }); + + test('external cancellation is distinct from a download deadline', () async { + final client = SlowModelClient(); + final cancel = Timer(const Duration(milliseconds: 20), client.close); + addTearDown(cancel.cancel); + await expectLater( + prepareModel(profile, cache, client: client), + throwsA(isA()), + ); + expect(cache.listSync(recursive: true).whereType(), isEmpty); + }); +} diff --git a/packages/llamadart_validation/test/native_reference_request_test.dart b/packages/llamadart_validation/test/native_reference_request_test.dart new file mode 100644 index 000000000..e9a087561 --- /dev/null +++ b/packages/llamadart_validation/test/native_reference_request_test.dart @@ -0,0 +1,115 @@ +import 'dart:convert'; + +import 'package:llamadart/llamadart.dart'; +import 'package:llamadart_validation/src/native_reference_request.dart'; +import 'package:test/test.dart'; + +void main() { + const prompt = 'What is the secret code? Reply with only the code.'; + const messages = [ + LlamaChatMessage.fromText( + role: LlamaChatRole.system, + text: 'Remember the secret code exactly.', + ), + LlamaChatMessage.fromText( + role: LlamaChatRole.user, + text: 'The secret code is cedar17.', + ), + LlamaChatMessage.fromText( + role: LlamaChatRole.assistant, + text: 'I will remember the code.', + ), + LlamaChatMessage.fromText(role: LlamaChatRole.user, text: prompt), + ]; + test('C API receives system content and seeds only prior turns', () { + final wire = nativeReferenceRequest(prompt, history: messages); + // This is the pinned upstream C API's parse-and-wrap contract. + final preface = [ + { + 'role': 'system', + 'content': jsonDecode(wire['system_message_json'] as String), + }, + ...jsonDecode(wire['messages_json'] as String) as List, + ]; + expect(preface.first, { + 'role': 'system', + 'content': 'Remember the secret code exactly.', + }); + expect(preface.skip(1), [ + { + 'role': 'user', + 'content': [ + {'type': 'text', 'text': 'The secret code is cedar17.'}, + ], + }, + { + 'role': 'assistant', + 'content': [ + {'type': 'text', 'text': 'I will remember the code.'}, + ], + }, + ]); + expect(jsonDecode(wire['message_json'] as String), { + 'role': 'user', + 'content': [ + {'type': 'text', 'text': prompt}, + ], + }); + expect(wire['enable_constrained_decoding'], false); + expect(jsonDecode(wire['extra_context_json'] as String), { + 'enable_thinking': true, + }); + }); + test('literal system control reproduces the public double-encoded bytes', () { + final systemObject = { + 'role': 'system', + 'content': [ + {'type': 'text', 'text': messages.first.content}, + ], + }; + final wire = nativeReferenceRequest( + prompt, + history: [ + LlamaChatMessage.fromText( + role: LlamaChatRole.system, + text: jsonEncode(systemObject), + ), + ...messages.skip(1), + ], + ); + expect(wire['system_message_json'], jsonEncode(jsonEncode(systemObject))); + expect( + wire['messages_json'], + nativeReferenceRequest(prompt, history: messages)['messages_json'], + ); + }); + test('history-free prompt does not create a preface', () { + final wire = nativeReferenceRequest('héllo 👋'); + expect(wire['system_message_json'], isNull); + expect(wire['messages_json'], isNull); + expect( + jsonDecode(wire['message_json'] as String)['content'][0]['text'], + 'héllo 👋', + ); + }); + test( + 'mismatched final prompt and empty history fail before native calls', + () { + expect( + () => nativeReferenceRequest('different', history: messages), + throwsArgumentError, + ); + expect( + () => nativeReferenceRequest(prompt, history: []), + throwsArgumentError, + ); + expect( + () => nativeReferenceRequest( + 'I will remember the code.', + history: messages.take(3).toList(), + ), + throwsArgumentError, + ); + }, + ); +} diff --git a/packages/llamadart_validation/test/npu_evidence_test.dart b/packages/llamadart_validation/test/npu_evidence_test.dart new file mode 100644 index 000000000..d56d96223 --- /dev/null +++ b/packages/llamadart_validation/test/npu_evidence_test.dart @@ -0,0 +1,325 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:llamadart_validation/src/placement.dart'; +import 'package:test/test.dart'; + +void main() { + final profile = ValidationProfile.fromJson( + jsonDecode( + File('assets/profiles/npu-qualcomm-sm8650.json').readAsStringSync(), + ) + as Map, + ); + test( + 'NPU device guard rejects the wrong SoC, ABI and API before loading', + () { + final valid = { + 'soc_model': 'SM8650', + 'abi': 'arm64-v8a', + 'android_api': 36, + }; + validateNpuDevice(profile, valid); + for (final invalid in [ + {...valid, 'soc_model': 'Tensor G4'}, + {...valid, 'abi': 'x86_64'}, + {...valid, 'android_api': 30}, + {...valid, 'android_api': null}, + ]) { + expect(() => validateNpuDevice(profile, invalid), throwsStateError); + } + }, + ); + test( + 'only completed synchronous work in this generation is positive proof', + () { + const before = [1, 10, 10, 0, 0, 0, 0]; + expect( + npuGenerationEvidence(before, [1, 12, 12, 0, 0, 0, 0])['verified'], + true, + ); + for (final after in [ + before, + [1, 12, 11, 1, 0, 0, 0], + [1, 12, 12, 0, 0, 0, 1], + [1, 0, 0, 0, 0, 0, 0], + [1, 10, 10, 0, 1, 0, 0], + [1, 10, 10, 0, 0, 1, 0], + [2, 12, 12, 0, 0, 0, 0], + [1], + ]) { + expect( + npuGenerationEvidence(before, after)['verified'], + false, + reason: '$after', + ); + } + }, + ); + for (final native in [true, false]) { + for (final focused in [false, true]) { + test( + 'NPU proof requires all generations: native=$native focused=$focused', + () { + final data = profile.toJson(); + if (native) data['execution_path'] = 'native_c_api'; + if (focused) { + data['selection'] = 'focused'; + data['focus_features'] = ['lifecycle', 'streaming', 'guards']; + } + final target = data['npu_target'] as Map; + final locks = target['libraries'] as Map; + final manifest = { + 'schema_version': focused ? 2 : 1, + if (focused) 'catalog': {'version': 3}, + 'profile': data, + 'accelerator_evidence_required': true, + 'environment': {'litert_tag': '0.17.0-3'}, + 'preparation': { + 'npu': { + 'device': { + 'soc_model': 'SM8650', + 'abi': 'arm64-v8a', + 'verified': true, + 'android_api': 36, + }, + 'kit': { + 'schema_version': 1, + 'litert_revision': '9fe5be45564c868408e6514c8aabb83e211a0911', + 'dispatch_header_sha256': + '11dd4d98bd084157ac987b1ee1951f3f96e2b3ca6b51a27c10e645686bf0e3ee', + 'target': target['soc'], + 'runtime_tag': '0.17.0-3', + 'libraries': { + for (final entry in locks.entries) + entry.key: { + 'bytes': 1024, + 'sha256': + (entry.value as Map)['sha256'] ?? + List.filled(64, 'a').join(), + }, + }, + }, + }, + }, + }; + final cases = >[]; + var count = 0; + for (final id in [ + if (!native) 'C03.raw', + 'C04.hello', + 'C04.arithmetic', + 'C06.history', + if (native) ...[ + 'C06.history.public_system_wire', + 'C06.history.no_system', + 'C06.history.combined', + ], + if (!native) 'C08.cancel', + 'C09.reload', + if (!native) ...['C10.limit', 'C12.recovery'], + 'B01.warmup', + 'B01.1', + 'B01.2', + 'B01.3', + if (focused && !native) ...['C10.stop', 'C12.guards'], + if (focused) 'C09.reload.second', + ]) { + Map generation() { + final before = [1, count, count, 0, 0, 0, 0]; + count++; + return { + 'npu_execution': { + 'before': before, + 'after': [1, count, count, 0, 0, 0, 0], + 'verified': true, + }, + }; + } + + if (id == 'C10.stop' || id == 'C12.guards') { + cases.add({ + 'case_id': id, + 'status': 'PASS', + if (id == 'C10.stop') ...{ + 'control': generation(), + 'stopped': generation(), + }, + 'recovery': generation(), + }); + continue; + } + final control = id == 'C08.cancel' ? generation() : null; + final record = generation(); + cases.add({ + 'case_id': id, + 'status': 'PASS', + ...record, + if (control != null) ...{ + 'uncancelled_control': control, + 'recovery': generation(), + }, + }); + } + expect(inspectPlacement(manifest, cases, null)['verified'], true); + if (focused && !native) { + final extendedData = { + ...data, + 'selection': 'focused', + 'focus_features': [ + 'lifecycle', + 'streaming', + 'guards', + 'unicode', + 'thinking', + 'tools', + ], + }; + final extendedManifest = { + ...manifest, + 'profile': extendedData, + 'catalog': {'version': 4}, + }; + final extendedCases = cases + .map((c) => jsonDecode(jsonEncode(c)) as Map) + .toList(); + extendedCases.addAll([ + {'case_id': 'C02.generate'}, + { + 'case_id': 'C05.thinking', + 'trials': [{}, {}], + }, + { + 'case_id': 'C07.tools', + 'trials': [ + {'tool_result_followup': {}}, + {'tool_result_followup': {}}, + {}, + ], + 'recovery': {}, + }, + ]); + final ordered = ValidationProfile.fromJson(extendedData).caseIds; + extendedCases.sort( + (a, b) => ordered + .indexOf(a['case_id']) + .compareTo(ordered.indexOf(b['case_id'])), + ); + final generations = []; + final added = []; + for (final record in extendedCases) { + final id = record['case_id']; + final parts = switch (id) { + 'C08.cancel' => [ + record['uncancelled_control'], + record, + record['recovery'], + ], + 'C10.stop' => [ + record['control'], + record['stopped'], + record['recovery'], + ], + 'C12.guards' => [record['recovery']], + 'C05.thinking' => record['trials'] as List, + 'C07.tools' => [ + for (final trial in record['trials'] as List) ...[ + trial, + if ((trial as Map).containsKey('tool_result_followup')) + trial['tool_result_followup'], + ], + record['recovery'], + ], + _ => [record], + }; + generations.addAll(parts.cast()); + if (['C02.generate', 'C05.thinking', 'C07.tools'].contains(id)) { + added.addAll(parts.cast()); + } + } + for (var i = 0; i < generations.length; i++) { + generations[i]['npu_execution'] = { + 'before': [1, i, i, 0, 0, 0, 0], + 'after': [1, i + 1, i + 1, 0, 0, 0, 0], + }; + } + expect( + inspectPlacement( + extendedManifest, + extendedCases, + null, + )['verified'], + true, + ); + for (final generation in added) { + final saved = generation.remove('npu_execution'); + expect( + inspectPlacement( + extendedManifest, + extendedCases, + null, + )['verified'], + false, + ); + generation['npu_execution'] = { + 'before': [1, 0, 0, 0, 0, 0, 0], + 'after': [1, 0, 0, 0, 0, 0, 1], + }; + expect( + inspectPlacement( + extendedManifest, + extendedCases, + null, + )['verified'], + false, + ); + generation['npu_execution'] = saved; + } + } + expect( + inspectPlacement(manifest, cases, null)['proven_generations'], + (native ? 11 : 14) + + (focused ? 1 : 0) + + (focused && !native ? 4 : 0), + ); + for (final missing in cases.map((record) => record['case_id'])) { + expect( + inspectPlacement( + manifest, + cases.where((record) => record['case_id'] != missing).toList(), + null, + )['verified'], + false, + reason: '$missing requires independent NPU proof', + ); + } + expect( + inspectPlacement(manifest, cases.sublist(1), null)['verified'], + false, + ); + for (final record in cases.where( + (r) => ['C10.stop', 'C12.guards'].contains(r['case_id']), + )) { + for (final key in ['control', 'stopped', 'recovery']) { + if (record[key] is! Map) continue; + final generation = record[key] as Map; + final proof = generation.remove('npu_execution'); + expect( + inspectPlacement(manifest, cases, null)['verified'], + false, + reason: '${record['case_id']} $key must prove NPU work', + ); + generation['npu_execution'] = proof; + } + } + final proof = cases.last.remove('npu_execution'); + expect(inspectPlacement(manifest, cases, null)['verified'], false); + cases.last['npu_execution'] = proof; + (cases.last['npu_execution'] as Map)['after'] = [1, 6, 6, 0, 1, 0, 0]; + expect(inspectPlacement(manifest, cases, null)['verified'], false); + }, + ); + } + } +} diff --git a/packages/llamadart_validation/test/public_engine_batching_test.dart b/packages/llamadart_validation/test/public_engine_batching_test.dart new file mode 100644 index 000000000..aa2f231b1 --- /dev/null +++ b/packages/llamadart_validation/test/public_engine_batching_test.dart @@ -0,0 +1,220 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:llamadart/llamadart.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:test/test.dart'; + +class CapturingEngine implements LlamaEngine { + final requests = []; + bool lateContent = false; + bool duplicateFinish = false; + bool emitTool = false; + final featureRequests = >[]; + + @override + Stream generate( + String prompt, { + GenerationParams params = const GenerationParams(), + List? parts, + }) async* { + requests.add(params); + yield 'Montréal '; + yield '👋'; + } + + @override + Stream create( + List messages, { + GenerationParams? params, + List? tools, + ToolChoice? toolChoice, + bool parallelToolCalls = false, + bool enableThinking = true, + Map? responseFormat, + String? sourceLangCode, + String? targetLangCode, + Map? chatTemplateKwargs, + DateTime? templateNow, + }) async* { + featureRequests.add({ + 'thinking': enableThinking, + 'tools': tools, + 'choice': toolChoice, + }); + requests.add(params!); + LlamaCompletionChunk chunk( + LlamaCompletionChunkDelta delta, { + String? finish, + }) => LlamaCompletionChunk( + id: 'test', + object: 'chat.completion.chunk', + created: 0, + model: 'test', + choices: [ + LlamaCompletionChunkChoice( + index: 0, + delta: delta, + finishReason: finish, + ), + ], + ); + yield chunk(LlamaCompletionChunkDelta(thinking: 'reason')); + yield chunk(LlamaCompletionChunkDelta(content: 'Montréal 👋')); + if (emitTool) { + yield chunk( + LlamaCompletionChunkDelta( + toolCalls: [ + LlamaCompletionChunkToolCall( + index: 0, + function: LlamaCompletionChunkFunction( + name: 'weather', + arguments: '{"city":"Montréal"}', + ), + ), + ], + ), + ); + } + yield chunk(LlamaCompletionChunkDelta(), finish: 'stop'); + if (lateContent) { + yield chunk(LlamaCompletionChunkDelta(content: 'late')); + } + if (duplicateFinish) { + yield chunk(LlamaCompletionChunkDelta(), finish: 'stop'); + } + } + + // Optional timing/tokenizer reads are deliberately unavailable in this double. + @override + dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation); +} + +void main() { + final profile = ValidationProfile.fromJson( + jsonDecode(File('assets/profiles/chat-litert-cpu.json').readAsStringSync()) + as Map, + ); + + test( + 'public feature controls reach the engine and restore defaults', + () async { + final captured = CapturingEngine(); + final adapter = PublicValidationEngine(engineFactory: () => captured); + final tool = ToolDefinition( + name: 'get_weather', + description: 'Weather', + parameters: [ToolParam.string('city', required: true)], + handler: (_) async => {}, + ); + for (final mode in ToolChoice.values) { + final output = await adapter.generate( + 'prompt', + profile, + enableThinking: true, + tools: [tool], + toolChoice: mode, + maxTokens: 512, + ); + expect(captured.featureRequests.last['thinking'], true); + expect(captured.featureRequests.last['choice'], mode); + expect(captured.featureRequests.last['tools'], [tool]); + expect(captured.requests.last.maxTokens, 512); + expect(output['tool_choice'], mode.name); + expect(output['tools'], [tool.toJson()]); + } + await adapter.generate('prompt', profile); + expect(captured.featureRequests.last['thinking'], profile.enableThinking); + expect(captured.featureRequests.last['choice'], isNull); + expect(captured.featureRequests.last['tools'], isNull); + }, + ); + + test( + 'public adapter preserves the unloaded engine context rejection', + () async { + final adapter = PublicValidationEngine(); + try { + for (final raw in [false, true]) { + await expectLater( + adapter.generate('hello', profile, raw: raw), + throwsA(isA()), + ); + } + } finally { + await adapter.dispose(); + } + }, + ); + + test( + 'public adapter forwards batching to both raw and chat generation', + () async { + final captured = CapturingEngine(); + final adapter = PublicValidationEngine(engineFactory: () => captured); + for (final raw in [true, false]) { + final result = await adapter.generate( + 'prompt', + profile, + raw: raw, + maxTokens: 17, + streamBatchTokens: 1, + streamBatchBytes: 1, + stopSequences: ['cedar17'], + ); + final request = captured.requests.last; + expect(request.streamBatchTokenThreshold, 1); + expect(request.streamBatchByteThreshold, 1); + expect(request.maxTokens, 17); + expect(request.stopSequences, ['cedar17']); + expect(result['stop_sequences'], ['cedar17']); + expect(request.temp, 0); + expect(request.seed, 1); + expect(result['content'], 'Montréal 👋'); + expect(result['thinking'], raw ? '' : 'reason'); + expect(result['stream_completed'], true); + expect(result['completion_order_valid'], true); + expect( + result['stream_batch_tokens'], + request.streamBatchTokenThreshold, + ); + expect(result['stream_batch_bytes'], request.streamBatchByteThreshold); + await adapter.generate('prompt', profile, raw: raw); + expect(captured.requests.last.streamBatchTokenThreshold, 8); + expect(captured.requests.last.streamBatchByteThreshold, 512); + expect(captured.requests.last.stopSequences, isEmpty); + } + }, + ); + + test( + 'public adapter records late data and duplicate completion as invalid', + () async { + for (final late in [true, false]) { + final captured = CapturingEngine() + ..lateContent = late + ..duplicateFinish = !late; + final result = await PublicValidationEngine( + engineFactory: () => captured, + ).generate('prompt', profile); + expect(result['completion_order_valid'], false); + } + }, + ); + + test( + 'public adapter retains unexpected tool deltas for incomplete coverage', + () async { + final captured = CapturingEngine()..emitTool = true; + final result = await PublicValidationEngine( + engineFactory: () => captured, + ).generate('prompt', profile); + expect(result['tool_call_deltas'], [ + { + 'index': 0, + 'function': {'name': 'weather', 'arguments': '{"city":"Montréal"}'}, + }, + ]); + }, + ); +} diff --git a/packages/llamadart_validation/test/speech_runner_test.dart b/packages/llamadart_validation/test/speech_runner_test.dart new file mode 100644 index 000000000..46b75e44b --- /dev/null +++ b/packages/llamadart_validation/test/speech_runner_test.dart @@ -0,0 +1,212 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; +import 'package:llamadart/llamadart.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:llamadart_validation/src/speech_runner.dart'; +import 'package:test/test.dart'; + +class FakeSpeech implements SpeechValidationAdapter { + final calls = []; + bool ignoreInvalid = false; + bool wrongWords = false; + bool failLoad = false; + bool failCleanup = false; + Object? invalidError; + @override + Future load() async { + calls.add('load'); + if (failLoad) throw StateError('load failure'); + } + + @override + Future dispose() async { + calls.add('dispose'); + if (failCleanup) throw StateError('cleanup failure'); + } + + @override + Future> execute({ + bool cancel = false, + bool invalid = false, + bool bytesInput = false, + }) async { + calls.add( + cancel + ? 'cancel' + : invalid + ? 'invalid' + : 'execute', + ); + if (invalid && !ignoreInvalid) { + throw invalidError ?? ArgumentError('invalid'); + } + return {'predicate_passed': !wrongWords, if (cancel) 'cancelled': true}; + } +} + +void main() { + test( + 'speech input rejection accepts contract errors, not inference failures', + () async { + for (final error in [ + LlamaAudioFormatException('Encoded audio bytes must not be empty.'), + LlamaTextToSpeechException('Text to synthesize must not be empty.'), + LlamaTextToSpeechException('Synthesis failed.'), + LlamaInferenceException('Generation failed.'), + ]) { + final adapter = FakeSpeech()..invalidError = error; + final result = await runSpeechValidation(adapter); + final checks = result['checks'] as List; + final rejection = checks.singleWhere( + (item) => item['id'] == 'invalid_input', + ); + final accepted = + error is LlamaAudioFormatException || + error.message == 'Text to synthesize must not be empty.'; + expect(rejection['status'], accepted ? 'PASS' : 'FAIL'); + expect(result['functional_pass'], accepted); + expect(result['qualified'], false); + expect( + checks.singleWhere((item) => item['id'] == 'after_invalid')['status'], + 'PASS', + ); + expect(adapter.calls.last, 'dispose'); + } + }, + ); + + test( + 'WER counts substitutions, insertions, deletions and rejects empty oracle', + () { + expect(speechWordErrorRate('Hello, WORLD!', 'hello world'), 0); + expect(speechWordErrorRate('one two', 'one three four'), 1); + expect(speechWordErrorRate('one two', 'one'), .5); + expect(speechWordErrorRate('one', 'two three four'), 3); + expect(speechWordErrorRate('Montréal 한글', 'Montréal 한글'), 0); + expect(() => speechWordErrorRate(' ', 'hello'), throwsArgumentError); + }, + ); + TextToSpeechResult audio(List values, {bool truncated = false}) => + TextToSpeechResult( + samples: Float32List.fromList(values), + sampleRateHz: 24000, + channelCount: 1, + framesGenerated: 1, + truncated: truncated, + ); + test('nonempty bytes alone cannot qualify invalid or truncated TTS', () { + for (final result in [ + audio([]), + audio([0, 0]), + audio([double.nan]), + audio([double.infinity]), + audio([.5], truncated: true), + ]) { + expect(() => inspectSpeechAudio(result), throwsStateError); + } + final result = inspectSpeechAudio(audio([.25, -.25])); + expect(result['audio_seconds'], closeTo(2 / 24000, .0000001)); + expect(result['listening_check'], 'NOT_RUN'); + }); + test( + 'locked fixture validates PCM duration and rejects malformed headers', + () { + final bytes = File('assets/speech/jfk.wav').readAsBytesSync(); + expect(speechFixtureSeconds(bytes), greaterThan(1)); + for (final bad in [ + Uint8List(0), + Uint8List.fromList(bytes.sublist(0, 50)), + Uint8List.fromList(bytes)..[0] = 0, + ]) { + expect(() => speechFixtureSeconds(bad), throwsFormatException); + } + final lock = jsonDecode( + File('assets/speech/stt.json').readAsStringSync(), + ); + expect(sha256.convert(bytes).toString(), lock['fixture']['sha256']); + expect(bytes.length, lock['fixture']['bytes']); + }, + ); + test('locks parse as immutable model and projector inputs', () { + for (final pack in ['stt', 'tts']) { + final lock = jsonDecode( + File('assets/speech/$pack.json').readAsStringSync(), + ); + for (final name in ['model', 'projector']) { + final profile = ValidationProfile.fromJson({ + 'schema_version': 1, + 'id': 'speech-$pack-$name', + 'runtime': 'gguf', + 'backend': 'cpu', + 'model': lock[name], + }); + profile.requireRunnable(); + } + } + }); + test( + 'successful lifecycle includes cancellation, rejection and independent reload', + () async { + final adapter = FakeSpeech(); + final result = await runSpeechValidation(adapter); + expect(result['functional_pass'], true); + expect(result['qualified'], false); + expect(adapter.calls, [ + 'load', + 'execute', + 'cancel', + 'execute', + 'invalid', + 'execute', + 'dispose', + 'load', + 'execute', + 'dispose', + ]); + }, + ); + test('wrong transcript and ignored invalid inputs cannot pass', () async { + for (final adapter in [ + FakeSpeech()..wrongWords = true, + FakeSpeech()..ignoreInvalid = true, + ]) { + expect((await runSpeechValidation(adapter))['functional_pass'], false); + expect(adapter.calls.last, 'dispose'); + } + }); + test('load and cleanup failures remain failures', () async { + for (final adapter in [ + FakeSpeech()..failLoad = true, + FakeSpeech()..failCleanup = true, + ]) { + expect((await runSpeechValidation(adapter))['functional_pass'], false); + expect(adapter.calls.last, 'dispose'); + } + }); + test( + 'existing output directory is never modified on CLI rejection', + () async { + final directory = Directory.systemTemp.createTempSync('speech-output-'); + try { + final sentinel = File('${directory.path}/failure.json') + ..writeAsStringSync('keep'); + final result = await Process.run(Platform.resolvedExecutable, [ + 'run', + 'bin/speech.dart', + '--pack', + 'tts', + '--out', + directory.path, + ]); + expect(result.exitCode, isNot(0)); + expect(sentinel.readAsStringSync(), 'keep'); + expect(directory.listSync(), hasLength(1)); + } finally { + directory.deleteSync(recursive: true); + } + }, + ); +} diff --git a/packages/llamadart_validation/test/validation_test.dart b/packages/llamadart_validation/test/validation_test.dart new file mode 100644 index 000000000..9e58eb0ca --- /dev/null +++ b/packages/llamadart_validation/test/validation_test.dart @@ -0,0 +1,1662 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:llamadart/llamadart.dart'; +import 'package:llamadart_validation/io.dart'; +import 'package:llamadart_validation/llamadart_validation.dart'; +import 'package:llamadart_validation/src/placement.dart'; +import 'package:test/test.dart'; + +ValidationProfile profile({String backend = 'cpu', bool release = false}) { + final json = + jsonDecode( + File('assets/profiles/chat-litert-cpu.json').readAsStringSync(), + ) + as Map; + json['backend'] = backend; + json['selection'] = release ? 'release' : 'quick'; + return ValidationProfile.fromJson(json); +} + +class FakeEngine implements ValidationEngine { + @override + bool isWeb = false; + bool rejectBatching = false; + String? batchingFault; + bool _batchingSeen = false; + var disposed = false; + var cancelled = false; + var loads = 0; + var ready = false; + bool ignoresReadiness = false; + bool ignoresStop = false; + int? failOnLoad; + var disposeCalls = 0; + var generated = 0; + var wrongArithmetic = false; + var wrongHistory = false; + final requests = >[]; + var ignoresCancellation = false; + String backendName = 'LiteRT-LM CPU'; + Map metadata = {}; + int? switchAfterLoad; + Completer? pauseReload; + var timeout = false; + bool tokenizeTimeout = false; + String? featureFault; + bool toolsSeen = false; + var cleanupFails = false; + @override + Future load(String location, ValidationProfile profile) async { + if (location.endsWith('.missing')) { + throw LlamaModelException('missing model'); + } + loads++; + ready = true; + if (loads == failOnLoad) throw LlamaModelException('reload failed'); + if (loads == 2) await pauseReload?.future; + } + + @override + Future unload() async { + ready = false; + } + + @override + Future dispose() async { + disposeCalls++; + disposed = true; + if (cleanupFails) throw StateError('cleanup failure'); + } + + @override + void cancel() { + cancelled = true; + } + + @override + Future> diagnostics() async => { + 'model_metadata': metadata, + 'backend_name': switchAfterLoad != null && loads >= switchAfterLoad! + ? 'Metal' + : backendName, + }; + @override + Future> tokenize(String text) async { + if (tokenizeTimeout) await Completer().future; + return utf8.encode(text); + } + + @override + Future detokenize(List tokens) async => utf8.decode(tokens); + @override + Future> generate( + String prompt, + ValidationProfile profile, { + bool raw = false, + int? maxTokens, + int? streamBatchTokens, + int? streamBatchBytes, + bool cancelAfterFirst = false, + List? history, + List? stopSequences, + bool? enableThinking, + List? tools, + ToolChoice? toolChoice, + }) async { + generated++; + if (featureFault == 'tool_followup_error' && + prompt.contains('temperature_celsius from the tool result')) { + throw StateError('synthetic tool followup failure'); + } + if (tools != null) toolsSeen = true; + if (!ready && !ignoresReadiness) throw LlamaContextException('Not loaded'); + final adjusted = streamBatchTokens != null || streamBatchBytes != null; + requests.add({ + 'prompt': prompt, + 'history': history, + 'batch_tokens': streamBatchTokens, + 'batch_bytes': streamBatchBytes, + 'stop_sequences': stopSequences, + }); + if (adjusted && rejectBatching) { + _batchingSeen = true; + if (batchingFault == 'wrong_error') throw StateError('failed'); + throw LlamaUnsupportedException( + batchingFault == 'wrong_option' + ? 'unrelated option' + : 'streamBatchTokenThreshold streamBatchByteThreshold', + ); + } + final recovering = _batchingSeen && !adjusted; + if (adjusted) _batchingSeen = true; + if (timeout) return Completer>().future; + return { + 'enable_thinking': featureFault == 'thinking_config' + ? false + : enableThinking ?? profile.enableThinking, + 'tools': tools?.map((t) => t.toJson()).toList(), + 'tool_choice': featureFault == 'tool_config' ? null : toolChoice?.name, + 'stop_sequences': stopSequences ?? const [], + 'content': prompt.contains('temperature_celsius from the tool result') + ? featureFault == 'tool_followup' + ? '18' + : '17' + : prompt == profile.fixtureText('unicode_generation', 'prompt') + ? featureFault == 'unicode' + ? 'Montréal �' + : profile.fixtureText('unicode_generation', 'expected') + : prompt.contains('alpha cedar17 omega') + ? stopSequences != null && !ignoresStop + ? 'alpha ' + : 'alpha cedar17 omega' + : (adjusted && batchingFault == 'content') || + (recovering && batchingFault == 'recovery') + ? 'corrupted' + : history != null || prompt.contains('The secret code is cedar17.') + ? wrongHistory + ? '77777777777777777777777777777777' + : 'cedar17' + : prompt.contains('2 + 2') + ? wrongArithmetic + ? '2' + : '4' + : 'hello', + 'thinking': featureFault == 'thinking_leak' + ? 'leaked' + : enableThinking == true && featureFault != 'thinking_missing' + ? 'Two plus two is four.' + : adjusted && batchingFault == 'thinking' + ? 'changed' + : '', + 'chunks': adjusted ? 32 : 5, + 'stream_batch_tokens': batchingFault == 'config' + ? 8 + : streamBatchTokens ?? 8, + 'stream_batch_bytes': streamBatchBytes ?? 512, + 'stream_completed': + !(adjusted && batchingFault == 'incomplete') && + !(recovering && batchingFault == 'recovery'), + 'completion_order_valid': !(adjusted && batchingFault == 'order'), + 'tool_call_deltas': + featureFault == 'tool_recovery' && toolsSeen && tools == null + ? [ + { + 'index': featureFault == 'tool_index' ? 1 : 0, + 'function': {'name': 'get_weather'}, + }, + ] + : tools != null && + (toolChoice != ToolChoice.none || featureFault == 'tool_none') + ? [ + { + 'index': featureFault == 'tool_index' ? 1 : 0, + 'function': { + 'name': 'get_weather', + 'arguments': featureFault == 'tool_malformed' + ? '{broken' + : featureFault == 'tool_arguments' + ? '{"city":"Paris"}' + : '{"city":"Montréal"}', + }, + }, + ] + : adjusted && batchingFault == 'tools' + ? [ + {'index': 0}, + ] + : [], + 'finish_reasons': tools != null && toolChoice != ToolChoice.none + ? [featureFault == 'tool_finish' ? 'length' : 'tool_calls'] + : raw + ? [] + : [adjusted && batchingFault == 'finish' ? 'length' : 'stop'], + 'prompt': prompt, + 'cancel_requested': cancelAfterFirst, + 'cancel_to_done_ms': cancelAfterFirst ? 1 : null, + 'metrics': { + 'native_decode_tokens': + maxTokens == 1 || (cancelAfterFirst && !ignoresCancellation) + ? 1 + : 8, + 'native_decode_tps': 12, + 'estimated_wall_tps': 10, + 'ttfa_ms': 5, + }, + }; + } +} + +Future<({ValidationReport report, List> events})> run( + FakeEngine engine, { + ValidationProfile? selected, +}) async { + final events = >[]; + await ValidationRunner( + profile: selected ?? profile(), + engine: engine, + emit: (event) async { + events.add(event); + }, + caseTimeout: const Duration(milliseconds: 30), + ).run( + 'model.litertlm', + runId: 'test-run', + preparation: { + 'verified': true, + 'sha256': (selected ?? profile()).modelHash, + 'bytes': (selected ?? profile()).model['bytes'], + }, + environment: { + 'source_commit': List.filled(40, 'a').join(), + 'source_dirty': false, + 'hook_sha256': List.filled(64, 'b').join(), + 'native_tag': 'v0.4.0', + 'litert_tag': '0.17.0-3', + }, + ); + return ( + report: ValidationReport.parse(events.map(jsonEncode).join('\n')), + events: events, + ); +} + +void main() { + test( + 'tool followup errors retain successful call evidence and phase', + () async { + final result = await run( + FakeEngine()..featureFault = 'tool_followup_error', + selected: profile(release: true), + ); + final record = result.report.cases.singleWhere( + (c) => c['case_id'] == 'C07.tools', + ); + expect(record['status'], 'ERROR'); + expect(record['operation_phase'], 'tools.auto.tool_result_followup'); + expect((record['trials'] as List).single['mode_passed'], true); + expect((record['trials'] as List).single['tool_choice'], 'auto'); + expect(result.report.qualified, false); + }, + ); + + test('native control cannot execute public release feature cases', () async { + final data = + jsonDecode( + File( + 'assets/profiles/npu-qualcomm-sm8650.json', + ).readAsStringSync(), + ) + as Map; + data['execution_path'] = 'native_c_api'; + data['selection'] = 'release'; + data['enable_thinking'] = true; + final result = await run( + FakeEngine(), + selected: ValidationProfile.fromJson(data), + ); + for (final id in ['C02.generate', 'C05.thinking', 'C07.tools']) { + final record = result.report.cases.singleWhere((c) => c['case_id'] == id); + expect(record['status'], 'NOT_RUN'); + expect(record['reason'], 'Requires public chat feature controls'); + } + }); + + for (final entry in { + 'unicode': 'C02.generate', + 'thinking_config': 'C05.thinking', + 'thinking_leak': 'C05.thinking', + 'thinking_missing': 'C05.thinking', + 'tool_config': 'C07.tools', + 'tool_arguments': 'C07.tools', + 'tool_finish': 'C07.tools', + 'tool_followup': 'C07.tools', + 'tool_recovery': 'C07.tools', + 'tool_none': 'C07.tools', + 'tool_index': 'C07.tools', + 'tool_malformed': 'C07.tools', + }.entries) { + test('release feature rejects ${entry.key}', () async { + final result = await run( + FakeEngine()..featureFault = entry.key, + selected: profile(release: true), + ); + expect( + result.report.cases.singleWhere( + (c) => c['case_id'] == entry.value, + )['status'], + 'FAIL', + ); + expect(result.report.qualified, false); + }); + } + + test('catalog three preserves unimplemented feature obligations', () async { + final selected = profile(release: true); + final result = await run(FakeEngine(), selected: selected); + final events = result.events; + events.first['catalog'] = selected.catalogForVersion(3); + events.first['catalog_hash'] = jsonHash(events.first['catalog']); + for (final record in events.where((e) => e['type'] == 'case')) { + final id = record['case_id'] as String; + record['fixture_hash'] = jsonHash( + selected.caseFixtures(id, catalogVersion: 3), + ); + if (['C05.thinking', 'C07.tools', 'C02.generate'].contains(id)) { + record['case_version'] = 1; + record['status'] = 'NOT_RUN'; + } + } + final historical = ValidationReport.parse( + events.map(jsonEncode).join('\n'), + ); + expect(historical.problems, isEmpty); + expect(historical.qualified, false); + events.firstWhere( + (e) => e['type'] == 'case' && e['case_id'] == 'C07.tools', + )['status'] = 'PASS'; + expect( + ValidationReport.parse(events.map(jsonEncode).join('\n')).problems, + contains( + 'Unimplemented catalog case cannot claim an executed result: C07.tools', + ), + ); + }); + + test( + 'first-use timeout preserves deferred initialization attribution', + () async { + final result = await run(FakeEngine()..tokenizeTimeout = true); + final load = result.report.cases.first; + expect(load['load_scope'], 'public_load_and_readiness'); + expect(load['native_initialization_proven'], false); + final failed = result.report.cases.singleWhere( + (c) => c['case_id'] == 'C02.unicode', + ); + expect(failed['status'], 'ERROR'); + expect(failed['reason'], 'case_timeout'); + expect( + failed['operation_phase'], + 'tokenize_including_possible_deferred_initialization', + ); + expect(failed['timeout_ms'], 30); + expect(failed['elapsed_ms'], greaterThanOrEqualTo(30)); + expect( + result.report.cases.skip(2).every((c) => c['status'] == 'NOT_RUN'), + true, + ); + expect(result.report.qualified, false); + + final success = await run(FakeEngine()); + final unicode = success.report.cases.singleWhere( + (c) => c['case_id'] == 'C02.unicode', + ); + expect(unicode['tokenize_call_ms'], isNonNegative); + expect(unicode['detokenize_call_ms'], isNonNegative); + expect( + unicode['tokenize_timing_scope'], + 'public_call_including_possible_deferred_initialization', + ); + }, + ); + + test('report separates missing GPU proof from functional failures', () async { + final passing = await run( + FakeEngine()..backendName = 'LiteRT-LM GPU', + selected: profile(backend: 'gpu'), + ); + expect(passing.report.assertionsPassed, true); + expect(passing.report.qualified, false); + expect(passing.report.qualificationGaps, ['accelerator_evidence_missing']); + expect((passing.report.toJson()['summary'] as Map)['qualification_gaps'], [ + 'accelerator_evidence_missing', + ]); + expect( + passing.report.toHtml(), + contains('does not by itself establish incompatibility'), + ); + + final failed = await run(FakeEngine()..wrongArithmetic = true); + expect(failed.report.qualificationGaps, contains('assertion_failure')); + expect( + failed.report.qualificationGaps, + isNot(contains('accelerator_evidence_missing')), + ); + + final incomplete = await run( + FakeEngine(), + selected: profile(release: true), + ); + expect(incomplete.report.qualificationGaps, isEmpty); + final broken = await run(FakeEngine()..failOnLoad = 1); + expect( + broken.report.qualificationGaps, + containsAll(['execution_error', 'cases_not_run']), + ); + expect((await run(FakeEngine())).report.qualificationGaps, isEmpty); + }); + + ValidationProfile focused(List features) => + ValidationProfile.fromJson( + profile().toJson() + ..['selection'] = 'focused' + ..['focus_features'] = features, + ); + + test( + 'focused lifecycle runs a second dispose/load/generation cycle', + () async { + final engine = FakeEngine(); + final result = await run(engine, selected: focused(['lifecycle'])); + expect(result.report.qualified, true); + expect( + engine.loads, + 4, + ); // Initial, first cycle, invalid-file recovery, second. + expect(engine.disposeCalls, 3); // Both cycles and final cleanup. + expect(result.report.cases.last['case_id'], 'C09.reload.second'); + expect(result.report.cases.last['status'], 'PASS'); + expect( + result.report.cases.map((c) => c['case_id']), + isNot(contains('C07.tools')), + ); + }, + ); + + test( + 'second-cycle load failure stays visible and cleanup still runs', + () async { + final engine = FakeEngine()..failOnLoad = 4; + final result = await run(engine, selected: focused(['lifecycle'])); + expect(result.report.qualified, false); + expect(result.report.cases.last['status'], 'ERROR'); + expect(result.report.cases.last['error_type'], 'LlamaModelException'); + expect(engine.disposeCalls, 3); + expect(result.report.cleanupPassed, true); + }, + ); + + test( + 'focused selection adds only matching obligations to the quick core', + () async { + final selected = focused(['streaming', 'tools']); + expect(selected.caseIds, [ + ...profile().caseIds, + 'C07.tools', + 'C10.stop', + 'C11.batching', + ]); + expect(selected.focusFeatures, ['streaming', 'tools']); + final result = await run(FakeEngine(), selected: selected); + expect(result.report.qualified, true); + expect( + result.report.cases + .where((c) => c['status'] == 'NOT_RUN') + .map((c) => c['case_id']), + isEmpty, + ); + expect(result.report.problems, isEmpty); + }, + ); + + test( + 'focused batching verifies reconstruction, configuration and recovery', + () async { + final engine = FakeEngine(); + final result = await run(engine, selected: focused(['batching'])); + final record = result.report.cases.last; + expect(record['case_id'], 'C11.batching'); + expect(result.report.qualified, true); + expect(record['case_version'], 2); + expect((record['batched'] as Map)['chunks'], 32); + expect((record['control'] as Map)['chunks'], 5); + expect( + engine.requests.where( + (r) => r['batch_tokens'] == 1 && r['batch_bytes'] == 1, + ), + hasLength(1), + ); + expect(record['reconstruction_equal'], true); + expect(record['configurations_verified'], true); + }, + ); + + for (final fault in [ + 'content', + 'thinking', + 'finish', + 'config', + 'incomplete', + 'order', + 'recovery', + ]) { + test('batching rejects $fault mismatch', () async { + final result = await run( + FakeEngine()..batchingFault = fault, + selected: focused(['batching']), + ); + expect(result.report.cases.last['status'], 'FAIL'); + expect(result.report.qualified, false); + expect(result.report.cleanupPassed, true); + }); + } + + test( + 'unqualified tool output and NPU defaults remain explicit gaps', + () async { + final tool = await run( + FakeEngine()..batchingFault = 'tools', + selected: focused(['batching']), + ); + expect(tool.report.cases.last['status'], 'NOT_RUN'); + expect(tool.report.qualified, false); + final webData = + jsonDecode( + File( + 'assets/profiles/tiny-gguf-batching.json', + ).readAsStringSync(), + ) + as Map; + final web = await run( + FakeEngine() + ..isWeb = true + ..backendName = 'CPU', + selected: ValidationProfile.fromJson(webData), + ); + expect(web.report.cases.last['status'], 'NOT_RUN'); + expect(web.report.cases.last['reason'], contains('browser')); + final data = focused(['batching']).toJson()..['backend'] = 'npu'; + for (final native in [false, true]) { + if (native) data['execution_path'] = 'native_c_api'; + final result = await run( + FakeEngine()..backendName = 'LiteRT-LM NPU', + selected: ValidationProfile.fromJson(data), + ); + expect(result.report.cases.last['status'], 'NOT_RUN'); + expect( + result.report.cases.last['reason'], + contains(native ? 'bypasses' : 'sampling'), + ); + } + }, + ); + + test( + 'LiteRT Web requires both named typed rejections and recovery', + () async { + for (final fault in [null, 'wrong_option', 'wrong_error', 'recovery']) { + final result = await run( + FakeEngine() + ..isWeb = true + ..rejectBatching = true + ..batchingFault = fault, + selected: focused(['batching']), + ); + expect( + result.report.cases.last['status'], + fault == null + ? 'PASS' + : fault == 'wrong_error' + ? 'ERROR' + : 'FAIL', + ); + if (fault == null) { + expect( + (result.report.cases.last['rejected_options'] as List), + hasLength(2), + ); + expect( + result.report.cases.last['coverage'], + 'litert_web_native_option_rejection_and_recovery', + ); + } + } + final ignored = await run( + FakeEngine()..isWeb = true, + selected: focused(['batching']), + ); + expect(ignored.report.cases.last['status'], 'FAIL'); + }, + ); + + test( + 'previous schema-2 catalog preserves its original obligations', + () async { + final selected = focused(['streaming']); + final result = await run(FakeEngine(), selected: selected); + final events = result.events; + events.first['catalog'] = selected.catalogForVersion(1); + events.first['catalog_hash'] = jsonHash(events.first['catalog']); + final batching = events.singleWhere( + (e) => e['type'] == 'case' && e['case_id'] == 'C11.batching', + ); + batching['case_version'] = 1; + batching['fixture_hash'] = jsonHash( + selected.caseFixtures('C11.batching', catalogVersion: 1), + ); + batching['status'] = 'NOT_RUN'; + final stop = events.singleWhere( + (e) => e['type'] == 'case' && e['case_id'] == 'C10.stop', + ); + stop['case_version'] = 1; + stop['fixture_hash'] = jsonHash( + selected.caseFixtures('C10.stop', catalogVersion: 1), + ); + stop['status'] = 'NOT_RUN'; + final report = ValidationReport.parse(events.map(jsonEncode).join('\n')); + expect(report.problems, isEmpty); + expect(report.cases.last['status'], 'NOT_RUN'); + events.first['catalog'] = selected.catalogForVersion(1) + ..['version'] = 999; + events.first['catalog_hash'] = jsonHash(events.first['catalog']); + expect( + ValidationReport.parse(events.map(jsonEncode).join('\n')).qualified, + false, + ); + }, + ); + + test( + 'stop marker requires the exact prefix, forwarded control and recovery', + () async { + for (final ignored in [false, true]) { + final engine = FakeEngine()..ignoresStop = ignored; + final result = await run(engine, selected: focused(['streaming'])); + final record = result.report.cases.singleWhere( + (c) => c['case_id'] == 'C10.stop', + ); + expect(record['status'], ignored ? 'FAIL' : 'PASS'); + expect(record['expected_prefix'], 'alpha '); + expect( + engine.requests.any( + (r) => + r['stop_sequences'] is List && + (r['stop_sequences'] as List).contains('cedar17'), + ), + true, + ); + expect(record['recovery']['content'], isNotEmpty); + } + }, + ); + test( + 'readiness guard cannot pass when unloaded generation is accepted', + () async { + for (final ignored in [false, true]) { + final result = await run( + FakeEngine()..ignoresReadiness = ignored, + selected: focused(['guards']), + ); + final record = result.report.cases.singleWhere( + (c) => c['case_id'] == 'C12.guards', + ); + expect(record['status'], ignored ? 'FAIL' : 'PASS'); + expect(record['recovery']['content'], isNotEmpty); + expect(result.report.cleanupPassed, true); + } + }, + ); + test( + 'catalog two cannot claim new stop and readiness guard execution', + () async { + final selected = profile(release: true); + final result = await run(FakeEngine(), selected: selected); + final events = result.events; + events.first['catalog'] = selected.catalogForVersion(2); + events.first['catalog_hash'] = jsonHash(events.first['catalog']); + expect( + (events.first['catalog']['fixtures'] as Map).containsKey('stop'), + false, + ); + final report = ValidationReport.parse(events.map(jsonEncode).join('\n')); + expect(report.qualified, false); + expect( + report.problems, + contains( + 'Unimplemented catalog case cannot claim an executed result: C10.stop', + ), + ); + expect( + report.problems, + contains( + 'Unimplemented catalog case cannot claim an executed result: C12.guards', + ), + ); + }, + ); + + test('batching feature cannot downgrade to old catalog', () async { + final selected = focused(['batching']); + expect(() => selected.catalogForVersion(1), throwsFormatException); + }); + + test( + 'catalog three requires guard reload placement without changing old logs', + () async { + final data = + jsonDecode( + File('assets/profiles/chat-gguf-metal.json').readAsStringSync(), + ) + as Map; + data['selection'] = 'focused'; + data['focus_features'] = ['guards']; + final selected = ValidationProfile.fromJson(data); + final engine = FakeEngine()..backendName = 'Metal'; + final result = await run(engine, selected: selected); + final cases = result.report.cases; + const load = + 'load_tensors: offloaded 7/7 layers to GPU\nMTL0 compute buffer size = 64.0 MiB\n'; + final manifest = result.events.first; + expect(engine.loads, 4); + expect(inspectPlacement(manifest, cases, load * 4)['verified'], true); + expect(inspectPlacement(manifest, cases, load * 3)['verified'], false); + expect( + inspectPlacement( + manifest, + cases.where((c) => c['case_id'] != 'C12.guards').toList(), + load * 4, + )['verified'], + false, + ); + final old = {...manifest, 'catalog': selected.catalogForVersion(2)}; + expect( + inspectPlacement( + old, + cases.where((c) => c['case_id'] != 'C12.guards').toList(), + load * 3, + )['verified'], + true, + ); + }, + ); + + test( + 'invalid feature selections and fixture overrides fail before execution', + () { + for (final patch in >[ + {'selection': 'focused'}, + {'selection': 'focused', 'focus_features': []}, + { + 'selection': 'focused', + 'focus_features': ['tools', 'tools'], + }, + { + 'selection': 'focused', + 'focus_features': ['unknown'], + }, + {'focus_features': []}, + { + 'focus_features': ['tools'], + }, + { + 'fixtures': { + 'hello': {'prompt': 3}, + }, + }, + { + 'fixtures': { + 'hello': {'unknown': 'ignored'}, + }, + }, + { + 'fixtures': { + 'unknown': {'prompt': 'ignored'}, + }, + }, + ]) { + expect( + () => ValidationProfile.fromJson(profile().toJson()..addAll(patch)), + throwsFormatException, + ); + } + }, + ); + + test( + 'catalog records resolved prompts, versions and explicit omissions', + () async { + final data = profile().toJson(); + (data['fixtures'] as Map)['hello'] = {'prompt': 'Please answer hello.'}; + final selected = ValidationProfile.fromJson(data); + final engine = FakeEngine(); + final result = await run(engine, selected: selected); + expect(result.report.qualified, true); + final manifest = result.events.first; + expect(manifest['schema_version'], 2); + expect(manifest['catalog_hash'], jsonHash(selected.catalog)); + expect((manifest['catalog'] as Map)['fixtures'], selected.fixtures); + expect( + engine.requests + .where((r) => r['prompt'] == 'Please answer hello.') + .length, + greaterThan(1), + ); + final omitted = ((manifest['catalog'] as Map)['cases'] as List) + .cast() + .singleWhere((c) => c['id'] == 'C07.tools'); + expect(omitted['selected'], false); + expect(omitted['omission_reason'], 'outside_selected_features'); + for (final record in result.report.cases) { + expect(record['case_version'], 1); + expect( + record['fixture_hash'], + jsonHash(selected.caseFixtures(record['case_id'] as String)), + ); + } + }, + ); + + test('rehashed catalog edits and missing metadata cannot qualify', () async { + for (final omit in [false, true]) { + final result = await run(FakeEngine()); + final manifest = result.events.first; + if (omit) { + manifest.remove('catalog'); + } else { + final catalog = jsonDecode(jsonEncode(manifest['catalog'])) as Map; + (catalog['fixtures'] as Map)['history']['expected'] = + 'a different code'; + manifest['catalog'] = catalog; + } + manifest['catalog_hash'] = jsonHash(manifest['catalog']); + final report = ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ); + expect(report.qualified, false); + expect( + report.problems, + contains('Catalog does not match the executable profile'), + ); + } + }); + + test('case version and fixture identity are independently checked', () async { + for (final field in ['case_version', 'fixture_hash']) { + final result = await run(FakeEngine()); + result.events.firstWhere((e) => e['type'] == 'case')[field] = + field == 'case_version' ? 2 : '0' * 64; + final report = ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ); + expect(report.qualified, false); + expect( + report.problems, + contains('Case version or fixture identity mismatch: C01.load'), + ); + } + }); + + test( + 'legacy quick journals remain readable without invented catalog metadata', + () async { + final result = await run(FakeEngine()); + result.events.first + ..['schema_version'] = 1 + ..remove('catalog') + ..remove('catalog_hash'); + for (final event in result.events) { + event + ..remove('case_version') + ..remove('fixture_hash'); + } + final report = ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ); + expect(report.qualified, true); + expect(report.manifest.containsKey('catalog'), false); + }, + ); + + test('legacy schema cannot claim unchecked catalog provenance', () async { + final result = await run(FakeEngine()); + result.events.first['schema_version'] = 1; + final report = ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ); + expect(report.qualified, false); + expect( + report.problems, + contains('Catalog metadata requires result schema 2'), + ); + expect( + report.toHtml(), + contains('catalog version: unavailable in legacy journal'), + ); + }); + + test( + 'focused journals cannot downgrade to legacy selection semantics', + () async { + final result = await run(FakeEngine(), selected: focused(['lifecycle'])); + result.events.first['schema_version'] = 1; + final report = ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ); + expect(report.qualified, false); + expect( + report.problems, + contains('Focused selection requires result schema 2'), + ); + }, + ); + + test('Gemma CPU control locks identity and matches NPU prompt settings', () { + final selected = ValidationProfile.fromJson( + jsonDecode( + File('assets/profiles/gemma3-litert-cpu.json').readAsStringSync(), + ) + as Map, + ); + expect(selected.backend, 'cpu'); + expect(selected.contextSize, 1280); + expect(selected.threads, 4); + expect(selected.maxTokens, 32); + expect(selected.enableThinking, true); + expect(selected.effectiveConfig['enable_thinking'], true); + expect( + selected.effectiveConfig['sampling_application'], + 'requested_sampler', + ); + expect(selected.historyControls, true); + expect(selected.nativeReference, false); + expect( + selected.modelHash, + '1325ae366d31950f137c9c357b9fa89448b176d76998180c08ceaca78bba98be', + ); + expect(profile().enableThinking, false); + expect(profile().historyControls, false); + expect(profile(backend: 'npu').enableThinking, true); + }); + + test('history controls and thinking overrides reject invalid contracts', () { + for (final patch in >[ + {'enable_thinking': 'true'}, + {'history_controls': 1}, + {'history_controls': true, 'backend': 'gpu'}, + {'history_controls': true, 'backend': 'npu'}, + { + 'execution_path': 'native_c_api', + 'backend': 'npu', + 'enable_thinking': false, + }, + ]) { + expect( + () => ValidationProfile.fromJson(profile().toJson()..addAll(patch)), + throwsFormatException, + ); + } + }); + + for (final native in [false, true]) { + test( + 'history controls preserve distinct input and strict oracle: native=$native', + () async { + final data = profile(backend: native ? 'npu' : 'cpu').toJson() + ..['execution_path'] = native ? 'native_c_api' : 'public_api'; + if (!native) data['history_controls'] = true; + final engine = FakeEngine() + ..backendName = native + ? 'LiteRT-LM NPU direct C API' + : 'LiteRT-LM CPU'; + final result = await run( + engine, + selected: ValidationProfile.fromJson(data), + ); + expect(result.report.cases.length, native ? 12 : 17); + expect(result.report.assertionsPassed, true); + final histories = engine.requests + .where((r) => r['history'] != null) + .toList(); + expect(histories, hasLength(3)); + final canonical = histories[0]['history'] as List; + expect(canonical.map((m) => m.role.name), [ + 'system', + 'user', + 'assistant', + 'user', + ]); + expect(canonical.map((m) => m.content), [ + 'Remember the secret code exactly.', + 'The secret code is cedar17.', + 'I will remember the code.', + 'What is the secret code? Reply with only the code.', + ]); + final literal = histories[1]['history'] as List; + expect(jsonDecode(literal.first.content), { + 'role': 'system', + 'content': [ + {'type': 'text', 'text': canonical.first.content}, + ], + }); + expect( + literal.skip(1).map((m) => m.content), + canonical.skip(1).map((m) => m.content), + ); + final noSystem = histories[2]['history'] as List; + expect(noSystem.map((m) => m.role.name), ['user', 'assistant', 'user']); + final combined = engine.requests.singleWhere( + (r) => + r['history'] == null && + (r['prompt'] as String).contains('cedar17'), + ); + expect(combined['prompt'], canonical.map((m) => m.content).join('\n')); + + final failed = await run( + FakeEngine() + ..wrongHistory = true + ..backendName = native + ? 'LiteRT-LM NPU direct C API' + : 'LiteRT-LM CPU', + selected: ValidationProfile.fromJson(data), + ); + expect(failed.report.assertionsPassed, false); + expect( + failed.report.cases + .where((c) => (c['case_id'] as String).startsWith('C06.')) + .map((c) => c['status']), + ['FAIL', 'FAIL', 'FAIL', 'FAIL'], + ); + }, + ); + } + test( + 'NPU candidates are locked and reject preparation before any download', + () async { + for (final target in ['qualcomm-sm8650', 'tensor-g5']) { + final selected = ValidationProfile.fromJson( + jsonDecode( + File('assets/profiles/npu-$target.json').readAsStringSync(), + ) + as Map, + ); + expect(selected.backend, 'npu'); + expect(selected.contextSize, 1280); + expect(selected.requiresAcceleratorProof, true); + final directory = Directory.systemTemp.createTempSync( + 'npu-no-download-', + ); + try { + await expectLater( + prepareModel(selected, directory), + throwsA(isA()), + ); + expect(directory.listSync(), isEmpty); + } finally { + directory.deleteSync(); + } + } + }, + ); + test( + 'WASM CPU diagnostics require a CPU-only core and zero GPU layers', + () async { + final selected = profile().toJson()..['runtime'] = 'gguf'; + final model = Map.from(selected['model'] as Map); + model['filename'] = 'model.gguf'; + model['url'] = (model['url'] as String).replaceAll('.litertlm', '.gguf'); + selected['model'] = model; + for (final layers in ['0', '1', null]) { + final engine = FakeEngine() + ..backendName = 'WASM (Prototype bridge)' + ..metadata = { + 'llamadart.webgpu.n_gpu_layers': layers, + 'llamadart.webgpu.core_variant': 'wasm32', + }; + final result = await run( + engine, + selected: ValidationProfile.fromJson(selected), + ); + for (final id in ['C01.load', 'C09.reload', 'C12.recovery']) { + expect( + result.report.cases.firstWhere( + (record) => record['case_id'] == id, + )['status'], + layers == '0' ? 'PASS' : 'FAIL', + ); + } + } + }, + ); + test( + 'missing or dirty provenance preserves assertions but cannot qualify', + () async { + final result = await run(FakeEngine()); + expect(result.report.assertionsPassed, true); + for (final key in [ + 'source_commit', + 'source_dirty', + 'hook_sha256', + 'litert_tag', + ]) { + final events = (jsonDecode(jsonEncode(result.events)) as List) + .cast(); + (events.first['environment'] as Map).remove(key); + final report = ValidationReport.parse( + events.map(jsonEncode).join('\n'), + ); + expect(report.assertionsPassed, true); + expect(report.qualified, false); + expect(report.toJUnit(), contains('run-integrity')); + expect(report.provenanceProblems, isNotEmpty); + } + (result.events.first['environment'] as Map)['source_dirty'] = true; + expect( + ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ).qualified, + false, + ); + }, + ); + test('model preparation must prove the exact locked hash and size', () async { + final result = await run(FakeEngine()); + expect(result.report.qualified, true); + final valid = result.events.first['preparation'] as Map; + for (final preparation in [ + null, + {}, + {...valid, 'verified': false}, + {...valid, 'sha256': '0' * 64}, + {...valid, 'bytes': 1}, + {...valid, 'bytes': '${valid['bytes']}'}, + ]) { + final events = (jsonDecode(jsonEncode(result.events)) as List) + .cast(); + events.first['preparation'] = preparation; + final report = ValidationReport.parse(events.map(jsonEncode).join('\n')); + expect(report.assertionsPassed, true); + expect(report.qualified, false); + expect(report.provenanceProblems, contains(contains('model hash'))); + } + }); + test( + 'desktop payload verification is required even for complete assertions', + () async { + final result = await run(FakeEngine()); + final environment = result.events.first['environment'] as Map; + environment['os'] = 'macos'; + ValidationReport report() => + ValidationReport.parse(result.events.map(jsonEncode).join('\n')); + expect(report().qualified, false); + environment['runtime_payload_verified'] = true; + expect(report().qualified, false); + environment['runtime_bundle_sha256'] = 'a' * 64; + expect(report().qualified, true); + environment['runtime_payload_verified'] = false; + expect(report().qualified, false); + environment.remove('os'); + environment['platform'] = 'macOS'; + expect( + report().provenanceProblems, + contains(contains('Desktop runtime')), + ); + environment['web'] = true; + environment['platform'] = 'linux'; + environment['bridge_tag'] = 'fixture-bridge'; + expect(report().provenanceProblems, isEmpty); + }, + ); + test( + 'CPU backend changes during reload or recovery cannot qualify TPS', + () async { + for (final threshold in [2, 3, 4]) { + final result = await run( + FakeEngine()..switchAfterLoad = threshold, + selected: focused(['lifecycle']), + ); + expect( + result.report.cases.singleWhere( + (c) => + c['case_id'] == + (threshold == 2 + ? 'C09.reload' + : threshold == 3 + ? 'C12.recovery' + : 'C09.reload.second'), + )['status'], + 'FAIL', + ); + expect(result.report.qualified, false); + } + }, + ); + + test( + 'native GPU evidence requires matching backend and all load allocations', + () { + final manifest = { + 'schema_version': 1, + 'profile': + (jsonDecode( + File( + 'assets/profiles/tiny-gguf-cpu.json', + ).readAsStringSync(), + ) + as Map) + ..['backend'] = 'metal', + 'accelerator_evidence_required': true, + }; + final cases = >[ + { + 'case_id': 'C01.load', + 'status': 'PASS', + 'diagnostics': {'backend_name': 'Metal'}, + }, + { + 'case_id': 'C09.reload', + 'status': 'PASS', + 'diagnostics': {'backend_name': 'Metal'}, + }, + { + 'case_id': 'C12.recovery', + 'status': 'PASS', + 'diagnostics': {'backend_name': 'Metal'}, + }, + ]; + const load = + 'load_tensors: offloaded 7/7 layers to GPU\nsched_reserve: MTL0 compute buffer size = 63.62 MiB\n'; + expect(inspectPlacement(manifest, cases, load * 3)['verified'], true); + expect(inspectPlacement(manifest, cases, load * 2)['verified'], false); + expect( + inspectPlacement( + manifest, + cases, + load.replaceAll('7/7', '0/7') * 3, + )['verified'], + false, + ); + expect( + inspectPlacement( + manifest, + cases, + load.replaceAll('MTL0', 'CPU') * 3, + )['verified'], + false, + ); + expect( + inspectPlacement(manifest, cases, 'GPU found: Metal')['verified'], + false, + ); + cases.first['diagnostics'] = {'backend_name': 'CPU'}; + expect(inspectPlacement(manifest, cases, load * 3)['verified'], false); + }, + ); + + test('software Vulkan cannot qualify despite positive offload records', () { + final profile = + jsonDecode( + File('assets/profiles/tiny-gguf-cpu.json').readAsStringSync(), + ) + as Map; + profile['backend'] = 'vulkan'; + final manifest = {'schema_version': 1, 'profile': profile}; + final cases = [ + for (final id in ['C01.load', 'C09.reload', 'C12.recovery']) + { + 'case_id': id, + 'status': 'PASS', + 'diagnostics': {'backend_name': 'Vulkan0'}, + }, + ]; + final allocations = + 'load_tensors: offloaded 7/7 layers to GPU\n' + 'Vulkan0 compute buffer size = 64.0 MiB\n' * + 3; + final hardware = 'ggml_vulkan: 0 = NVIDIA L4\n$allocations'; + expect(inspectPlacement(manifest, cases, hardware)['verified'], true); + String selected(String name, {int index = 0, int free = 21831}) => + 'llama_prepare_model_devices: using device Vulkan$index ' + '($name) (0000:00:03.0) - $free MiB free\n'; + final selectedHardware = selected('NVIDIA L4'); + for (final valid in [ + '$selectedHardware$allocations', + '${selected('NVIDIA L4', free: 21760)}$selectedHardware$allocations', + '$selectedHardware$hardware', + 'ggml_vulkan: 0 = NVIDIA L4 (NVIDIA) | uma: 0\n' + '$selectedHardware$allocations', + '${selected('Intel(R) Arc(TM) A770')}$allocations', + ]) { + expect(inspectPlacement(manifest, cases, valid)['verified'], true); + } + for (final invalid in [ + allocations, + 'ggml_vulkan: 1 = NVIDIA L4\n$allocations', + 'ggml_vulkan: 0 = unknown device\n$allocations', + 'ggml_vulkan: 0 = Intel CPU\n$allocations', + '$hardware\nggml_vulkan: 0 = AMD Radeon', + '${selected('AMD Radeon')}$hardware', + '${selected('NVIDIA L4', index: 1)}$allocations', + '${selected('Intel CPU')}$allocations', + '${selected('NVIDIA virtual device')}$allocations', + '${selected('NVIDIA L4 | virtual device')}$allocations', + 'ggml_vulkan: 0 = NVIDIA L4 | virtual device\n$allocations', + 'ggml_vulkan: 0 = NVIDIA L4 | CPU\n$allocations', + 'ggml_vulkan: 0 = NVIDIA L4 | software\n$allocations', + '${selected('unknown device')}$allocations', + '${selected('NVIDIA (L4')}$allocations', + '${selectedHardware.replaceFirst(' MiB free', '')}$allocations', + 'unrecognized: $selectedHardware$allocations', + '$selectedHardware${allocations.replaceFirst('7/7', '0/7')}', + '$selectedHardware${allocations.replaceFirst('64.0', '0.0')}', + '$selectedHardware$allocations$allocations', + ]) { + expect(inspectPlacement(manifest, cases, invalid)['verified'], false); + } + for (final device in [ + 'llvmpipe (LLVM 20.1.2, 256 bits)', + 'Lavapipe', + 'SwiftShader Device', + 'Microsoft Basic Render Driver', + 'Software Rasterizer', + ]) { + final result = inspectPlacement( + manifest, + cases, + 'ggml_vulkan: 0 = $device\n$allocations', + ); + expect(result['verified'], false, reason: device); + expect(result['reason'], contains('software Vulkan')); + expect( + inspectPlacement( + manifest, + cases, + '${selected(device)}$allocations', + )['verified'], + false, + ); + // Mixed inventory cannot identify which physical device executed work. + expect( + inspectPlacement(manifest, cases, '$hardware\n$device')['verified'], + false, + ); + } + profile['backend'] = 'cpu'; + expect(inspectPlacement(manifest, cases, 'llvmpipe')['required'], false); + }); + + for (final selection in ['focused', 'release']) { + test('$selection GPU proof includes every selected reload', () { + final data = + jsonDecode( + File('assets/profiles/tiny-gguf-cpu.json').readAsStringSync(), + ) + as Map; + data['backend'] = 'metal'; + data['selection'] = selection; + if (selection == 'focused') data['focus_features'] = ['lifecycle']; + final manifest = { + 'schema_version': 2, + 'profile': data, + // A producer cannot waive the fourth load by omitting it here. + 'case_ids': ['C01.load', 'C09.reload', 'C12.recovery'], + 'accelerator_evidence_required': false, + }; + final cases = >[ + for (final id in [ + 'C01.load', + 'C09.reload', + 'C12.recovery', + 'C09.reload.second', + ]) + { + 'case_id': id, + 'status': 'PASS', + 'diagnostics': {'backend_name': 'Metal'}, + }, + ]; + const load = + 'load_tensors: offloaded 7/7 layers to GPU\nsched_reserve: MTL0 compute buffer size = 63.62 MiB\n'; + final proof = inspectPlacement(manifest, cases, load * 4); + expect(proof['verified'], true); + expect(proof['expected_loads'], 4); + expect(inspectPlacement(manifest, cases, load * 3)['verified'], false); + expect( + inspectPlacement( + manifest, + cases.take(3).toList(), + load * 4, + )['verified'], + false, + ); + expect( + inspectPlacement(manifest, [ + ...cases.take(3), + cases.first, + ], load * 4)['verified'], + false, + ); + cases.last['diagnostics'] = {'backend_name': 'CPU'}; + expect(inspectPlacement(manifest, cases, load * 4)['verified'], false); + cases.last['diagnostics'] = {'backend_name': 'Metal'}; + cases.last['status'] = 'FAIL'; + expect(inspectPlacement(manifest, cases, load * 4)['verified'], false); + }); + } + + test( + 'late timed-out reload cannot start more inference after cleanup', + () async { + final paused = Completer(); + final engine = FakeEngine()..pauseReload = paused; + final result = await run(engine); + final generated = engine.generated; + expect(result.report.cleanupPassed, false); + paused.complete(); + await Future.delayed(Duration.zero); + expect(engine.generated, generated); + expect(engine.loads, 2); + }, + ); + test('explicit CPU case rejects accelerator diagnostics', () async { + final result = await run(FakeEngine()..backendName = 'Metal'); + expect(result.report.cases.first['status'], 'FAIL'); + expect(result.report.qualified, false); + }); + + test( + 'ignored cancellation cannot pass the cancellation obligation', + () async { + final result = await run(FakeEngine()..ignoresCancellation = true); + expect( + result.report.cases.singleWhere( + (c) => c['case_id'] == 'C08.cancel', + )['status'], + 'NOT_RUN', + ); + expect(result.report.qualified, false); + }, + ); + test('journal sink failure still disposes the engine', () async { + final engine = FakeEngine(); + await expectLater( + ValidationRunner( + profile: profile(), + engine: engine, + emit: (_) async => throw StateError('disk full'), + ).run('model', runId: 'test', environment: {}), + throwsStateError, + ); + expect(engine.disposed, true); + }); + test('validated profile cannot be mutated through nested JSON', () { + final selected = profile(); + expect(() => selected.model['sha256'] = 'changed', throwsUnsupportedError); + }); + + test('rejects floating or unchecked model inputs', () { + for (final field in ['revision', 'sha256']) { + final json = profile().toJson(); + (json['model'] as Map)[field] = 'main'; + expect(() => ValidationProfile.fromJson(json), throwsFormatException); + } + }); + test('canonical identity is independent of JSON key order', () { + expect(jsonHash({'b': 2, 'a': 1}), jsonHash({'a': 1, 'b': 2})); + }); + test('one warmup and three measured samples; lifecycle loads only', () async { + final engine = FakeEngine(); + final result = await run(engine); + expect(result.report.qualified, isTrue); + expect(result.report.samples, hasLength(3)); + expect(engine.loads, 3); + expect(engine.disposed, isTrue); + }); + test('semantic failure preserves output and still measures TPS', () async { + final result = await run(FakeEngine()..wrongArithmetic = true); + final failure = result.report.cases.singleWhere( + (c) => c['case_id'] == 'C04.arithmetic', + ); + expect(failure['status'], 'FAIL'); + expect(failure['content'], '2'); + expect(result.report.samples, hasLength(3)); + expect(result.report.assertionsPassed, isFalse); + }); + test( + 'timeout cancels and prevents overlapping subsequent inference', + () async { + final engine = FakeEngine()..timeout = true; + final result = await run(engine); + expect(engine.generated, 1); + expect(engine.cancelled, isTrue); + expect(engine.disposed, isTrue); + expect(result.report.assertionsPassed, isFalse); + expect( + result.report.cases.where((c) => c['status'] == 'NOT_RUN'), + isNotEmpty, + ); + }, + ); + test('cleanup failure cannot produce a passing report', () async { + final result = await run(FakeEngine()..cleanupFails = true); + expect(result.report.qualified, isFalse); + expect(result.report.toJUnit(), contains('run-integrity')); + }); + test( + 'GPU preference and reported layers are not accelerator proof', + () async { + final result = await run(FakeEngine(), selected: profile(backend: 'gpu')); + expect(result.report.assertionsPassed, isTrue); + expect(result.report.qualified, isFalse); + expect(result.report.toHtml(), contains('unverified')); + }, + ); + test('release selection executes every implemented obligation', () async { + final result = await run(FakeEngine(), selected: profile(release: true)); + expect(result.report.qualified, isTrue); + expect( + result.report.cases.singleWhere( + (c) => c['case_id'] == 'C07.tools', + )['status'], + 'PASS', + ); + }); + test( + 'removing an obligation and its inventory entry cannot qualify', + () async { + final result = await run(FakeEngine()); + final events = result.events; + (events.first['case_ids'] as List).remove('C06.history'); + events.removeWhere((event) => event['case_id'] == 'C06.history'); + for (var index = 1; index < events.length; index++) { + events[index]['sequence'] = index - 1; + } + final report = ValidationReport.parse(events.map(jsonEncode).join('\n')); + expect(report.qualified, false); + expect( + report.problems, + contains('Case inventory does not match the profile'), + ); + expect( + report.cases.singleWhere( + (c) => c['case_id'] == 'C06.history', + )['status'], + 'NOT_RUN', + ); + }, + ); + test('rehashed configuration must match the executable profile', () async { + final result = await run(FakeEngine()); + final manifest = result.events.first; + (manifest['effective_config'] as Map)['temperature'] = 0.8; + manifest['config_hash'] = jsonHash(manifest['effective_config']); + final report = ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ); + expect(report.qualified, false); + expect( + report.problems, + contains('Effective configuration does not match the profile'), + ); + }); + test('report rejects a self-hashed invalid profile', () async { + final result = await run(FakeEngine()); + final manifest = result.events.first; + (manifest['profile'] as Map)['runtime'] = 'invalid'; + manifest['profile_hash'] = jsonHash(manifest['profile']); + final report = ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ); + expect(report.qualified, false); + expect(report.problems, contains('Invalid validation profile')); + }); + test('manifest flag cannot waive accelerator evidence', () async { + for (final flag in [false, null]) { + final result = await run(FakeEngine(), selected: profile(backend: 'gpu')); + result.events.first['accelerator_evidence_required'] = flag; + final report = ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ); + expect(report.qualified, false); + expect(report.acceleratorVerified, false); + expect( + report.problems, + contains('Accelerator evidence requirement does not match the profile'), + ); + } + }); + test('record cannot declare its own unsupported exemption', () async { + final result = await run(FakeEngine()); + result.events.firstWhere((e) => e['type'] == 'case') + ..['status'] = 'UNSUPPORTED' + ..['expected_unsupported'] = true; + final report = ValidationReport.parse( + result.events.map(jsonEncode).join('\n'), + ); + expect(report.qualified, false); + expect(report.assertionsPassed, false); + }); + test( + 'duplicate, missing, malformed or truncated records fail closed', + () async { + final result = await run(FakeEngine()); + final lines = result.events.map(jsonEncode).toList(); + for (final altered in [ + [ + ...lines, + jsonEncode(result.events.firstWhere((e) => e['type'] == 'case')), + ], + lines.take(lines.length - 1).toList(), + [...lines, '{"type":'], + lines + .where((s) => !s.contains('C02.unicode') || s.contains('manifest')) + .toList(), + ]) { + expect(ValidationReport.parse(altered.join('\n')).qualified, isFalse); + } + }, + ); + test('reports escape generated markup and retain warmup exclusion', () async { + final result = await run(FakeEngine()); + result.report.cases.first['content'] = ''; + expect(result.report.toHtml(), isNot(contains('