From 8e531d00ec2ee589b9d4e63f9510988d3c4cbaf1 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:36:37 +0000 Subject: [PATCH] Fix HPC compatibility, harden CI and document verification gaps Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .claude/CLAUDE.md | 115 ++++----- .github/CONTRIBUTING.md | 124 +--------- .github/workflows/boj-build.yml | 34 ++- .github/workflows/casket-pages.yml | 2 + .github/workflows/hpc-ci.yml | 94 +++----- .github/workflows/workflow-linter.yml | 18 +- .machine_readable/6a2/STATE.a2ml | 12 +- .mise.toml | 16 -- CONTRIBUTING.adoc | 59 +++++ Justfile | 9 + README.adoc | 12 +- SECURITY.adoc | 2 +- .../2026-09-26-repository-audit.adoc | 176 ++++++++++++++ .../2026-05-30-docudactyl-chapel-rehab.adoc | 7 + .../distributed-aggregation-obligations.adoc | 64 +++++ ffi/zig/src/dragonfly.zig | 226 ++++++++++++------ ffi/zig/src/gpu_ocr.zig | 15 +- ffi/zig/src/hw_crypto.zig | 31 ++- ffi/zig/src/ml_inference.zig | 27 ++- ffi/zig/src/prefetch.zig | 29 ++- ffi/zig/src/quality_stats.zig | 24 +- ffi/zig/src/speaker_id.zig | 9 +- mise.toml | 71 ++---- scripts/ci/check-workflow-pins.sh | 27 +++ scripts/ci/install-chapel.sh | 13 + scripts/ci/install-idris2.sh | 18 ++ scripts/ci/smoke-hpc.sh | 19 ++ scripts/ci/test-ci.sh | 41 ++++ scripts/ci/test-zig-standalone.sh | 10 + src/chapel/FFIBridge.chpl | 12 +- src/chapel/ManifestLoader.chpl | 10 +- 31 files changed, 862 insertions(+), 464 deletions(-) delete mode 100644 .mise.toml create mode 100644 CONTRIBUTING.adoc create mode 100644 docs/compliance/2026-09-26-repository-audit.adoc create mode 100644 docs/decisions/distributed-aggregation-obligations.adoc create mode 100755 scripts/ci/check-workflow-pins.sh create mode 100755 scripts/ci/install-chapel.sh create mode 100755 scripts/ci/install-idris2.sh create mode 100755 scripts/ci/smoke-hpc.sh create mode 100755 scripts/ci/test-ci.sh create mode 100755 scripts/ci/test-zig-standalone.sh diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 10afdee..ad64cc3 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -1,70 +1,45 @@ -# CLAUDE.md - AI Assistant Instructions - -## Language Policy (Hyperpolymath Standard) - -### ALLOWED Languages & Tools - -| Language/Tool | Use Case | Notes | -|---------------|----------|-------| -| **AffineScript** | Primary application code | Affine-typed, compiles to typed-wasm or ESM | -| **Bun** | JS runtime & package management (tier 1) | Default for all new work. Runs compiled ESM/JS directly — no bundler step. Uses an npm-compatible `package.json` plus `bun.lock` — both are expected, not anti-patterns. | -| **Rust** | Performance-critical, systems, WASM | Preferred for CLI tools | -| **Tauri 2.0+** | Mobile apps (iOS/Android) | Rust backend + web UI | -| **Dioxus** | Mobile apps (native UI) | Pure Rust, React-like | -| **Gleam** | Backend services | Runs on BEAM or compiles to JS | -| **Bash/POSIX Shell** | Scripts, automation | Keep minimal | -| **JavaScript** | Only where AffineScript cannot | MCP protocol glue, Bun APIs | -| **Nickel** | Configuration language | For complex configs | -| **Guile Scheme** | State/meta files | .machine_readable/6a2/STATE.a2ml, .machine_readable/6a2/META.a2ml, .machine_readable/6a2/ECOSYSTEM.a2ml | -| **Julia** | Batch scripts, data processing | Per RSR | -| **OCaml** | AffineScript compiler | Language-specific | -| **Ada** | Safety-critical systems | Where required | - -### BANNED - Do Not Use - -| Banned | Replacement | -|--------|-------------| -| TypeScript | AffineScript | -| ReScript | AffineScript | -| Deno | Bun | -| Node.js | Bun | -| npm | Bun | -| pnpm/yarn | Bun | -| Go | Rust | -| Python | Julia/Rust/AffineScript | -| Java/Kotlin | Rust/Tauri/Dioxus | -| Swift | Tauri/Dioxus | -| React Native | Tauri/Dioxus | -| Flutter/Dart | Tauri/Dioxus | - -### Mobile Development - -**No exceptions for Kotlin/Swift** - use Rust-first approach: - -1. **Tauri 2.0+** - Web UI (AffineScript) + Rust backend, MIT/Apache-2.0 -2. **Dioxus** - Pure Rust native UI, MIT/Apache-2.0 - -Both are FOSS with independent governance (no Big Tech). - -### Enforcement Rules - -1. **No new TypeScript files** - Convert existing TS to AffineScript -2. **Use `package.json` + `bun.lock` for JS runtime deps** - Bun is npm-compatible; a manifest is REQUIRED -3. **`bun install --production` for production deps** - resolved from `package.json`, pinned via `bun.lock` -4. **No Go code** - Use Rust instead -5. **No Python anywhere** - Use Julia for data/batch, Rust for systems, AffineScript for apps -6. **No Kotlin/Swift for mobile** - Use Tauri 2.0+ or Dioxus - -### Package Management - -- **Primary**: Guix (guix.scm) -- **Fallback**: Guix (flake.guix) -- **JS deps**: Bun (`package.json` + `bun.lock`). Declare tooling as a devDependency and run `bunx --no-install --bun ` — a bare `bunx ` can fetch an unpinned package and may start Node via its shebang. - -### Security Requirements - -- No MD5/SHA1 for security (use SHA256+) -- HTTPS only (no HTTP URLs) -- No hardcoded secrets -- SHA-pinned dependencies -- SPDX license headers on all files +# Docudactyl agent instructions + +Read `0-AI-MANIFEST.a2ml` and `.machine_readable/6a2/AGENTIC.a2ml` first. +Repository-specific constraints take precedence over generic scaffolding. + +## Sources and scope + +The estate standards source is `hyperpolymath/standards`; the operational +scaffold is `hyperpolymath/rsr-template-repo`. Do not confuse a draft, +historical checklist, or generated status report with ratified policy. +The 2026-09-26 review and source revisions are recorded in +`docs/compliance/2026-09-26-repository-audit.adoc`. + +## Actual implementation languages + +- Chapel: HPC orchestration and distributed processing. +- Zig: native FFI and parser integrations. +- Idris2: ABI types and proofs. +- OCaml: offline Scheme transformation; Ada: standalone TUI. +- Julia: existing legacy code, not the HPC hot path. +- Bash: minimal CI/build automation. Guile Scheme/A2ML: existing metadata. +- Agda: deferred distributed-invariant formalisation, not an existing proof. + +Do not migrate these components into an unrelated preferred language. +No new Python, Go, TypeScript, ReScript, Nix, or Deno code/configuration. +If JavaScript tooling is needed, use plain JavaScript with Bun and a pinned +manifest/lockfile; do not introduce AffineScript as an invented requirement. +The current estate tooling order makes npm a last resort, not a blanket ban. +Use Guix rather than Nix, and Podman/Containerfile rather than Docker. + +## Required behaviour + +1. Fix soundness/security holes before features or performance work. +2. Run the actual tools. Missing prerequisites and skipped checks are not green. +3. Never bypass a gate, invent evidence, or call a model of assumptions a proof + of this implementation. Read the audit before making production claims. +4. Keep state under `.machine_readable/`; do not move canonical files blindly + when upstream directory layouts differ. +5. Preserve existing licences. No automated relicensing, licence sweeps, or + changes to third-party notices. Escalate conflicting notices to the owner. +6. Never commit secrets; pin remote dependencies and Actions; use HTTPS. +7. Run the relevant checks in `CONTRIBUTING.adoc`, including regression tests + and `git diff --check`, and report exact scope and blockers. +8. Use signed commits for estate submissions. Publishing and external service + operations require deliberate authorisation; local edits are not deployment. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index af3f9d5..795c5e6 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,121 +1,7 @@ -# Clone the repository +# Contributing to Docudactyl -git clone https://github.com/hyperpolymath/docudactyl.git -cd docudactyl +The repository-specific guide is [CONTRIBUTING.adoc](../CONTRIBUTING.adoc). +It describes the actual Chapel/Zig/Idris2 toolchain, validation commands, +change discipline, and optional service setup. -# Using Guix (recommended for reproducibility) - -guix develop - -# Or using toolbox/distrobox - -toolbox create docudactyl-dev -toolbox enter docudactyl-dev -# Install dependencies manually - -# Verify setup - -just check # or: cargo check / mix compile / etc. -just test # Run test suite - -### Repository Structure - -```text -docudactyl/ -├── src/ # Source code (Perimeter 1-2) -├── lib/ # Library code (Perimeter 1-2) -├── extensions/ # Extensions (Perimeter 2) -├── plugins/ # Plugins (Perimeter 2) -├── tools/ # Tooling (Perimeter 2) -├── docs/ # Documentation (Perimeter 3) -│ ├── architecture/ # ADRs, specs (Perimeter 2) -│ └── proposals/ # RFCs (Perimeter 3) -├── examples/ # Examples (Perimeter 3) -├── spec/ # Spec tests (Perimeter 3) -├── tests/ # Test suite (Perimeter 2-3) -├── .well-known/ # Protocol files (Perimeter 1-3) -├── .github/ # GitHub config (Perimeter 1) -│ ├── CONTRIBUTING.md # This file -│ ├── ISSUE_TEMPLATE/ -│ └── workflows/ -├── CHANGELOG.md -├── CODE_OF_CONDUCT.md -├── GOVERNANCE.md -├── LICENSE -├── MAINTAINERS.md -├── README.adoc -├── SECURITY.md -├── flake.guix # Guix flake (Perimeter 1) -└── justfile # Task runner (Perimeter 1) -``` - - --- - -## How to Contribute - -### Reporting Bugs - - **Before reporting**: - 1. Search existing issues - 2. Check if it's already fixed in `main` - 3. Determine which perimeter the bug affects - - **When reporting**: - - Use the [bug report template](.github/ISSUE_TEMPLATE/bug_report.md) and include: - - - Clear, descriptive title - - Environment details (OS, versions, toolchain) - - Steps to reproduce - - Expected vs actual behaviour - - Logs, screenshots, or minimal reproduction - -### Suggesting Features - - **Before suggesting**: - 1. Check the [roadmap](ROADMAP.md) if available - 2. Search existing issues and discussions - 3. Consider which perimeter the feature belongs to - - **When suggesting**: - - Use the [feature request template](.github/ISSUE_TEMPLATE/feature_request.md) and include: - - - Problem statement (what pain point does this solve?) - - Proposed solution - - Alternatives considered - - Which perimeter this affects - -### Your First Contribution - - Look for issues labelled: - - - [`good first issue`](https://github.com/hyperpolymath/docudactyl/labels/good%20first%20issue) — Simple Perimeter 3 tasks - - [`help wanted`](https://github.com/hyperpolymath/docudactyl/labels/help%20wanted) — Community help needed - - [`documentation`](https://github.com/hyperpolymath/docudactyl/labels/documentation) — Docs improvements - - [`perimeter-3`](https://github.com/hyperpolymath/docudactyl/labels/perimeter-3) — Community sandbox scope - - --- - -## Development Workflow - -### Branch Naming - -docs/short-description # Documentation (P3) test/what-added # Test -additions (P3) feat/short-description # New features (P2) -fix/issue-number-description # Bug fixes (P2) refactor/what-changed # -Code improvements (P2) security/what-fixed # Security fixes (P1-2) - - -### Commit Messages - - We follow [Conventional Commits](https://www.conventionalcommits.org/): - -type(scope): description - -Body: what changed and why. - -Footer: issue reference, e.g. Closes #123 -\[optional body\] - -\[optional footer\] +For private vulnerability reporting, see [SECURITY.adoc](../SECURITY.adoc). diff --git a/.github/workflows/boj-build.yml b/.github/workflows/boj-build.yml index 6c50439..e8db6ba 100644 --- a/.github/workflows/boj-build.yml +++ b/.github/workflows/boj-build.yml @@ -1,20 +1,30 @@ # SPDX-License-Identifier: MPL-2.0 name: BoJ Server Build Trigger +# Opt-in: GitHub-hosted runners cannot resolve the old boj-server.local URL. on: - push: - branches: [main, master] workflow_dispatch: +permissions: + contents: read jobs: trigger-boj: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 5 steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Trigger BoJ Server (Casket/ssg-mcp) + - name: Trigger configured BoJ server + env: + BOJ_SERVER_URL: ${{ vars.BOJ_SERVER_URL }} + BOJ_TOKEN: ${{ secrets.BOJ_TOKEN }} + REPO: ${{ github.repository }} + BRANCH: ${{ github.ref_name }} run: | - # Send a secure trigger to boj-server to build this repository - curl -X POST "http://boj-server.local:7700/cartridges/ssg-mcp/invoke" -H "Content-Type: application/json" -d "{\"repo\": \"${{ github.repository }}\", \"branch\": \"${{ github.ref_name }}\", \"engine\": \"casket\\"}"} - continue-on-error: true -permissions: - actions: read - contents: read + set -euo pipefail + : "${BOJ_SERVER_URL:?Set BOJ_SERVER_URL to an HTTPS endpoint reachable by this runner}" + : "${BOJ_TOKEN:?Configure BOJ_TOKEN before dispatching}" + [[ "$BOJ_SERVER_URL" == https://* ]] || { echo 'HTTPS is required' >&2; exit 1; } + jq -n --arg repo "$REPO" --arg branch "$BRANCH" \ + '{repo: $repo, branch: $branch, engine: "casket"}' > "$RUNNER_TEMP/boj-payload.json" + curl --fail-with-body --silent --show-error --proto '=https' \ + --connect-timeout 10 --max-time 60 \ + -H "Authorization: Bearer $BOJ_TOKEN" -H 'Content-Type: application/json' \ + --data-binary "@$RUNNER_TEMP/boj-payload.json" \ + "${BOJ_SERVER_URL%/}/cartridges/ssg-mcp/invoke" diff --git a/.github/workflows/casket-pages.yml b/.github/workflows/casket-pages.yml index 96650f7..8d39e36 100644 --- a/.github/workflows/casket-pages.yml +++ b/.github/workflows/casket-pages.yml @@ -27,6 +27,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 with: repository: hyperpolymath/casket-ssg + ref: 45e72796456b91e0b800304ab0283cd4872c57c5 + persist-credentials: false path: .casket-ssg - name: Setup GHCup diff --git a/.github/workflows/hpc-ci.yml b/.github/workflows/hpc-ci.yml index 85d033c..5ce6e93 100644 --- a/.github/workflows/hpc-ci.yml +++ b/.github/workflows/hpc-ci.yml @@ -9,6 +9,7 @@ name: HPC Build & Test on: + workflow_dispatch: push: branches: [main] paths: @@ -17,6 +18,9 @@ on: - 'src/Docudactyl/**' - 'src/abi/**' - 'generated/abi/**' + - 'docudactyl.ipkg' + - 'mise.toml' + - 'scripts/ci/**' - '.github/workflows/hpc-ci.yml' pull_request: paths: @@ -25,8 +29,15 @@ on: - 'src/Docudactyl/**' - 'src/abi/**' - 'generated/abi/**' + - 'docudactyl.ipkg' + - 'mise.toml' + - 'scripts/ci/**' - '.github/workflows/hpc-ci.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: actions: read contents: read @@ -54,6 +65,9 @@ jobs: with: version: ${{ env.ZIG_VERSION }} + - name: Standalone Zig regression tests + run: bash scripts/ci/test-zig-standalone.sh + - name: Install C library headers run: | sudo apt-get update -qq @@ -107,13 +121,7 @@ jobs: path: ffi/zig/zig-out/lib/ - name: Install Chapel - run: | - CHPL_TARBALL="chapel-${CHAPEL_VERSION}-linux64-x86_64.tar.gz" - CHPL_URL="https://github.com/chapel-lang/chapel/releases/download/${CHAPEL_VERSION}/${CHPL_TARBALL}" - wget -q "${CHPL_URL}" -O "/tmp/${CHPL_TARBALL}" - tar xzf "/tmp/${CHPL_TARBALL}" -C /opt/ - echo "CHPL_HOME=/opt/chapel-${CHAPEL_VERSION}" >> "$GITHUB_ENV" - echo "/opt/chapel-${CHAPEL_VERSION}/bin/linux64-x86_64" >> "$GITHUB_PATH" + run: bash scripts/ci/install-chapel.sh - name: Install C runtime libraries (FFI .so transitively pulls these) run: | @@ -124,7 +132,7 @@ jobs: libavformat60 libavcodec60 libavutil58 \ libxml2 libgdal34t64 libvips42t64 \ liblmdb0 libarchive13t64 libcurl4t64 \ - tesseract-ocr-eng + tesseract-ocr-eng jq - name: Verify Chapel installation run: chpl --version @@ -133,7 +141,6 @@ jobs: run: | mkdir -p bin ABSPATH=$(cd ffi/zig/zig-out/lib && pwd) - chmod +x ffi/zig/zig-out/lib/libdocudactyl_ffi.so* || true chpl src/chapel/smoke.chpl \ src/chapel/FFIBridge.chpl \ -o bin/docudactyl-smoke \ @@ -165,14 +172,7 @@ jobs: path: ffi/zig/zig-out/lib/ - name: Install Chapel - run: | - CHPL_TARBALL="chapel-${CHAPEL_VERSION}-linux64-x86_64.tar.gz" - CHPL_URL="https://github.com/chapel-lang/chapel/releases/download/${CHAPEL_VERSION}/${CHPL_TARBALL}" - echo "Downloading Chapel ${CHAPEL_VERSION} from ${CHPL_URL}..." - wget -q "${CHPL_URL}" -O "/tmp/${CHPL_TARBALL}" - tar xzf "/tmp/${CHPL_TARBALL}" -C /opt/ - echo "CHPL_HOME=/opt/chapel-${CHAPEL_VERSION}" >> "$GITHUB_ENV" - echo "/opt/chapel-${CHAPEL_VERSION}/bin/linux64-x86_64" >> "$GITHUB_PATH" + run: bash scripts/ci/install-chapel.sh - name: Install C runtime libraries run: | @@ -183,7 +183,7 @@ jobs: libavformat60 libavcodec60 libavutil58 \ libxml2 libgdal34t64 libvips42t64 \ liblmdb0 libarchive13t64 libcurl4t64 \ - tesseract-ocr-eng + tesseract-ocr-eng jq - name: Verify Chapel installation run: chpl --version @@ -192,7 +192,6 @@ jobs: run: | mkdir -p bin ABSPATH=$(cd ffi/zig/zig-out/lib && pwd) - chmod +x ffi/zig/zig-out/lib/libdocudactyl_ffi.so* || true chpl src/chapel/DocudactylHPC.chpl \ src/chapel/Config.chpl \ src/chapel/ContentType.chpl \ @@ -209,32 +208,8 @@ jobs: --ldflags="-Wl,-rpath,$ABSPATH" \ --fast - - name: Smoke test (5 files) - run: | - # Create a small manifest from available system files - TMPDIR=$(mktemp -d) - find /usr/share/icons -name '*.png' 2>/dev/null | head -5 > "$TMPDIR/manifest.txt" - COUNT=$(wc -l < "$TMPDIR/manifest.txt") - echo "Smoke test manifest: $COUNT files" - - if [ "$COUNT" -lt 1 ]; then - echo "SKIP: No test files available" - exit 0 - fi - - export LD_LIBRARY_PATH="ffi/zig/zig-out/lib:$LD_LIBRARY_PATH" - bin/docudactyl-hpc \ - --manifestPath="$TMPDIR/manifest.txt" \ - --outputDir="$TMPDIR/output" \ - --chunkSize=1 - - # Verify reports were generated - ls "$TMPDIR/output/run-report.scm" "$TMPDIR/output/run-report.json" - - # Verify non-zero success count - SUCCEEDED=$(grep -oP 'Succeeded:\s+\K\d+' <<< "$(cat /dev/stdin)" || echo 0) - echo "Smoke test complete" - rm -rf "$TMPDIR" + - name: Smoke test (deterministic fixture, strict report assertions) + run: bash scripts/ci/smoke-hpc.sh # ═══════════════════════════════════════════════════════════════════════════ # Job 3: Idris2 ABI — Compile Proofs @@ -242,32 +217,35 @@ jobs: check-abi: name: Idris2 ABI Proofs runs-on: ubuntu-24.04 - continue-on-error: true # Idris2 CI setup can be fragile + timeout-minutes: 30 steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 - - name: Install Idris2 + - name: Install Idris2 prerequisites run: | - # Install Idris2 via pack (the Idris2 package manager) sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends \ - build-essential libgmp-dev chezscheme + sudo apt-get install -y --no-install-recommends build-essential libgmp-dev chezscheme + + - name: Cache pinned Idris2 installation + id: idris-cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.local/idris2 + key: idris2-${{ runner.os }}-${{ runner.arch }}-0.8.0-15a3e4e70843f7a34100f6470c04b791330788df-v1 + + - name: Install pinned Idris2 + if: steps.idris-cache.outputs.cache-hit != 'true' + run: bash scripts/ci/install-idris2.sh - # Clone and build Idris2 from source - git clone --depth 1 --branch v${IDRIS2_VERSION} \ - https://github.com/idris-lang/Idris2.git /tmp/idris2-src - cd /tmp/idris2-src - make bootstrap SCHEME=chezscheme - sudo make install - echo "$HOME/.idris2/bin" >> "$GITHUB_PATH" + - name: Add Idris2 to PATH + run: echo "$HOME/.local/idris2/bin" >> "$GITHUB_PATH" - name: Verify Idris2 installation run: idris2 --version - name: Compile ABI proofs run: | - cd /home/runner/work/docudactyl/docudactyl idris2 --build docudactyl.ipkg - name: Verify no banned patterns diff --git a/.github/workflows/workflow-linter.yml b/.github/workflows/workflow-linter.yml index 5844dfb..dcd6fd7 100644 --- a/.github/workflows/workflow-linter.yml +++ b/.github/workflows/workflow-linter.yml @@ -6,11 +6,14 @@ on: pull_request: paths: - '.github/workflows/**' + - 'scripts/ci/**' push: paths: - '.github/workflows/**' + - 'scripts/ci/**' -permissions: read-all +permissions: + contents: read jobs: lint-workflows: @@ -18,6 +21,9 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + - name: Test CI harnesses + run: bash scripts/ci/test-ci.sh + - name: Check SPDX headers run: | errors=0 @@ -43,12 +49,4 @@ jobs: exit $errors - name: Check pinned actions - run: | - errors=0 - for f in .github/workflows/*.yml .github/workflows/*.yaml; do - [ -f "$f" ] || continue - # Look for uses: without SHA - if grep -E "uses:.*@v[0-9]" "$f" | grep -v "#"; then - echo "WARNING: $f has unpinned actions (missing SHA comment)" - fi - done + run: bash scripts/ci/check-workflow-pins.sh diff --git a/.machine_readable/6a2/STATE.a2ml b/.machine_readable/6a2/STATE.a2ml index 80e7515..e883114 100644 --- a/.machine_readable/6a2/STATE.a2ml +++ b/.machine_readable/6a2/STATE.a2ml @@ -5,10 +5,20 @@ [metadata] project = "docudactyl" version = "0.1.0" -last-updated = "2026-03-15" +last-updated = "2026-09-26" status = "active" [project-context] name = "docudactyl" completion-percentage = 0 phase = "In development" + +[maintenance-review] +date = "2026-09-26" +report = "docs/compliance/2026-09-26-repository-audit.adoc" +standalone-zig-tests = 66 +standalone-zig-status = "passed on Zig 0.15.2" +full-native-status = "blocked: parser headers/libraries and Chapel/Idris2 unavailable locally" +issue-32 = "deferred; implementation gaps and proof obligations documented" +issues-33-35 = "changes prepared; pinned-toolchain CI acceptance still required" +production-readiness = "not certified: concurrency, resume and cache semantics need further work" diff --git a/.mise.toml b/.mise.toml deleted file mode 100644 index 5b59868..0000000 --- a/.mise.toml +++ /dev/null @@ -1,16 +0,0 @@ -[tools] -# = "Docudactyl HPC — Pinned tool versions (mise)" -# = "These versions are tested and known to work with the current codebase." -# = "Upgrading any of these requires testing:" -# = "- Zig: build.zig API changes (0.14→0.15 broke addSharedLibrary)" -# = "- Idris2: ABI proof syntax may change" -# = "- OCaml: dune build compatibility" -# = "- Julia: legacy component only" -# = "Chapel is NOT managed by mise —" -# = "Current tested version: 2.7.0" -# = "SPDX-License-Identifier: MPL-2.0" -zig = "0.15.2" -idris2 = "0.8.0" -ocaml = "5.4.1" -julia = "1.12.5" -just = "1.46.0" diff --git a/CONTRIBUTING.adoc b/CONTRIBUTING.adoc new file mode 100644 index 0000000..b3edd90 --- /dev/null +++ b/CONTRIBUTING.adoc @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += Contributing to Docudactyl + +Read `0-AI-MANIFEST.a2ml` and `.machine_readable/6a2/AGENTIC.a2ml` first. +The hot path is Chapel plus Zig; Idris2 describes the ABI. OCaml and Ada +are separate tools, and Julia is legacy code, not the HPC implementation. + +== Development and checks + +The canonical tool pins are in `mise.toml`. The HPC CI target is Linux +x86-64, Zig 0.15.2, Chapel 2.8.0 and Idris2 0.8.0. Native development +headers are listed in `.github/workflows/hpc-ci.yml`. Do not introduce +an unrelated language toolchain to work around a failed gate. + +[source,shell] +---- +# Fast checks: Zig + libc; Bash + jq +just test-standalone +just check-ci + +# Full FFI checks: all native parser development libraries required +(cd ffi/zig && zig build -Doptimize=ReleaseFast && zig build test) + +# ABI type/proof checking: Idris2 required +idris2 --build docudactyl.ipkg + +# Existing toolbox-based Chapel/FFI boundary checks +just check-smoke +just build-smoke +just run-smoke + +git diff --check +---- + +The standalone suite is not a replacement for the linked FFI tests, +Chapel binary, or ABI proofs. Missing tools, skipped tests and unavailable +external services must be reported as blockers, not passes. See +`docs/compliance/2026-09-26-repository-audit.adoc` for the current evidence +and production limitations. + +== Change discipline + +* Fix correctness and security holes before adding features. +* Add a regression test and rerun it after each fix. +* Pin remote Actions to full commit hashes; a version comment is not a pin. +* Do not bypass required checks with `continue-on-error`. +* Preserve existing licences; conflicting notices require owner review. +* Keep structured state under `.machine_readable/`, not the repository root. +* Never commit secrets or build artifacts. Report security issues privately + using `SECURITY.adoc`. +* Use signed commits when submitting a contribution to the estate. + +== Optional BoJ integration + +The BoJ workflow is manual-only. Before dispatch, configure the repository +variable `BOJ_SERVER_URL` with an HTTPS endpoint reachable from the runner +and the secret `BOJ_TOKEN` with a server-accepted bearer credential. +Do not put credentials in source or issue comments. The old unauthenticated +`.local` HTTP endpoint cannot be reached by GitHub-hosted runners. diff --git a/Justfile b/Justfile index f6767f9..dfd934e 100644 --- a/Justfile +++ b/Justfile @@ -1042,3 +1042,12 @@ crg-badge: D) color="orange" ;; E) color="red" ;; F) color="critical" ;; \ *) color="lightgrey" ;; esac; \ echo "[![CRG $$grade](https://img.shields.io/badge/CRG-$$grade-$$color?style=flat-square)](https://github.com/hyperpolymath/standards/tree/main/component-readiness-grades)" + +# Dependency-light Zig tests (still requires Zig 0.15.2 and libc) +test-standalone: + bash scripts/ci/test-zig-standalone.sh + +# Check CI action pins and test fail-closed smoke/report handling +check-ci: + bash scripts/ci/check-workflow-pins.sh + bash scripts/ci/test-ci.sh diff --git a/README.adoc b/README.adoc index e10130c..dfa29f8 100644 --- a/README.adoc +++ b/README.adoc @@ -8,7 +8,7 @@ image:https://img.shields.io/badge/License-PMPL_1.0-blue.svg[MPL-2.0,link="https // Badges image:https://img.shields.io/badge/RSR-Tier%201-gold[RSR Tier 1] image:https://img.shields.io/badge/Phase-v0.4.0-green[Phase] -image:https://img.shields.io/badge/Chapel-2.3+-4E9A06?logo=data:image/svg+xml;base64,[Chapel] +image:https://img.shields.io/badge/Chapel-2.8.0-4E9A06?logo=data:image/svg+xml;base64,[Chapel] image:https://img.shields.io/badge/Zig-0.15+-F7A41D?logo=zig[Zig] image:https://img.shields.io/badge/Idris2-0.8+-5E5086[Idris2] image:https://img.shields.io/badge/OCaml-4.14+-EC6813?logo=ocaml[OCaml] @@ -20,6 +20,16 @@ This project is licensed under **MPL-2.0** (Palimpsest License). The full licence text is in `license/PMPL-1.0.txt`. The canonical source is the https://github.com/hyperpolymath/palimpsest-license[palimpsest-license] repository. +== Verification status + +The HPC toolchain target is Zig 0.15.2, Chapel 2.8.0 and Idris2 0.8.0. +See link:CONTRIBUTING.adoc[contribution and validation commands] and +link:docs/compliance/2026-09-26-repository-audit.adoc[the current audit]. +Standalone tests do not certify distributed production readiness. In particular, +checkpoint/reduction guarantees and optional GPU/ML backend execution remain +unverified or incomplete; the throughput figures below are estimates, not +measurements established by this audit. + == Overview **Docudactyl** is a multi-format HPC document extraction engine designed for British Library scale (~170 million items). It processes PDFs, images, audio, video, EPUB, and geospatial data across hundreds of cluster nodes. diff --git a/SECURITY.adoc b/SECURITY.adoc index c44ef4b..edc16fd 100644 --- a/SECURITY.adoc +++ b/SECURITY.adoc @@ -35,7 +35,7 @@ a Vulnerability] This method ensures: -* End-to-end encryption of your report +* A private advisory workspace hosted by GitHub (not end-to-end encrypted) * Private discussion space for collaboration * Coordinated disclosure tooling * Automatic credit when the advisory is published diff --git a/docs/compliance/2026-09-26-repository-audit.adoc b/docs/compliance/2026-09-26-repository-audit.adoc new file mode 100644 index 0000000..e570e33 --- /dev/null +++ b/docs/compliance/2026-09-26-repository-audit.adoc @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += Docudactyl maintenance and standards audit — 2026-09-26 +:toc: + +== Scope and authority + +Reviewed the four open GitHub issues (#32–#35), HPC source/CI, local agent +constraints, toolchain configuration and immediate correctness hazards. +This is a bounded maintenance review, not a complete security audit or an +RSR certification. No GitHub issues have been closed, no release published, +and no licence text or identifier changed. + +Sources consulted: + +* `hyperpolymath/rsr-template-repo` at + `05bf1b97dd647e02e2a66a33e511a8922ca60e61`: agent doctrine and contribution guide. +* `hyperpolymath/standards` at + `2479cf769ed5f0481ccf64860a2ab954514c2b59`: machine entry point, agent + instructions, constitutional status and RSR checklist. +* Local `0-AI-MANIFEST.a2ml` and `.machine_readable/6a2/AGENTIC.a2ml`. + +Applied the template's fixes-first, no-silent-green, pinned-dependency and +honest-evidence principles. The standards constitutional district is marked +draft; the historical RSR checklist includes Nix and root-state conventions +that conflict with current local constraints. Neither was copied wholesale. +The repository's canonical metadata stays in `.machine_readable/6a2/`. +Licensing conflicts are escalated, not resolved by automated editing. + +== Open issue disposition + +[cols="1,3,2",options="header"] +|=== +|Issue |Changes / treatment |Remaining acceptance gate +|#35 — Zig 0.15.2 +|Mutable DynLib receivers without const-casts; public GPU submit export; +removed an unused obsolete target-feature conversion; POSIX socket timeouts; +fixed io_uring flags, error-union handling and FADVISE SQE construction. +|Standalone modules compile and tests pass on 0.15.2. Full linked FFI build +and integration suite remain blocked locally by missing native libraries. +Do not close on standalone evidence alone. +|#34 — strict Idris2 CI +|Removed `continue-on-error`; cache keyed by OS/architecture and the pinned +0.8.0 source commit; explicit non-root install prefix, verified executable, +and no hardcoded workspace path. Package/config/script edits trigger HPC CI. +|Cold install and cache-hit proof builds must pass in CI. Retained a cached +source bootstrap deliberately: the proposed setup action and prebuilt repo +returned 404, and official v0.8.0 has no binary assets. This is a documented +alternative to the issue's proposed binary installer, not a claim that pack +provides a verified binary here. +|#33 — Chapel warning cleanup +|Renamed six unstable padding declarations without changing field types/order; +replaced three `dmapped` domains with `blockDist.createDomain`. Added an empty +post-load validation guard. +|Chapel 2.8.0 compile/smoke run required. Retain and document DynamicIters, +`string.c_str()` and cross-locale array warnings; do not globally suppress +them or claim they disappeared. Reassess their safety with multi-locale tests. +|#32 — deferred Agda formalisation +|Added link:../decisions/distributed-aggregation-obligations.adoc[implementation-specific +proof obligations] and corrected the historical ADR's unsupported guarantee +language. No placeholder Agda proof or axiom-based "completion" added. +|Deliberately open: ownership, cache semantics, persistence and refinement +must be established before a load-bearing formalisation is possible. +|=== + +== Additional fixes + +* Chapel installation now downloads the actual Ubuntu 24.04 amd64 release + package, verifies its published SHA-256, and installs its dependencies. + The former `linux64-x86_64.tar.gz` asset does not exist in that release. +* HPC smoke uses a generated WAV, exercises shared and broadcast loading, + and asserts exact success/failure counts from JSON. Removed the runner-icon + dependency, silent skip, and ineffective stdin-based success check. +* Added a dependency-light standalone Zig gate and fail-closed CI harness tests. +* Dragonfly replies are bounded and frame-aware across partial/coalesced TCP + reads. Invalid replies disable reuse after protocol errors. Socket timeout + installation failure rejects the connection; allocation failure closes it. + This does not establish thread safety or a connect-time deadline. +* Failed batch hashes no longer hex-encode uninitialised stack bytes: failed + positions return an empty C string; successful positions retain their digest. +* Zero-sized prefetch windows are clamped before modulo arithmetic; queue-full + prefetch falls back to fadvise. +* Fixed quality percentile intermediate integer overflow and nearest-rank + indexing, with zero/one/full-buffer regression tests. +* Corrected a speaker-count test to assert six spoken words (2 + 4); production + word counting was already correct. The assertion is now exact, not loosened. +* GPU state allocation unwinds on text-buffer failure; the unimplemented + Tesseract probe no longer intentionally leaks a library handle. +* Consolidated conflicting mise files into `mise.toml`, retaining the project + pins instead of unrelated `latest` runtimes and success-masking aliases. +* Workflow pin checks now fail on mutable action refs even with trailing + comments; reduced the linter's token to `contents: read`. +* Pinned the Pages generator checkout and disabled credential persistence for it. +* BoJ dispatch is explicit opt-in, requires configured HTTPS/authentication, + constructs JSON with jq, times out, and propagates HTTP failures. This + changes the former automatically failing `.local` push integration; server + configuration remains a maintainer task. +* Replaced generic contribution/agent scaffolding with repository-specific + instructions and corrected the security policy's end-to-end-encryption claim. + +== Validation evidence + +Environment: Debian 12 x86-64; Zig 0.15.2 obtained as a development tool outside +the Git working tree. No downloaded tools, caches or datasets added to Git. + +[cols="2,3",options="header"] +|=== +|Check |Result +|`bash scripts/ci/test-zig-standalone.sh` +|PASS: 66 tests across the explicit module set; modules without tests also +compile. Includes CPU crypto detection, failed hash output, zero prefetch, +RESP framing and percentile regressions. Also reran 25 tests in the changed +Dragonfly, crypto, prefetch, quality-stats and speaker modules with +`-O ReleaseFast`: PASS. GPU/ONNX execution is not tested. +|`bash scripts/ci/test-ci.sh` +|PASS: mutable refs rejected, fixed refs accepted, successful smoke reports +accepted and zero-success reports rejected. Mock checks validate the harness, +not the HPC binary. +|`bash scripts/ci/check-workflow-pins.sh` +|PASS for current workflow refs; this is a syntactic pin check, not an upstream +supply-chain attestation. +|C ABI header with `gcc -std=c11 -fsyntax-only` +|PASS, including the checked-in header's static assertions. Does not prove +agreement with a running Chapel or Idris2 implementation. +|Workflow YAML parsing (YAML 1.2, duplicate keys rejected) +|PASS for all 15 workflows. Actionlint binary download was blocked, so this +is not equivalent to Actionlint semantic validation. +|Bash syntax, modified Zig formatting, banned Idris proof-pattern scan, +`git diff --check` +|PASS. +|Full FFI build / linked tests +|BLOCKED: apt repository access failed over HTTP and HTTPS. Compilation now +reports missing `lmdb.h` / Poppler headers; linked build reports missing +native parser libraries. Further diagnostics may emerge once installed. +|Chapel and Idris2 builds +|NOT RUN: compilers/native prerequisites unavailable. Workflow changes must +be exercised on Ubuntu 24.04 before issue closure or release. +|Podman container, multi-locale stress, GPU/ONNX and external integrations +|NOT RUN; not certified by this patch. +|=== + +== Immediate remaining risks and owner decisions + +. *Do not treat distributed execution as production-certified.* Shared + statistics, checkpoint sets, prefetch/GPU/ML handles and the Dragonfly + request stream need explicit task ownership/synchronisation. See the #32 + plan for counterexamples to the earlier invariant claims. +. *Resume is not aggregate-equivalent.* Indices alone do not restore previous + statistics, and topology changes can hide old checkpoint files. Checkpoint + writes truncate in place and errors are swallowed by callers. +. *Cache hits need semantic and artifact coherence.* Keys omit stage/output + configuration and parser version; timings are not deterministic content; + cache lookup is not a substitute for recreating requested output files. +. *Optional GPU/ONNX implementations contain placeholders.* Library discovery + is not evidence that an inference backend executes. Performance estimates + and backend availability must not be advertised as measured capability. +. *Licensing needs owner review.* README mixes MPL and Palimpsest naming and + points to PMPL text while source headers/local manifest specify MPL-2.0. + Existing licence files/notices were left untouched in accordance with the + estate's no-automated-licence-edit rule. +. *Container reproducibility remains unverified.* Builder/runtime images and + native packages are not fully locked; the Fedora-to-Wolfi dependency seam + needs an actual Podman build and runtime test, not guessed image digests. +. *RSR metadata/deployment still needs review.* Do not fabricate completion + percentages, identity deeds, signing keys or externally served `.well-known` + policies. Older metadata, the licence discrepancy, and release claims + prevent an honest blanket compliance assertion. + +== Next verification sequence + +. Run `HPC Build & Test` on this branch with all native packages available. +. Require strict FFI build/tests, Chapel boundary and end-to-end smoke, Idris2 + cold/cache-hit builds, C header assertions and workflow checks. +. Resolve any further diagnostics without skips or `continue-on-error`. +. Review #33's remaining unstable APIs on the actual pinned compiler. +. Keep #32 open and resolve the documented production blockers before + production-scale deployment; obtain an owner ruling on the licence conflict. diff --git a/docs/decisions/2026-05-30-docudactyl-chapel-rehab.adoc b/docs/decisions/2026-05-30-docudactyl-chapel-rehab.adoc index 3c80b58..e9c6bc0 100644 --- a/docs/decisions/2026-05-30-docudactyl-chapel-rehab.adoc +++ b/docs/decisions/2026-05-30-docudactyl-chapel-rehab.adoc @@ -109,6 +109,13 @@ against in Wave 1. === Parallel-dispatch invariants (informal — deferred to follow-up if formalised) +IMPORTANT: The 2026-09-26 source review supersedes the guarantee language in +this historical inventory. Per-locale slots do not prevent same-locale task +races, timing fields are not deterministic integers, and checkpoints do not +persist aggregate statistics. These are proof obligations, not established +invariants. See link:../compliance/2026-09-26-repository-audit.adoc[the audit] +and link:distributed-aggregation-obligations.adoc[the issue #32 proof plan]. + These document the invariants the docudactyl `+forall idx in dynamic(...)+` loop relies on. They are NOT proved here; they are recorded so a future Agda module has a target. diff --git a/docs/decisions/distributed-aggregation-obligations.adoc b/docs/decisions/distributed-aggregation-obligations.adoc new file mode 100644 index 0000000..0bae5ed --- /dev/null +++ b/docs/decisions/distributed-aggregation-obligations.adoc @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += Distributed aggregation: proof obligations for issue #32 +:toc: + +Status: deferred design and implementation work, NOT a mechanised proof. +No claim of production-scale distributed correctness is made. + +The target module remains `proofs/agda/DistributedAggregationInvariants.agda`. +Do not reuse echidna's speculative first-success proof: Docudactyl dispatches +independent document indices and reduces results. Before writing theorems, +state an operational model and demonstrate refinement from the actual Chapel +and Zig implementations. A theorem over an assumed race-free abstract system +would not establish race freedom of these sources. + +[cols="1,2,2",options="header"] +|=== +|Obligation |Current implementation gap |Required evidence +|At-most-once +|`isAlreadyProcessed` and `recordCheckpoint` are separate operations on +mutable sets. The scheduler's one-visit property is different from exactly-once +side effects under retry or crash. +|Concurrent duplicate-index tests, explicit ownership/locking, idempotent +output commit semantics and crash-point tests. +|Abort-bounded +|`shouldAbort` is polled before processing; in-flight work may finish. +Aborted/conduit-rejected documents are not fully represented by the report's +`accumulate` counts. +|Define a bound in terms of active tasks and retries, and report pending, +skipped, failed and successful indices without double counting. +|Aggregation commutativity +|`perLocaleStats[here.id]` uses non-atomic updates in parallel tasks. Duration +and parse-time fields are floating point, not commutative integer reductions. +|Task-local reductions or locking; exact integer totals; a specified rounding +or ordering policy for floating-point totals, with multi-locale stress tests. +|L1 coherence +|Path/mtime/size do not identify parser version, output format or stages. +Multiple tasks may reference the same path. LMDB transaction guarantees alone +do not prove application cache-key correctness. +|Versioned keys including processing configuration; duplicate-path and +same-metadata/content-change tests; explicit file immutability assumptions. +|L2 coherence +|Content SHA-256 alone does not describe parser configuration. Cached parse +results include timing; a cache hit does not itself recreate output artifacts. +The Dragonfly client is shared by parallel tasks without request serialization. +|Separate deterministic semantic values from timing; versioned keys and output +materialisation; task-owned clients or synchronized full request/reply pairs. +|Checkpoint resumability +|Only indices are persisted, not aggregate statistics or a manifest identity. +The reader scans files for the current locale count, potentially missing old +locale files after shrinking the topology. Writes truncate in place. +|Atomic durable checkpoint commits; manifest/version binding; migration across +locale counts; restored statistics; crash/restart equivalence tests. +|Content-type determinism +|Magic-byte classification can be deterministic for immutable bytes, but the +file may change between validation, hashing, and parsing. +|Define immutable-input/snapshot assumptions and bind detection, hash and parse +to the same input version; test mutation and unsupported formats. +|=== + +Additional shared handles (prefetch, GPU and ML) require an ownership audit. +The changes in this maintenance pass do not fix or certify those concurrency +properties. Model locks/ownership and persistence failures before claiming +these invariants as load-bearing. Keep issue #32 open until the implementation, +refinement argument, regression suite and Agda gate have all been reviewed. diff --git a/ffi/zig/src/dragonfly.zig b/ffi/zig/src/dragonfly.zig index f2dcecd..a06eeb7 100644 --- a/ffi/zig/src/dragonfly.zig +++ b/ffi/zig/src/dragonfly.zig @@ -5,7 +5,7 @@ // Minimal RESP2 client for Dragonfly (Redis-compatible) cross-locale cache. // Supports GET, SET (with TTL), and DEL operations on binary keys and values. // -// Cache key format: "ddac:{sha256_hex}" (65 bytes) +// Cache key format: "ddac:{sha256_hex}" (69 bytes) // Cache value format: raw bytes of ddac_parse_result_t (952 bytes) // // Dragonfly advantages over Redis: @@ -33,6 +33,7 @@ const RESP_NULL_BULK = "$-1\r\n"; pub const DragonflyClient = struct { stream: std.net.Stream, recv_buf: [4096]u8, + healthy: bool = true, /// Connect to a Dragonfly/Redis server. /// Returns null if connection fails. @@ -40,15 +41,15 @@ pub const DragonflyClient = struct { const addr = std.net.Address.parseIp4(host, port) catch return null; const stream = std.net.tcpConnectToAddress(addr) catch return null; - // Set a reasonable timeout (5 seconds). Failure to set timeouts is - // non-fatal — operations will block indefinitely on network stalls, - // but the connection itself is still usable. - stream.handle.setReadTimeout(5_000_000_000) catch |err| { - std.log.debug("Dragonfly: failed to set read timeout: {s}", .{@errorName(err)}); - }; - stream.handle.setWriteTimeout(5_000_000_000) catch |err| { - std.log.debug("Dragonfly: failed to set write timeout: {s}", .{@errorName(err)}); - }; + // A cache must not block extraction indefinitely. Reject connections + // if either timeout cannot be installed (Stream.handle is a raw fd). + const timeout = std.posix.timeval{ .sec = 5, .usec = 0 }; + for ([_]u32{ std.posix.SO.RCVTIMEO, std.posix.SO.SNDTIMEO }) |option| { + std.posix.setsockopt(stream.handle, std.posix.SOL.SOCKET, option, std.mem.asBytes(&timeout)) catch { + stream.close(); + return null; + }; + } return .{ .stream = stream, @@ -65,37 +66,66 @@ pub const DragonflyClient = struct { /// Returns the value bytes (pointing into recv_buf — valid until next call), or null. pub fn get(self: *DragonflyClient, key: []const u8) ?[]const u8 { // Send: *2\r\n$3\r\nGET\r\n${key.len}\r\n{key}\r\n - self.sendCommand(&[_][]const u8{ "GET", key }) catch return null; - return self.readBulkReply() catch null; + if (!self.healthy) return null; + self.sendCommand(&[_][]const u8{ "GET", key }) catch { + self.healthy = false; + return null; + }; + return self.readBulkReply() catch { + self.healthy = false; + return null; + }; } /// SET a binary key-value pair with optional TTL in seconds. /// Returns true on success. pub fn set(self: *DragonflyClient, key: []const u8, value: []const u8, ttl_secs: u32) bool { + if (!self.healthy) return false; if (ttl_secs > 0) { var ttl_buf: [16]u8 = undefined; - const ttl_str = std.fmt.bufPrint(&ttl_buf, "{d}", .{ttl_secs}) catch return false; - self.sendCommand(&[_][]const u8{ "SET", key, value, "EX", ttl_str }) catch return false; + const ttl_str = std.fmt.bufPrint(&ttl_buf, "{d}", .{ttl_secs}) catch { + self.healthy = false; + return false; + }; + self.sendCommand(&[_][]const u8{ "SET", key, value, "EX", ttl_str }) catch { + self.healthy = false; + return false; + }; } else { - self.sendCommand(&[_][]const u8{ "SET", key, value }) catch return false; + self.sendCommand(&[_][]const u8{ "SET", key, value }) catch { + self.healthy = false; + return false; + }; } return self.readSimpleReply() catch false; } /// DEL a key. Returns true if the key was deleted. pub fn del(self: *DragonflyClient, key: []const u8) bool { - self.sendCommand(&[_][]const u8{ "DEL", key }) catch return false; - const n = self.readIntegerReply() catch return false; + if (!self.healthy) return false; + self.sendCommand(&[_][]const u8{ "DEL", key }) catch { + self.healthy = false; + return false; + }; + const n = self.readIntegerReply() catch { + self.healthy = false; + return false; + }; return n > 0; } /// PING — returns true if server responds with PONG. pub fn ping(self: *DragonflyClient) bool { - self.sendCommand(&[_][]const u8{"PING"}) catch return false; - // Expect +PONG\r\n - const n = self.stream.read(&self.recv_buf) catch return false; - if (n >= 7 and std.mem.startsWith(u8, self.recv_buf[0..n], "+PONG\r\n")) return true; - return false; + if (!self.healthy) return false; + self.sendCommand(&[_][]const u8{"PING"}) catch { + self.healthy = false; + return false; + }; + const line = self.readLine() catch { + self.healthy = false; + return false; + }; + return std.mem.eql(u8, line, "+PONG"); } // ── Internal: RESP2 encoding ────────────────────────────────────── @@ -116,62 +146,58 @@ pub const DragonflyClient = struct { } } - fn readBulkReply(self: *DragonflyClient) !?[]const u8 { - // Read response into recv_buf - const n = try self.stream.read(&self.recv_buf); - if (n == 0) return error.ConnectionClosed; - - const resp = self.recv_buf[0..n]; - - // Null bulk string: $-1\r\n - if (n >= 5 and resp[0] == '$' and resp[1] == '-' and resp[2] == '1') return null; - - // Bulk string: ${len}\r\n{data}\r\n - if (resp[0] != '$') return null; - - // Parse length - var i: usize = 1; - var len: usize = 0; - while (i < n and resp[i] != '\r') : (i += 1) { - if (resp[i] < '0' or resp[i] > '9') return null; - len = len * 10 + (resp[i] - '0'); + // TCP preserves bytes, not RESP frame boundaries. Read exactly one frame + // without consuming any bytes from the next response. Bound every length + // before indexing or arithmetic so an untrusted cache cannot overflow. + fn readExact(self: *DragonflyClient, bytes: []u8) !void { + var offset: usize = 0; + while (offset < bytes.len) { + const n = try self.stream.read(bytes[offset..]); + if (n == 0) return error.ConnectionClosed; + offset += n; } - i += 2; // skip \r\n + } - // Data may span multiple reads for large values - if (i + len <= n) { - return resp[i .. i + len]; + fn readLine(self: *DragonflyClient) ![]const u8 { + var used: usize = 0; + while (used < self.recv_buf.len) { + try self.readExact(self.recv_buf[used .. used + 1]); + if (self.recv_buf[used] == '\r') { + var lf: [1]u8 = undefined; + try self.readExact(&lf); + if (lf[0] != '\n') return error.InvalidReply; + return self.recv_buf[0..used]; + } + if (self.recv_buf[used] == '\n') return error.InvalidReply; + used += 1; } + return error.ReplyTooLarge; + } - // Need more data — for our use case (952 bytes), this shouldn't happen - // with a 4KB buffer, but handle gracefully - return null; + fn readBulkReply(self: *DragonflyClient) !?[]const u8 { + const line = try self.readLine(); + if (std.mem.eql(u8, line, "$-1")) return null; + if (line.len < 2 or line[0] != '$') return error.InvalidReply; + for (line[1..]) |ch| { + if (ch < '0' or ch > '9') return error.InvalidReply; + } + const len = std.fmt.parseInt(usize, line[1..], 10) catch return error.InvalidReply; + if (len > self.recv_buf.len) return error.ReplyTooLarge; + try self.readExact(self.recv_buf[0..len]); + var terminator: [2]u8 = undefined; + try self.readExact(&terminator); + if (!std.mem.eql(u8, &terminator, "\r\n")) return error.InvalidReply; + return self.recv_buf[0..len]; } fn readSimpleReply(self: *DragonflyClient) !bool { - const n = try self.stream.read(&self.recv_buf); - if (n == 0) return error.ConnectionClosed; - // +OK\r\n - return n >= 5 and self.recv_buf[0] == RESP_SIMPLE_STRING; + return std.mem.eql(u8, try self.readLine(), "+OK"); } fn readIntegerReply(self: *DragonflyClient) !i64 { - const n = try self.stream.read(&self.recv_buf); - if (n == 0) return error.ConnectionClosed; - if (self.recv_buf[0] != RESP_INTEGER) return 0; - - var i: usize = 1; - var negative = false; - if (i < n and self.recv_buf[i] == '-') { - negative = true; - i += 1; - } - var val: i64 = 0; - while (i < n and self.recv_buf[i] != '\r') : (i += 1) { - if (self.recv_buf[i] < '0' or self.recv_buf[i] > '9') break; - val = val * 10 + @as(i64, self.recv_buf[i] - '0'); - } - return if (negative) -val else val; + const line = try self.readLine(); + if (line.len < 2 or line[0] != ':') return error.InvalidReply; + return std.fmt.parseInt(i64, line[1..], 10) catch error.InvalidReply; } }; @@ -202,10 +228,13 @@ export fn ddac_dragonfly_connect(host_port: [*:0]const u8) ?*anyopaque { break :blk std.fmt.parseInt(u16, port_str, 10) catch 6379; } else 6379; - const client = DragonflyClient.connect(host, port) orelse return null; + var client = DragonflyClient.connect(host, port) orelse return null; // Verify connection - var handle = std.heap.c_allocator.create(DfHandle) catch return null; + const handle = std.heap.c_allocator.create(DfHandle) catch { + client.close(); + return null; + }; handle.client = client; if (!handle.client.ping()) { @@ -291,3 +320,60 @@ export fn ddac_dragonfly_count(handle: *anyopaque) u64 { const n = df.client.readIntegerReply() catch return 0; return if (n >= 0) @intCast(n) else 0; } + +// Pipes deliberately model only the byte stream used by the reply decoder. +// Coalesced frames must remain separate and malformed/oversized frames fail. +fn testReply(bytes: []const u8) !DragonflyClient { + const fds = try std.posix.pipe(); + errdefer std.posix.close(fds[0]); + const writer = std.fs.File{ .handle = fds[1] }; + defer writer.close(); + try writer.writeAll(bytes); + return .{ .stream = .{ .handle = fds[0] }, .recv_buf = undefined }; +} + +test "RESP bulk consumes exactly one frame including binary payload" { + var client = try testReply("$3\r\na\x00b\r\n$0\r\n\r\n$-1\r\n:12\r\n+OK\r\n"); + defer client.close(); + try std.testing.expectEqualStrings("a\x00b", (try client.readBulkReply()).?); + try std.testing.expectEqualStrings("", (try client.readBulkReply()).?); + try std.testing.expect((try client.readBulkReply()) == null); + try std.testing.expectEqual(@as(i64, 12), try client.readIntegerReply()); + try std.testing.expect(try client.readSimpleReply()); +} + +test "RESP rejects invalid lengths and terminators" { + const cases = [_][]const u8{ "$-2\r\n", "$999999999999999999999999999999\r\n", "$1\r\naXX", "$1\nx" }; + for (cases) |bytes| { + var client = try testReply(bytes); + defer client.close(); + try std.testing.expectError(error.InvalidReply, client.readBulkReply()); + } + var oversized = try testReply("$4097\r\n"); + defer oversized.close(); + try std.testing.expectError(error.ReplyTooLarge, oversized.readBulkReply()); + var truncated = try testReply("$3\r\na"); + defer truncated.close(); + try std.testing.expectError(error.ConnectionClosed, truncated.readBulkReply()); +} + +fn writeFragmentedReply(fd: std.posix.fd_t) void { + const writer = std.fs.File{ .handle = fd }; + defer writer.close(); + // Feed the header, payload bytes and trailer independently. + writer.writeAll("$4096\r\n") catch return; + for (0..4096) |_| writer.writeAll("x") catch return; + writer.writeAll("\r\n:1\r\n") catch return; +} + +test "RESP handles payload arriving over multiple writes" { + const fds = try std.posix.pipe(); + var client = DragonflyClient{ .stream = .{ .handle = fds[0] }, .recv_buf = undefined }; + defer client.close(); + const thread = try std.Thread.spawn(.{}, writeFragmentedReply, .{fds[1]}); + defer thread.join(); + const payload = (try client.readBulkReply()).?; + try std.testing.expectEqual(@as(usize, 4096), payload.len); + for (payload) |ch| try std.testing.expectEqual(@as(u8, 'x'), ch); + try std.testing.expectEqual(@as(i64, 1), try client.readIntegerReply()); +} diff --git a/ffi/zig/src/gpu_ocr.zig b/ffi/zig/src/gpu_ocr.zig index 279f1ec..eb510d0 100644 --- a/ffi/zig/src/gpu_ocr.zig +++ b/ffi/zig/src/gpu_ocr.zig @@ -101,6 +101,7 @@ const GpuOcrState = struct { fn init(allocator: std.mem.Allocator) !*GpuOcrState { const state = try allocator.create(GpuOcrState); + errdefer allocator.destroy(state); state.* = .{ .allocator = allocator, .backend = detectBackend(), @@ -159,7 +160,7 @@ fn detectBackend() GpuBackend { fn probeCudaDevices() bool { // Try to dlopen libcudart and query device count // This avoids a hard link-time dependency on CUDA - const lib = std.DynLib.open("libcudart.so") catch + var lib = std.DynLib.open("libcudart.so") catch std.DynLib.open("libcudart.so.12") catch std.DynLib.open("libcudart.so.11") catch return false; @@ -175,7 +176,7 @@ fn probeCudaDevices() bool { /// Check if PaddleOCR inference library is available. fn probePaddleOcr() bool { - const lib = std.DynLib.open("libpaddle_inference.so") catch + var lib = std.DynLib.open("libpaddle_inference.so") catch std.DynLib.open("libpaddle_inference_c.so") catch return false; defer lib.close(); @@ -190,7 +191,7 @@ fn probeTesseractCuda() bool { // Tesseract CUDA is detected by checking if the LSTM engine // initialises with OEM_LSTM_ONLY mode on a GPU-enabled build. // For now, check the shared library for CUDA-specific symbols. - const lib = std.DynLib.open("libtesseract.so") catch + var lib = std.DynLib.open("libtesseract.so") catch std.DynLib.open("libtesseract.so.5") catch return false; defer lib.close(); @@ -207,11 +208,11 @@ fn probeTesseractCuda() bool { /// Initialise a Tesseract instance for CPU OCR fallback. /// Uses the C API directly via dlsym to avoid link-time dependency. fn initTesseract() ?*anyopaque { - const lib = std.DynLib.open("libtesseract.so") catch + var lib = std.DynLib.open("libtesseract.so") catch std.DynLib.open("libtesseract.so.5") catch return null; - // Keep lib open (leaked intentionally — needed for lifetime of program) - _ = lib; + // This probe does not retain any symbols or create a handle yet. + defer lib.close(); // Use the existing Tesseract init from docudactyl_ffi.zig // The GPU coprocessor shares the same Tesseract API @@ -353,7 +354,7 @@ export fn ddac_gpu_ocr_backend(handle: ?*anyopaque) u8 { /// Submit an image for GPU OCR processing. /// Images are queued until the batch is full or ddac_gpu_ocr_flush() is called. /// Returns: slot ID (0..MAX_BATCH_SIZE-1) on success, -1 if queue full. -export fn ddac_gpu_ocr_submit( +pub export fn ddac_gpu_ocr_submit( handle: ?*anyopaque, image_path: [*:0]const u8, output_path: [*:0]const u8, diff --git a/ffi/zig/src/hw_crypto.zig b/ffi/zig/src/hw_crypto.zig index f217af4..bba6113 100644 --- a/ffi/zig/src/hw_crypto.zig +++ b/ffi/zig/src/hw_crypto.zig @@ -62,8 +62,6 @@ fn detectCapabilities() CryptoCapabilities { switch (target.cpu.arch) { .x86_64 => { // Use CPUID to detect x86-64 features - const features = std.Target.x86.featureSet(target.cpu.features); - _ = features; // Runtime CPUID check (works on any x86-64 CPU) // CPUID leaf 7, ECX=0: EBX bit 29 = SHA, bit 5 = AVX2, bit 16 = AVX-512F @@ -152,9 +150,11 @@ const MULTI_BUFFER_LANES: usize = 4; fn batchSha256( paths: [*]const [*:0]const u8, digests: [*][32]u8, + valid: [*]bool, count: u32, ) u32 { var success: u32 = 0; + @memset(valid[0..count], false); // Process in groups of MULTI_BUFFER_LANES for better cache utilisation var base: u32 = 0; @@ -193,6 +193,7 @@ fn batchSha256( if (n == 0) { // File complete — finalise digests[base + i] = hashers[i].finalResult(); + valid[base + i] = true; success += 1; active[i] = false; } else { @@ -255,6 +256,7 @@ export fn ddac_crypto_sha256_name() [*:0]const u8 { /// Batch SHA-256: compute digests for N files. /// paths: array of N null-terminated path pointers /// hex_out: array of N * 65-byte hex digest buffers +/// Failed positions contain an empty C string (all 65 bytes zeroed). /// count: number of files /// Returns number of successfully hashed files. export fn ddac_crypto_batch_sha256( @@ -266,11 +268,16 @@ export fn ddac_crypto_batch_sha256( if (count > 1024) return 0; // Safety limit var raw_digests: [1024][32]u8 = undefined; - const success = batchSha256(paths, &raw_digests, count); + var valid: [1024]bool = undefined; + const success = batchSha256(paths, &raw_digests, &valid, count); // Convert to hex for (0..count) |i| { - digestToHex(raw_digests[i], &hex_out[i]); + if (valid[i]) { + digestToHex(raw_digests[i], &hex_out[i]); + } else { + @memset(&hex_out[i], 0); // empty C string, never uninitialised stack bytes + } } return success; @@ -280,3 +287,19 @@ export fn ddac_crypto_batch_sha256( export fn ddac_crypto_caps_size() usize { return @sizeOf(CryptoCapabilities); } + +test "batch hashing isolates failed files without exposing stack bytes" { + const paths = [_][*:0]const u8{ "/dev/null", "/nonexistent/docudactyl-missing-file", "/dev/null" }; + var output: [3][65]u8 = undefined; + @memset(std.mem.asBytes(&output), 0xaa); + try std.testing.expectEqual(@as(u32, 2), ddac_crypto_batch_sha256(&paths, &output, paths.len)); + const empty_sha = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + try std.testing.expectEqualStrings(empty_sha, std.mem.sliceTo(&output[0], 0)); + try std.testing.expectEqualStrings(empty_sha, std.mem.sliceTo(&output[2], 0)); + try std.testing.expectEqual([_]u8{0} ** 65, output[1]); +} + +test "runtime crypto detection returns a supported tier" { + const caps = detectCapabilities(); + try std.testing.expect(caps.sha256_tier <= 2); +} diff --git a/ffi/zig/src/ml_inference.zig b/ffi/zig/src/ml_inference.zig index 6e587c2..d7656e3 100644 --- a/ffi/zig/src/ml_inference.zig +++ b/ffi/zig/src/ml_inference.zig @@ -30,21 +30,21 @@ const std = @import("std"); /// ML-dependent stages (must match DDAC_STAGE_* bitmask positions) const MlStage = enum(u8) { - ner = 0, // STAGE_NER (bit 14) - whisper = 1, // STAGE_WHISPER (bit 15) - image_classify = 2, // STAGE_IMAGE_CLASSIFY (bit 16) - layout_analysis = 3, // STAGE_LAYOUT_ANALYSIS (bit 17) - handwriting_ocr = 4, // STAGE_HANDWRITING_OCR (bit 18) + ner = 0, // STAGE_NER (bit 14) + whisper = 1, // STAGE_WHISPER (bit 15) + image_classify = 2, // STAGE_IMAGE_CLASSIFY (bit 16) + layout_analysis = 3, // STAGE_LAYOUT_ANALYSIS (bit 17) + handwriting_ocr = 4, // STAGE_HANDWRITING_OCR (bit 18) }; const ML_STAGE_COUNT: usize = 5; /// Execution provider preference const ExecProvider = enum(u8) { - tensorrt = 0, // NVIDIA TensorRT (INT8/FP16) - cuda = 1, // NVIDIA CUDA (FP32) - openvino = 2, // Intel OpenVINO - cpu = 3, // CPU (ONNX default) + tensorrt = 0, // NVIDIA TensorRT (INT8/FP16) + cuda = 1, // NVIDIA CUDA (FP32) + openvino = 2, // Intel OpenVINO + cpu = 3, // CPU (ONNX default) }; // ============================================================================ @@ -113,17 +113,20 @@ fn probeOnnxRuntime() OrtApi { // Probe execution providers (best first) // TensorRT: check for libonnxruntime_providers_tensorrt - if (std.DynLib.open("libonnxruntime_providers_tensorrt.so")) |lib| { + if (std.DynLib.open("libonnxruntime_providers_tensorrt.so")) |loaded_lib| { + var lib = loaded_lib; lib.close(); api.provider = .tensorrt; } else |_| { // CUDA: check for libonnxruntime_providers_cuda - if (std.DynLib.open("libonnxruntime_providers_cuda.so")) |lib| { + if (std.DynLib.open("libonnxruntime_providers_cuda.so")) |loaded_lib| { + var lib = loaded_lib; lib.close(); api.provider = .cuda; } else |_| { // OpenVINO: check for libonnxruntime_providers_openvino - if (std.DynLib.open("libonnxruntime_providers_openvino.so")) |lib| { + if (std.DynLib.open("libonnxruntime_providers_openvino.so")) |loaded_lib| { + var lib = loaded_lib; lib.close(); api.provider = .openvino; } else |_| { diff --git a/ffi/zig/src/prefetch.zig b/ffi/zig/src/prefetch.zig index 515afd2..2f5e667 100644 --- a/ffi/zig/src/prefetch.zig +++ b/ffi/zig/src/prefetch.zig @@ -52,7 +52,7 @@ export fn ddac_prefetch_init(window_size: u32) ?*anyopaque { state.* = .{ .fds = [_]std.posix.fd_t{-1} ** MAX_INFLIGHT, .paths = [_]?[*:0]const u8{null} ** MAX_INFLIGHT, - .count = @min(window_size, MAX_INFLIGHT), + .count = @max(1, @min(window_size, MAX_INFLIGHT)), .head = 0, .use_uring = false, .ring = if (builtin.os.tag == .linux) null else {}, @@ -60,7 +60,7 @@ export fn ddac_prefetch_init(window_size: u32) ?*anyopaque { // Try to initialise io_uring (Linux 5.6+) if (builtin.os.tag == .linux) { - if (std.os.linux.IoUring.init(64, .{})) |ring| { + if (std.os.linux.IoUring.init(64, 0)) |ring| { state.ring = .{ .ring = ring }; state.use_uring = true; } else |_| { @@ -172,13 +172,14 @@ fn prefetchWithUring(state: *PrefetchState, fd: std.posix.fd_t) void { if (builtin.os.tag != .linux) return; if (state.ring) |*ring_state| { - // Submit a NOP operation as a readahead trigger - // (io_uring doesn't have a native readahead op, but opening + fadvise - // is triggered by the kernel when we submit a read) - // - // Use IORING_OP_FADVISE if available (kernel 5.6+) - var sqe = ring_state.ring.get_sqe() orelse return; - sqe.prep_fadvise(fd, 0, @intCast(READAHEAD_SIZE), 3); // POSIX_FADV_WILLNEED + // Zig 0.15 has no prep_fadvise helper. Initialise the complete SQE + // through prep_rw; Linux uses rw_flags as fadvise_advice for this op. + const sqe = ring_state.ring.get_sqe() catch { + prefetchWithFadvise(fd); + return; + }; + sqe.prep_rw(.FADVISE, fd, 0, READAHEAD_SIZE, 0); + sqe.rw_flags = 3; // POSIX_FADV_WILLNEED sqe.user_data = @intCast(fd); // Submit without waiting — failure is non-fatal; the file will still @@ -199,3 +200,13 @@ fn drainUring(state: *PrefetchState) void { } } } + +test "zero prefetch window is clamped before modulo" { + const handle = ddac_prefetch_init(0) orelse return error.OutOfMemory; + defer ddac_prefetch_free(handle); + const state: *PrefetchState = @ptrCast(@alignCast(handle)); + try std.testing.expectEqual(@as(usize, 1), state.count); + ddac_prefetch_hint(handle, "/dev/null"); + try std.testing.expectEqual(@as(u32, 1), ddac_prefetch_inflight(handle)); + ddac_prefetch_done(handle, "/dev/null"); +} diff --git a/ffi/zig/src/quality_stats.zig b/ffi/zig/src/quality_stats.zig index a955b41..fc72832 100644 --- a/ffi/zig/src/quality_stats.zig +++ b/ffi/zig/src/quality_stats.zig @@ -379,7 +379,7 @@ pub fn computeTimingPercentiles(stats: *const QualityStats, stage_idx: usize) Ti const count = stats.stage_timing_counts[stage_idx]; if (count == 0) return result; - const n = @min(@as(usize, count), MAX_TIMING_SAMPLES); + const n: usize = @min(@as(usize, count), MAX_TIMING_SAMPLES); const base = stage_idx * MAX_TIMING_SAMPLES; // Copy samples to sort @@ -398,9 +398,9 @@ pub fn computeTimingPercentiles(stats: *const QualityStats, stage_idx: usize) Ti result.mean_us = sum / @as(f64, @floatFromInt(n)); // Percentiles via nearest-rank method - result.p50_us = samples[@min(n * 50 / 100, n - 1)]; - result.p95_us = samples[@min(n * 95 / 100, n - 1)]; - result.p99_us = samples[@min(n * 99 / 100, n - 1)]; + result.p50_us = samples[(n * 50 + 99) / 100 - 1]; + result.p95_us = samples[(n * 95 + 99) / 100 - 1]; + result.p99_us = samples[(n * 99 + 99) / 100 - 1]; return result; } @@ -702,3 +702,19 @@ test "empty stats produces valid JSON" { try std.testing.expect(json[0] == '{'); try std.testing.expect(json[len - 1] == '}'); } + +test "nearest-rank percentiles at zero one and maximum sample capacity" { + var stats: QualityStats = undefined; + qualityStatsInit(&stats); + try std.testing.expectEqual(@as(f64, 0), computeTimingPercentiles(&stats, 0).p99_us); + recordTiming(&stats, 0, 42); + const one = computeTimingPercentiles(&stats, 0); + try std.testing.expectEqual(@as(f64, 42), one.p50_us); + try std.testing.expectEqual(@as(f64, 42), one.p99_us); + qualityStatsInit(&stats); + for (1..MAX_TIMING_SAMPLES + 1) |i| recordTiming(&stats, 0, @floatFromInt(i)); + const full = computeTimingPercentiles(&stats, 0); + try std.testing.expectEqual(@as(f64, 512), full.p50_us); + try std.testing.expectEqual(@as(f64, 973), full.p95_us); + try std.testing.expectEqual(@as(f64, 1014), full.p99_us); +} diff --git a/ffi/zig/src/speaker_id.zig b/ffi/zig/src/speaker_id.zig index f61eed4..3d39d36 100644 --- a/ffi/zig/src/speaker_id.zig +++ b/ffi/zig/src/speaker_id.zig @@ -235,8 +235,8 @@ fn extractSpeakerLabel(line: []const u8) ?struct { label: []const u8, content_st // "MR./MS./MRS./DR./JUDGE LASTNAME:" pattern const title_prefixes = [_][]const u8{ - "MR. ", "MS. ", "MRS. ", "DR. ", - "Mr. ", "Ms. ", "Mrs. ", "Dr. ", + "MR. ", "MS. ", "MRS. ", "DR. ", + "Mr. ", "Ms. ", "Mrs. ", "Dr. ", "JUDGE ", "Judge ", }; for (title_prefixes) |prefix| { @@ -259,8 +259,7 @@ fn extractSpeakerLabel(line: []const u8) ?struct { label: []const u8, content_st var has_alpha = false; while (end < trimmed.len and end < 50 and trimmed[end] != ':') : (end += 1) { const ch = trimmed[end]; - if (isUpper(ch)) has_alpha = true - else if (ch != ' ' and ch != '.' and ch != '-') break; + if (isUpper(ch)) has_alpha = true else if (ch != ' ' and ch != '.' and ch != '-') break; } if (has_alpha and end >= 2 and end < trimmed.len and trimmed[end] == ':') { const label = trimmed[0..end]; @@ -506,7 +505,7 @@ test "word counting per speaker" { \\A: Good morning to you. ; _ = speakerIdProcess(text, &result); - try std.testing.expect(result.total_words >= 7); // "Hello world" + "Good morning to you" + try std.testing.expectEqual(@as(u32, 6), result.total_words); // 2 + 4 spoken words; labels excluded. } test "THE COURT label detection" { diff --git a/mise.toml b/mise.toml index 6dd983f..2f1092a 100644 --- a/mise.toml +++ b/mise.toml @@ -1,57 +1,16 @@ [tools] -# Language runtimes -node = "latest" -python = "latest" -rust = "latest" -go = "latest" -zig = "latest" -java = "latest" -bun = "latest" -denojs = "latest" - -# Package managers -npm = "latest" -yarn = "latest" -pnpm = "latest" -pip = "latest" -cargo = "latest" -go-task = "latest" - -# Formatting & Linting -gofmt = "latest" -black = "latest" -isort = "latest" -ruff = "latest" -prettier = "latest" -shfmt = "latest" -stylua = "latest" - -# Build tools -cmake = "latest" -make = "latest" -ninja = "latest" - -# Shell tools -git = "latest" -gnu-sed = "latest" -gnu-tar = "latest" -gnu-grep = "latest" - -# Testing -vitest = "latest" -pytest = "latest" -jest = "latest" - -[env] -# Common environment variables -NODE_ENV = "development" -PYTHONDONTWRITEBYTECODE = "1" -PYTHONUNBUFFERED = "1" - -# Task runner alias -[alias] -task = "go-task" -build = "cargo build --release || npm run build || go build" -test = "cargo test || npm test || go test ./..." -lint = "ruff check . || prettier --check . || black --check ." -fmt = "ruff format . || prettier --write . || black ." +# = "Docudactyl HPC — Pinned tool versions (mise)" +# = "Project toolchain targets; see the audit for actual verification scope." +# = "Upgrading any of these requires testing:" +# = "- Zig: build.zig API changes (0.14→0.15 broke addSharedLibrary)" +# = "- Idris2: ABI proof syntax may change" +# = "- OCaml: dune build compatibility" +# = "- Julia: legacy component only" +# = "Chapel is NOT managed by mise —" +# = "Required version: 2.8.0" +# = "SPDX-License-Identifier: MPL-2.0" +zig = "0.15.2" +idris2 = "0.8.0" +ocaml = "5.4.1" +julia = "1.12.5" +just = "1.46.0" diff --git a/scripts/ci/check-workflow-pins.sh b/scripts/ci/check-workflow-pins.sh new file mode 100755 index 0000000..a1e5480 --- /dev/null +++ b/scripts/ci/check-workflow-pins.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +# Check the actual ref, not a trailing '# SHA' comment. Local actions are +# allowed; container actions must be content-addressed too. +status=0 +if (( $# == 0 )); then set -- .github/workflows; fi +for file in "$@"; do + if [[ -d "$file" ]]; then + bash "$0" "$file"/*.yml "$file"/*.yaml || status=1 + continue + fi + [[ -f "$file" ]] || continue + while IFS= read -r line; do + if [[ $line =~ ^[[:space:]-]*uses:[[:space:]]*([^[:space:]#]+) ]]; then + ref=${BASH_REMATCH[1]} + ref=${ref//\'/} + ref=${ref//\"/} + if [[ $ref == ./* || $ref =~ ^[^@]+@[a-fA-F0-9]{40}$ || $ref =~ ^docker://[^@]+@sha256:[a-fA-F0-9]{64}$ ]]; then + continue + fi + echo "ERROR: $file: unpinned action $ref" >&2 + status=1 + fi + done < "$file" +done +exit "$status" diff --git a/scripts/ci/install-chapel.sh b/scripts/ci/install-chapel.sh new file mode 100755 index 0000000..00de4d0 --- /dev/null +++ b/scripts/ci/install-chapel.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +# Official Ubuntu 24.04 amd64 package. The linux64 tarball is not a release asset. +[[ ${CHAPEL_VERSION:-2.8.0} == 2.8.0 ]] || { echo 'Update the package digest when changing Chapel' >&2; exit 1; } +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +package=chapel-2.8.0-1.ubuntu24.amd64.deb +curl --fail --location --retry 3 "https://github.com/chapel-lang/chapel/releases/download/2.8.0/$package" -o "$tmp/$package" +echo "278e1111b8d8b8c3d45d8c2b0976bea21f68509a15847d513cf68113c2f4d59a $tmp/$package" | sha256sum --check --strict +sudo apt-get update -qq +sudo apt-get install -y --no-install-recommends "$tmp/$package" +chpl --version diff --git a/scripts/ci/install-idris2.sh b/scripts/ci/install-idris2.sh new file mode 100755 index 0000000..4c4dd60 --- /dev/null +++ b/scripts/ci/install-idris2.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +# Official v0.8.0, pinned to its commit. No sudo: bootstrap and install must +# agree on PREFIX, otherwise sudo installs under /root and CI cannot find it. +[[ ${IDRIS2_VERSION:-0.8.0} == 0.8.0 ]] || { echo 'Update the Idris2 commit and cache key together' >&2; exit 1; } +revision=15a3e4e70843f7a34100f6470c04b791330788df +prefix="$HOME/.local/idris2" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +git init -q "$tmp/src" +git -C "$tmp/src" remote add origin https://github.com/idris-lang/Idris2.git +git -C "$tmp/src" fetch --depth 1 origin "$revision" +git -C "$tmp/src" checkout --detach FETCH_HEAD +[[ $(git -C "$tmp/src" rev-parse HEAD) == "$revision" ]] +make -C "$tmp/src" bootstrap SCHEME=chezscheme PREFIX="$prefix" +make -C "$tmp/src" install PREFIX="$prefix" +"$prefix/bin/idris2" --version diff --git a/scripts/ci/smoke-hpc.sh b/scripts/ci/smoke-hpc.sh new file mode 100755 index 0000000..56fa7d3 --- /dev/null +++ b/scripts/ci/smoke-hpc.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +# A generated, deterministic WAV avoids dependencies on runner icons/documents. +# PCM: mono, 8000 Hz, 16-bit, eight samples. Audio parsing requires no OCR models. +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +printf 'RIFF\x34\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x40\x1f\x00\x00\x80\x3e\x00\x00\x02\x00\x10\x00data\x10\x00\x00\x00' > "$tmp/tone.wav" +printf '\0%.0s' {1..16} >> "$tmp/tone.wav" +printf '%s\n' "$tmp/tone.wav" > "$tmp/manifest.txt" +export LD_LIBRARY_PATH="$PWD/ffi/zig/zig-out/lib:${LD_LIBRARY_PATH:-}" +# Both loading strategies must actually parse the fixture, not merely exit zero. +for mode in shared broadcast; do + bin/docudactyl-hpc --manifestPath="$tmp/manifest.txt" \ + --manifestMode="$mode" --outputDir="$tmp/$mode" --chunkSize=1 + test -s "$tmp/$mode/run-report.scm" + jq -e '.summary | .totalDocs == 1 and .succeeded == 1 and .failed == 0' \ + "$tmp/$mode/run-report.json" +done diff --git a/scripts/ci/test-ci.sh b/scripts/ci/test-ci.sh new file mode 100755 index 0000000..8cd1971 --- /dev/null +++ b/scripts/ci/test-ci.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +root=$(cd "$(dirname "$0")/../.." && pwd) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +for ref in 'owner/action@v1 # 0123456789012345678901234567890123456789' 'owner/action@main' 'docker://image:latest'; do + printf ' - uses: %s\n' "$ref" > "$tmp/bad.yml" + if bash "$root/scripts/ci/check-workflow-pins.sh" "$tmp/bad.yml"; then + echo "FAIL: accepted unpinned ref: $ref" >&2; exit 1 + fi +done +printf '%s\n' ' - uses: owner/action@0123456789012345678901234567890123456789 # v1' \ + ' uses: ./local-action' > "$tmp/good.yml" +bash "$root/scripts/ci/check-workflow-pins.sh" "$tmp/good.yml" +# Prove the smoke harness rejects zero-success reports, independently of the +# expensive native build. This mock does NOT validate the HPC implementation. +mkdir -p "$tmp/bin" +cat > "$tmp/bin/docudactyl-hpc" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +for arg in "$@"; do + case "$arg" in + --manifestPath=*) manifest=${arg#*=} ;; + --outputDir=*) output=${arg#*=} ;; + esac +done +fixture=$(cat "$manifest") +[[ $(wc -c < "$fixture") == 60 ]] +[[ $(head -c 4 "$fixture") == RIFF ]] +mkdir -p "$output" +printf '(report)\n' > "$output/run-report.scm" +printf '{"summary":{"totalDocs":1,"succeeded":%s,"failed":%s}}\n' \ + "${MOCK_SUCCESS:-1}" "$((1 - ${MOCK_SUCCESS:-1}))" > "$output/run-report.json" +MOCK +chmod +x "$tmp/bin/docudactyl-hpc" +(cd "$tmp" && bash "$root/scripts/ci/smoke-hpc.sh") +if (cd "$tmp" && MOCK_SUCCESS=0 bash "$root/scripts/ci/smoke-hpc.sh"); then + echo 'FAIL: smoke accepted zero successful documents' >&2; exit 1 +fi +echo 'PASS: CI harness regression tests' diff --git a/scripts/ci/test-zig-standalone.sh b/scripts/ci/test-zig-standalone.sh new file mode 100755 index 0000000..f9105e2 --- /dev/null +++ b/scripts/ci/test-zig-standalone.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail +# Fast checks without Poppler/GDAL/etc. Explicit list: never silently skip a +# module after an import changes. The full `zig build test` remains mandatory. +for module in capnp conduit dragonfly entity_graph evasion_detect \ + financial_extract flight_log gpu_ocr hw_crypto investigator_summary \ + legal_ner ml_inference prefetch quality_stats speaker_id; do + zig test "ffi/zig/src/$module.zig" -lc +done diff --git a/src/chapel/FFIBridge.chpl b/src/chapel/FFIBridge.chpl index 2df44d1..223df89 100644 --- a/src/chapel/FFIBridge.chpl +++ b/src/chapel/FFIBridge.chpl @@ -182,7 +182,7 @@ module FFIBridge { var status: uint(8); // 0=ok, 1-4=error codes var stage: uint(8); // MlStage enum (0-4) var provider: uint(8); // ExecProvider enum (0-3) - var _pad: c_array(uint(8), 5); // alignment padding + var padA: c_array(uint(8), 5); // alignment padding var inference_time_us: int(64); // inference time (microseconds) var output_count: int(64); // entities/tokens/labels var confidence: real(64); // 0.0-1.0, -1.0 if N/A @@ -248,9 +248,9 @@ module FFIBridge { var has_arm_sha2: uint(8); // AArch64 SHA2 var has_arm_sha512: uint(8); // AArch64 SHA-512 var has_aes_ni: uint(8); // x86-64 AES-NI - var _pad: c_array(uint(8), 2); // padding + var padA: c_array(uint(8), 2); // padding var sha256_tier: uint(8); // 0=dedicated, 1=AVX2, 2=software - var _pad2: c_array(uint(8), 7); // padding + var padB: c_array(uint(8), 7); // padding } /** Detect hardware crypto capabilities. */ @@ -281,7 +281,7 @@ module FFIBridge { extern record ddac_ocr_result_t { var status: uint(8); // 0=success, 1=error, 2=skipped, 3=gpu_error var confidence: int(8); // OCR confidence 0-100, -1 if unavailable - var _pad: c_array(uint(8), 6); // alignment padding + var padA: c_array(uint(8), 6); // alignment padding var char_count: int(64); // characters extracted var word_count: int(64); // words extracted var gpu_time_us: int(64); // GPU processing time (microseconds) @@ -345,10 +345,10 @@ module FFIBridge { extern record ddac_conduit_result_t { var content_kind: uint(8); // ContentKind (0-6) from magic bytes var validation: uint(8); // 0=ok, 1=not_found, 2=empty, 3=unreadable - var _pad: c_array(uint(8), 6); // alignment padding + var padA: c_array(uint(8), 6); // alignment padding var file_size: int(64); // file size in bytes var sha256: c_array(c_char, 65); // hex SHA-256 + null - var _pad2: c_array(c_char, 7); // alignment padding + var padB: c_array(c_char, 7); // alignment padding } /** Pre-process a single file: detect content type via magic bytes, diff --git a/src/chapel/ManifestLoader.chpl b/src/chapel/ManifestLoader.chpl index a0d2145..a745728 100644 --- a/src/chapel/ManifestLoader.chpl +++ b/src/chapel/ManifestLoader.chpl @@ -82,7 +82,7 @@ module ManifestLoader { writeln("[manifest] ", lineCount, " entries found in ", manifestPath); // ── Pass 2: read into block-distributed array ─────────────────── - const entryDom = {0..#lineCount} dmapped new blockDist({0..#lineCount}); + const entryDom = blockDist.createDomain({0..#lineCount}); var docEntries: [entryDom] DocEntry; var actualCount = 0; @@ -111,7 +111,7 @@ module ManifestLoader { writeln("[manifest] Note: ", lineCount - actualCount, " fewer entries read in pass 2 (", actualCount, " vs ", lineCount, "); using ", actualCount, " entries"); - const fixedDom = {0..#actualCount} dmapped new blockDist({0..#actualCount}); + const fixedDom = blockDist.createDomain({0..#actualCount}); var fixedEntries: [fixedDom] DocEntry; forall i in fixedDom do fixedEntries[i] = docEntries[i]; validateEntrySample(fixedEntries, actualCount); @@ -192,7 +192,7 @@ module ManifestLoader { } // ── Distribute from locale 0 to all locales ───────────────────── - const entryDom = {0..#lineCount} dmapped new blockDist({0..#lineCount}); + const entryDom = blockDist.createDomain({0..#lineCount}); var docEntries: [entryDom] DocEntry; forall i in entryDom do docEntries[i] = localEntries[i]; @@ -210,6 +210,10 @@ module ManifestLoader { /** Sample 0.1% of entries for path existence (sanity check on locale 0). */ proc validateEntrySample(const ref entries, count: int) throws { + // A manifest may become empty between counting and reading. Do not sample + // the invalid range 0..-1 or index an empty distributed array. + if count <= 0 then + throw new Error("Manifest has no entries after loading"); const sampleSize = max(1, count / 1000); var rng = new randomStream(int); var existCount = 0;