From aa7dba438c0ff4e815991c76d69753d55e3bc07a Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sun, 4 Oct 2026 22:29:00 -0400 Subject: [PATCH 1/2] Migrate hosted Jev decision provider to Cloudflare Clef Adds the Clef adapter (clef and clef-flash) for typed decide cells in both the TypeScript and Rust runtimes, including embedded image support, four-decimal provider rounding feasibility, and a browser-safe image type contract. Credentials are environment-only and requests are single-attempt with strict envelope validation. The historical Jev pipeline tests and evidence stay byte-identical. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- AGENTS.md | 4 +- README.md | 71 +- cli.ts | 58 +- crates/algal/examples/clef_parity.rs | 92 +++ crates/algal/src/bench.rs | 11 +- crates/algal/src/clef.rs | 706 ++++++++++++++++++ crates/algal/src/credentials.rs | 24 + crates/algal/src/decisions.rs | 7 +- crates/algal/src/effects.rs | 55 +- crates/algal/src/lib.rs | 1 + crates/algal/src/main.rs | 62 +- crates/algal/tests/clef.rs | 213 ++++++ crates/algal/tests/cli_help.rs | 2 +- docs/cumulative-skill-experiment.md | 7 + docs/executors.md | 107 ++- docs/model-router.md | 25 +- examples/model-router.algal.json | 2 +- experiments/cumulative-skill/README.md | 17 +- .../pipeline/record-classify-clef.algal.json | 33 + .../pipeline/record-triage-clef.algal.json | 70 ++ index.ts | 2 + package.json | 1 + scripts/clef-fixture.ts | 63 ++ scripts/clef-parity.ts | 74 ++ scripts/fixtures/clef.json | 51 ++ spec/v1/organism.md | 55 +- src/clef-cli.test.ts | 50 ++ src/clef-fixture.test.ts | 19 + src/clef-image.ts | 1 + src/clef.test.ts | 184 +++++ src/clef.ts | 253 +++++++ src/cli-golden/help.txt | 2 +- src/cli-help.ts | 2 +- src/credentials.ts | 8 +- src/decisions.ts | 12 +- src/effects.ts | 2 + src/experiment-pipeline-clef.test.ts | 50 ++ src/jev.ts | 2 + 38 files changed, 2320 insertions(+), 78 deletions(-) create mode 100644 crates/algal/examples/clef_parity.rs create mode 100644 crates/algal/src/clef.rs create mode 100644 crates/algal/tests/clef.rs create mode 100644 experiments/cumulative-skill/pipeline/record-classify-clef.algal.json create mode 100644 experiments/cumulative-skill/pipeline/record-triage-clef.algal.json create mode 100644 scripts/clef-fixture.ts create mode 100644 scripts/clef-parity.ts create mode 100644 scripts/fixtures/clef.json create mode 100644 src/clef-cli.test.ts create mode 100644 src/clef-fixture.test.ts create mode 100644 src/clef-image.ts create mode 100644 src/clef.test.ts create mode 100644 src/clef.ts create mode 100644 src/experiment-pipeline-clef.test.ts diff --git a/AGENTS.md b/AGENTS.md index fee32a4e..2d670632 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,8 +19,8 @@ tools (`tools.ts`), opaque capability handles (`capabilities.ts`) and the bounded durable mailbox driver (`mailbox.ts`), the named durable process supervisor (`process.ts`), the provider-neutral - typed-decision layer (`decisions.ts`) with the TypeSafe Jev adapter - (`jev.ts`), cross-platform + typed-decision layer (`decisions.ts`) with the Cloudflare Clef adapter + (`clef.ts`, native `clef.rs`) and explicit legacy Jev adapter (`jev.ts`), cross-platform credential custody (`credentials.ts`), embeddings (`embeddings.ts`) and the derived semantic index plus recall executor (`semantic.ts`), foundry evaluation and search (`foundry.ts`, `search.ts`) with the habitat-wide diff --git a/README.md b/README.md index c070e629..726ae74b 100644 --- a/README.md +++ b/README.md @@ -522,15 +522,16 @@ Cell kinds: question map (`{"answers":{…}}`), the effect request carries `questions`, and the provider must return exactly the declared answers. Decision executors serve `decide` and `classifier` cells only — they can never - generate agent output or approve a gate. Jev (TypeSafe `systemone`) is - the first such provider, admitted with `--jev`; its key lives in - `TYPESAFE_API_KEY` or the local vault via `algal auth jev`. + generate agent output or approve a gate. Cloudflare Clef serves these + decisions when you select `--clef` (`clef` by default, or `clef-flash`). + Paid inference is opt-in. Set `CLOUDFLARE_ACCOUNT_ID` and + `CLOUDFLARE_API_TOKEN`; `CLOUDFLARE_AUTH_TOKEN` is a token alias. - `recall` — an expression-derived semantic query over a host-owned index, recorded as an ordinary effect. `out` carries bounded ranked hits; when the first hit names a value-store object, `ref` carries its `sha256:` token directly into `load`. Empty recall is successful and leaves ref consumers skipped. Optional `rerank:{route,take?}` sends one recorded `noul` relevance - question per hit to a decision provider such as Jev, then reorders the exact + question per hit to a decision provider such as Clef, then reorders the exact source records without summarizing them. The request binds query, `k`, and embedder; replay serves the recorded hits and decisions without consulting a mutable index. @@ -794,7 +795,7 @@ contract data: it changes nothing about digests, receipts, or replay. `recall` cell makes the same capability available inside an organism through a recorded effect. Its bounded expression produces the query, `out` exposes ranked text and provenance, and a top value hit also emits `ref` for direct -`load` resolution. Add `rerank:{route:{provider:"jev"},take:…}` to score each +`load` resolution. Add `rerank:{route:{provider:"clef"},take:…}` to score each dynamic hit through the typed decision seam before choosing that top ref; the original hit records remain intact and the `decide` answers ride the receipt. Index-backed recall is deliberately not effect-cached; replay comes from the @@ -812,28 +813,60 @@ bun run cli run organism-with-recall.json --dir .algal --recall local When one organism combines recall with another effect provider, give the recall cell an explicit route such as `{"provider":"memory"}` and its rerank policy a decision route such as `{"provider":"judge"}`. The Bun `--executors` -map can admit `"memory":"recall"` and `"judge":"jev"`; native +map can attach `"memory":"recall"` and `"judge":"clef"`; native `algal.host.v1` entries use `"memory":{"kind":"recall","dir":".algal","embedder":"local"}` and a -separate Jev backend. The derived index implementations use different +separate `"judge":{"kind":"clef"}` backend. The derived index implementations use different disposable storage (SQLite in TypeScript, JSONL natively) but produce the same local vectors and hybrid ranking. ## Provider credentials -Provider keys never enter manifests, receipts, digests, or logs — they resolve -at the executor boundary only. `auth` vaults them locally cross-platform -(macOS Keychain, libsecret, Windows DPAPI, or — when no vault is available — -a plaintext file under `~/.algal/credentials` (or `$ALGAL_HOME`) with mode -0600; both CLIs print a warning when they fall back to it); the provider env var always works as a CI -escape hatch. +The Cloudflare Clef CLIs read `CLOUDFLARE_ACCOUNT_ID` (32 lowercase +hexadecimal characters) and `CLOUDFLARE_API_TOKEN` from the environment. +The token may also come from `CLOUDFLARE_AUTH_TOKEN`. Library hosts may +supply an explicit in-memory credential or async resolver. Tokens never enter manifests, saved +runs, configuration digests, or logs. ALGAL does not save Cloudflare tokens +or reuse a TypeSafe key. `doctor --clef` checks configuration without an API +call; inference starts only when you attach a Clef executor. ```sh -bun run cli auth jev # vault a TypeSafe Jev key (TYPESAFE_API_KEY) -bun run cli auth jev --status # where the key resolves from (hint only) -bun run cli doctor --jev # live one-question check against systemone +bun run cli auth clef --status +bun run cli doctor --clef +bun run cli run examples/model-router.algal.json --args examples/model-router.args.json --clef ``` +Both CLIs accept `--clef clef-flash`. For embedded images, pass +`--images images.json` with `--clef`; the file contains an array of PNG, +JPEG or WebP data URLs or `{content_type,base64}` objects. Limits are 4 +images, 4 MiB per image and 8 MiB total after base64 decoding into file +bytes, 16 megapixels per image, and 13 MiB for the JSON file and request +body. Native `algal.host.v1` files may configure an `images` array on a Clef +backend, but the entire host file has a separate 1 MiB limit, including +base64 text. Use `--clef --images` for the larger image budget. + +Normal manifest runs use only executor-configured images; a manifest +cannot supply image files or implicitly attach source code. Direct adapter +calls can pass images through `clefAsker.ask(state, questions, signal, +images)`, TypeScript `EffectRequest.images`, or native `clef::prepare`. +These are low-level host interfaces, not manifest fields. Only one image +source is accepted. Remote URLs, invalid base64, mismatched signatures, +truncated containers and exceeded limits fail before credential resolution +or dispatch. The checks inspect container structure and dimensions, not +pixel decoding or PNG checksums; they do not establish that every image +payload is decodable. See [executor configuration](docs/executors.md#cloudflare-clef). +ALGAL rejects oversized state instead of truncating it; the provider's +context limits also apply. Clef requires nonempty +instructions, question IDs matching `[A-Za-z0-9_.-]{1,100}`, at least two +choice options, and 2 to 10 score levels. ALGAL also limits choices to 32. +Responses must match the questions and selected model, with valid probability +mass, confidence, score legends and token counts. + +The explicit `--jev`, `jevAsker`, `jevExecutor` and native `kind:"jev"` +paths remain for legacy integrations. They use their own TypeSafe credential +and never supply Cloudflare credentials. Historical Jev runs replay offline +without contacting either provider. + ## Foundry: select organisms by evidence A foundry evaluates a bounded population against explicit train and validation @@ -943,9 +976,9 @@ algal bench verify bench-report.json --dir .algal A `algal.bench.config.v1` file names case `args`/`expect` pairs and systems whose `executors` map names to `gateway:` (Vercel AI Gateway), -`scripted:`, or `cmd:` specs. The Bun CLI also accepts -`jev[:]` and `recall[:]`; the native CLI accepts `apple` for -the on-device bridge and admits Jev/recall through `algal.host.v1`. Named entries answer `route.preset` / +`clef[:]`, `scripted:`, or `cmd:` specs. The Bun CLI +also accepts `recall[:]`; the native CLI accepts `apple` for the +on-device bridge and attaches recall through `algal.host.v1`. Named entries answer `route.preset` / `route.provider`; the fallback is the first listed entry (in the native CLI, the first name alphabetically). The report records per-case results, work, token usage, per-model effect attribution, and the non-dominated pareto set on diff --git a/cli.ts b/cli.ts index 9ef475f3..7295fe59 100644 --- a/cli.ts +++ b/cli.ts @@ -69,6 +69,14 @@ async function jevSpecExecutor(spec: string): Promise { }); } +async function clefSpecExecutor(spec: string, images?: import("./src/clef").ClefImage[]): Promise { + if (spec !== "clef" && !spec.startsWith("clef:")) return undefined; + const { clefExecutor } = await import("./src/clef"); + const model = spec === "clef" ? "clef" : spec.slice(5); + if (model !== "clef" && model !== "clef-flash") usageError("Clef model must be clef or clef-flash"); + return clefExecutor({ model, ...(images !== undefined ? { images } : {}) }); +} + async function recallSpecExecutor(spec: string, dir: string): Promise { if (spec !== "recall" && !spec.startsWith("recall:")) return undefined; const embedderSpec = spec === "recall" ? "local" : spec.slice(7); @@ -183,7 +191,12 @@ usage: --base-url --model OpenAI-compatible endpoint (HTTPS or loopback HTTP) --credential-env endpoint credential environment variable (optional) --response-format json_schema (default), json_object, or prompt - --jev [model] TypeSafe Jev decision executor — serves + --clef [clef|clef-flash] Cloudflare Clef decision executor (paid, opt-in) + set CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN + (CLOUDFLARE_AUTH_TOKEN is also accepted) + --images JSON array of embedded PNG/JPEG/WebP images; + requires --clef; max 4, 4 MiB each, 8 MiB total + --jev [model] Legacy TypeSafe Jev decision executor — serves decide and classifier cells (never gates: approvals stay host/policy-routed) --recall [embedder] derived-index recall executor; embedder is @@ -379,7 +392,7 @@ usage: returns only declared interface outputs. options mirror algal run: --args, --responses, --executor-cmd, --executor-profile, --gateway-model, - --jev, --recall, + --clef, --images, --jev, --recall, --executors, --modules, --tools, --cache-effects, --dir algal tool-def [--modules ] print an OpenAI/Anthropic tool definition for the @@ -402,11 +415,13 @@ usage: receipts-touching-capability , unevaluated-revisions, largest-work, process-status, kinds + algal auth clef --status check environment-only Cloudflare credentials algal auth jev [--status | --forget | --stdin | --clipboard] vault a provider credential locally — keychain when available, permission-checked file otherwise; never echoes the key - algal doctor [--jev] runtime and provider availability check + algal doctor [--clef] runtime and Cloudflare configuration check (no API call) + algal doctor --jev legacy Jev live connectivity check algal --version | --help Source diagnostics (all commands that load .algal files): @@ -857,7 +872,7 @@ async function resolveExecutors( usageError("--base-url and --model must be supplied together"); if (baseUrl === undefined && (credentialEnv !== undefined || responseFormat !== undefined)) usageError("--credential-env and --response-format require --base-url and --model"); - if (baseUrl !== undefined && ["responses", "executor-cmd", "gateway-model", "jev", "recall"].some(key => flags[key] !== undefined)) + if (baseUrl !== undefined && ["responses", "executor-cmd", "gateway-model", "clef", "jev", "recall"].some(key => flags[key] !== undefined)) usageError("choose one default executor; --base-url cannot be combined with another provider"); if (responseFormat !== undefined && !["json_schema", "json_object", "prompt"].includes(responseFormat)) usageError("--response-format must be json_schema, json_object, or prompt"); @@ -903,6 +918,14 @@ async function resolveExecutors( const { vercelGatewayExecutor } = await import("./src/gateway"); executors.push(vercelGatewayExecutor({ model })); } + if (flags.images !== undefined && flags.clef === undefined) usageError("--images requires --clef"); + if (flags.clef !== undefined) { + if (["responses", "executor-cmd", "gateway-model", "jev", "recall"].some(key => flags[key] !== undefined)) usageError("choose one default executor; --clef cannot be combined with another provider"); + const { validateClefImages, CLEF_IMAGE_LIMITS } = await import("./src/clef"); + const imageFile = artifactFlag(flags, "images"); + const images = imageFile !== undefined ? validateClefImages(await readJsonBounded(imageFile, CLEF_IMAGE_LIMITS.maxBodyBytes, "Clef images")) : undefined; + executors.push((await clefSpecExecutor(typeof flags.clef === "string" ? `clef:${flags.clef}` : "clef", images))!); + } if (flags.jev !== undefined) { executors.push((await jevSpecExecutor(typeof flags.jev === "string" ? `jev:${flags.jev}` : "jev"))!); } @@ -922,6 +945,8 @@ async function resolveExecutors( `executors.${name} must be a shell command string`, ); } + const clef = await clefSpecExecutor(cmd); + if (clef !== undefined) { executors.push(named(name, clef)); continue; } const jev = await jevSpecExecutor(cmd); if (jev !== undefined) { executors.push(named(name, jev)); @@ -2787,6 +2812,8 @@ async function main(): Promise { const { vercelGatewayExecutor } = await import("./src/gateway"); return named(id, vercelGatewayExecutor({ model: spec.slice("gateway:".length) })); } + const clef = await clefSpecExecutor(spec); + if (clef !== undefined) return named(id, clef); const jev = await jevSpecExecutor(spec); if (jev !== undefined) return named(id, jev); const recall = await recallSpecExecutor(spec, dir); @@ -2807,7 +2834,7 @@ async function main(): Promise { } throw new AlgalError( "PARSE_FAILED", - `bench executor "${id}": unknown spec (want gateway:, jev[:], recall[:], scripted:, or cmd:)`, + `bench executor "${id}": unknown spec (want gateway:, clef[:], jev[:], recall[:], scripted:, or cmd:)`, ); }; const systems: BenchSystem[] = []; @@ -3731,7 +3758,8 @@ async function main(): Promise { case "auth": { const provider = positional[0]; - if (provider !== "jev") usageError("algal auth [--status|--forget|--stdin|--clipboard]"); + if (provider !== "jev" && provider !== "clef") usageError("algal auth clef --status or algal auth jev [--status|--forget|--stdin|--clipboard]"); + if (provider === "clef" && flags.status === undefined) usageError("Clef credentials are environment-only; set CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN (CLOUDFLARE_AUTH_TOKEN is also accepted)"); const { credentialStatus, forgetCredential, providerSpec, storeCredential } = await import("./src/credentials"); if (flags.status !== undefined) { out((await credentialStatus(provider)) as unknown as JsonObject); @@ -3769,6 +3797,16 @@ async function main(): Promise { } case "doctor": { + if (flags.clef !== undefined) { + const { credentialStatus } = await import("./src/credentials"); + const { clefEndpoint } = await import("./src/clef"); + const credential = await credentialStatus("clef"); + let accountConfigured = false; + try { clefEndpoint(process.env.CLOUDFLARE_ACCOUNT_ID ?? ""); accountConfigured = true; } catch { accountConfigured = false; } + const configured = accountConfigured && credential.configured; + out({ provider: "clef", configured, accountConfigured, credential: credential as unknown as JsonValue, liveChecked: false }); + return configured ? 0 : 1; + } if (flags.jev !== undefined) { const { credentialResolver, credentialStatus, providerSpec } = await import("./src/credentials"); const { jevAsker } = await import("./src/jev"); @@ -3809,12 +3847,12 @@ async function main(): Promise { } } if (humanReport(process.argv.slice(2))) { - const jevEnv = process.env.TYPESAFE_API_KEY !== undefined && process.env.TYPESAFE_API_KEY !== ""; + const clefEnv = Boolean(process.env.CLOUDFLARE_API_TOKEN || process.env.CLOUDFLARE_AUTH_TOKEN); printDoctor([ ["ok", `ALGAL ${PACKAGE_VERSION} is ready (Bun reference runtime on ${platformName()}).`], - jevEnv - ? ["ok", "Jev key found in TYPESAFE_API_KEY. Test it: algal doctor --jev"] - : ["skip", "Jev decisions: not checked. Test a saved key: algal doctor --jev"], + clefEnv + ? ["ok", "Cloudflare token found. Check configuration: algal doctor --clef"] + : ["skip", "Clef decisions: not configured. Set CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN."], ], undefined, "algal examples"); return 0; } diff --git a/crates/algal/examples/clef_parity.rs b/crates/algal/examples/clef_parity.rs new file mode 100644 index 00000000..f462349c --- /dev/null +++ b/crates/algal/examples/clef_parity.rs @@ -0,0 +1,92 @@ +use algal::{ + Result, + canonical::{canonical, digest}, + clef, + effects::{Backend, Host}, +}; +use serde_json::{Value, json}; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<()> { + let fixture: Value = serde_json::from_str(include_str!("../../../scripts/fixtures/clef.json"))?; + let account = fixture["accountId"].as_str().unwrap(); + let questions = &fixture["questions"]; + let mut cases = Vec::new(); + for item in fixture["cases"].as_array().unwrap() { + let model = item["model"].as_str().unwrap(); + let mut envelope = fixture["response"].clone(); + for (path, value) in item["patch"].as_object().unwrap() { + if let Some(target) = envelope.pointer_mut(path) { + *target = value.clone(); + } else { + let (parent, key) = path.rsplit_once('/').unwrap(); + envelope + .pointer_mut(parent) + .unwrap() + .as_object_mut() + .unwrap() + .insert(key.to_owned(), value.clone()); + } + } + let images = Value::Array( + item["imageIndices"] + .as_array() + .unwrap() + .iter() + .map(|index| fixture["images"][index.as_u64().unwrap() as usize].clone()) + .collect(), + ); + let configured = (item["source"] == "configured").then_some(&images); + let mut request = json!({"contract":"algal.effect.v1","cellId":"probe","kind":"decide","prompt":"Evaluate the observation.","context":{"inputs":{"text":"東京 observation"},"turn":0},"output":{"kind":"json","schema":{}},"budget":{"maxContextBytes":4096,"maxOutputBytes":4096},"questions":questions}); + if item["source"] == "request" { + request["images"] = images.clone(); + } + let (body, _) = clef::prepare(model, &request, configured)?; + let backend = Backend::Clef { + model: model.to_owned(), + account_id: account.to_owned(), + images: configured.cloned(), + }; + backend.validate()?; + let configuration = serde_json::to_value(&backend)?; + let configuration_digest = digest(&configuration)?; + let mut receipt = json!({"requestDigest":digest(&request)?,"executor":clef::executor_id(model),"configurationDigest":configuration_digest}); + let parsed = match clef::response(&envelope, model, questions) { + Ok((output, metadata)) => { + assert!( + item.get("error").is_none(), + "{} accepted invalid envelope", + item["name"] + ); + receipt["output"] = output.clone(); + receipt["usage"] = metadata["usage"].clone(); + json!({"answers":output["answers"],"usage":metadata["usage"]}) + } + Err(error) => { + assert_eq!( + item["error"], error.code, + "{} unexpected rejection", + item["name"] + ); + receipt["error"] = json!({"code":error.code,"message":error.message}); + json!({"error":receipt["error"]}) + } + }; + let replay = Host::replay(&json!([receipt.clone()]))? + .effect(&request, 1000, None) + .await?; + cases.push(json!({"name":item["name"],"request":request,"body":body,"parsed":parsed,"configuration":configuration,"configurationDigest":configuration_digest,"cacheIdentity":backend.cache_identity("clef")?,"imageDigest":digest(&images)?,"receipt":receipt,"replay":replay})); + } + let invalid_images: Vec = fixture["invalidImages"] + .as_array() + .unwrap() + .iter() + .map(|images| clef::check_images(images).is_err()) + .collect(); + assert!(invalid_images.iter().all(|value| *value)); + println!( + "{}", + canonical(&json!({"cases":cases,"invalidImages":invalid_images}))? + ); + Ok(()) +} diff --git a/crates/algal/src/bench.rs b/crates/algal/src/bench.rs index c1843adc..c61b5767 100644 --- a/crates/algal/src/bench.rs +++ b/crates/algal/src/bench.rs @@ -291,6 +291,15 @@ pub fn load_config(path: &Path, apple_bridge: Option<&Path>) -> Result) -> Result, scripted:, cmd:, or apple)" + "bench executor \"{name}\": unknown spec (want gateway:, clef[:], scripted:, cmd:, or apple)" ))); }; backend.validate()?; diff --git a/crates/algal/src/clef.rs b/crates/algal/src/clef.rs new file mode 100644 index 00000000..918d8f1c --- /dev/null +++ b/crates/algal/src/clef.rs @@ -0,0 +1,706 @@ +use crate::{ + Error, Result, + canonical::{canonical, digest}, + contract::{keys, object}, +}; +use serde_json::{Map, Value, json}; +use std::time::Duration; + +pub const DEFAULT_MODEL: &str = "clef"; +pub const MAX_BODY: usize = 13 * 1024 * 1024; +const MAX_IMAGE: usize = 4 * 1024 * 1024; +const MAX_TOTAL: usize = 8 * 1024 * 1024; +const MAX_RESPONSE: usize = 1_048_576; + +pub fn endpoint(account: &str, model: &str) -> Result { + if account.len() != 32 + || !account + .bytes() + .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) + || !matches!(model, "clef" | "clef-flash") + { + return Err(Error::invalid("Invalid Cloudflare Clef account or model")); + } + Ok(format!( + "https://api.cloudflare.com/client/v4/accounts/{account}/ai/run/@cf/cloudflare/{model}" + )) +} + +fn image_error() -> Error { + Error::invalid("Invalid Clef image: use embedded PNG, JPEG or WebP within the image limits") +} + +fn decode(value: &str) -> Result> { + if value.len() < 4 || value.len() > MAX_IMAGE.div_ceil(3) * 4 || !value.len().is_multiple_of(4) + { + return Err(image_error()); + } + let mut out = Vec::with_capacity(value.len() / 4 * 3); + for (index, chunk) in value.as_bytes().chunks_exact(4).enumerate() { + let last = index == value.len() / 4 - 1; + let padding = if chunk[2] == b'=' && chunk[3] == b'=' { + 2 + } else if chunk[3] == b'=' { + 1 + } else { + 0 + }; + if padding != 0 && !last { + return Err(image_error()); + } + let mut n = 0u32; + for (i, c) in chunk.iter().enumerate() { + let v = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' => 62, + b'/' => 63, + b'=' if i >= 4 - padding => 0, + _ => return Err(image_error()), + }; + n = (n << 6) | u32::from(v); + } + if (padding == 2 && n & 0xffff != 0) || (padding == 1 && n & 0xff != 0) { + return Err(image_error()); + } + out.push((n >> 16) as u8); + if padding < 2 { + out.push((n >> 8) as u8); + } + if padding == 0 { + out.push(n as u8); + } + } + if out.len() > MAX_IMAGE { + return Err(image_error()); + } + Ok(out) +} + +fn be16(b: &[u8], i: usize) -> u32 { + u32::from(u16::from_be_bytes([b[i], b[i + 1]])) +} +fn be32(b: &[u8], i: usize) -> u32 { + u32::from_be_bytes([b[i], b[i + 1], b[i + 2], b[i + 3]]) +} +fn le32(b: &[u8], i: usize) -> u32 { + u32::from_le_bytes([b[i], b[i + 1], b[i + 2], b[i + 3]]) +} +fn le24(b: &[u8], i: usize) -> u32 { + u32::from(b[i]) | u32::from(b[i + 1]) << 8 | u32::from(b[i + 2]) << 16 +} + +fn dimensions(b: &[u8], mime: &str) -> Result<(u32, u32)> { + if mime == "image/png" && b.len() >= 33 && b[..8] == [137, 80, 78, 71, 13, 10, 26, 10] { + let mut offset = 8; + let mut data = 0; + let mut size = None; + while offset + 12 <= b.len() { + let length = be32(b, offset) as usize; + let end = offset + 12 + length; + if end > b.len() { + return Err(image_error()); + } + let chunk = &b[offset + 4..offset + 8]; + if offset == 8 && chunk != b"IHDR" { + return Err(image_error()); + } + if chunk == b"IHDR" { + if size.is_some() + || length != 13 + || b[offset + 18] != 0 + || b[offset + 19] != 0 + || b[offset + 20] > 1 + { + return Err(image_error()); + } + let depths: &[u8] = match b[offset + 17] { + 0 => &[1, 2, 4, 8, 16], + 2 | 4 | 6 => &[8, 16], + 3 => &[1, 2, 4, 8], + _ => &[], + }; + if !depths.contains(&b[offset + 16]) { + return Err(image_error()); + } + size = Some((be32(b, offset + 8), be32(b, offset + 12))); + } + if chunk == b"IDAT" { + data += length; + } + if chunk == b"IEND" { + return if length == 0 && end == b.len() && data > 0 { + size.ok_or_else(image_error) + } else { + Err(image_error()) + }; + } + offset = end; + } + } + if mime == "image/jpeg" && b.len() >= 4 && b[..2] == [255, 216] { + let mut offset = 2; + let mut scans = 0; + let mut size = None; + while offset + 2 <= b.len() { + if b[offset] != 255 { + return Err(image_error()); + } + offset += 1; + while offset < b.len() && b[offset] == 255 { + offset += 1; + } + let Some(&marker) = b.get(offset) else { + return Err(image_error()); + }; + offset += 1; + if marker == 217 { + return if offset == b.len() && scans > 0 { + size.ok_or_else(image_error) + } else { + Err(image_error()) + }; + } + if marker == 1 { + continue; + } + if marker == 0 || marker == 216 || (208..=215).contains(&marker) || offset + 2 > b.len() + { + return Err(image_error()); + } + let length = be16(b, offset) as usize; + if length < 2 || offset + length > b.len() { + return Err(image_error()); + } + if [ + 192, 193, 194, 195, 197, 198, 199, 201, 202, 203, 205, 206, 207, + ] + .contains(&marker) + { + if size.is_some() || length < 11 || length != 8 + 3 * usize::from(b[offset + 7]) { + return Err(image_error()); + } + size = Some((be16(b, offset + 5), be16(b, offset + 3))); + } + if marker == 218 { + if size.is_none() || length < 8 || length != 6 + 2 * usize::from(b[offset + 2]) { + return Err(image_error()); + } + offset += length; + let start = offset; + while offset < b.len() { + if b[offset] != 255 { + offset += 1; + continue; + } + if b.get(offset + 1) + .is_some_and(|v| *v == 0 || (208..=215).contains(v)) + { + offset += 2; + continue; + } + break; + } + if offset == start { + return Err(image_error()); + } + scans += 1; + } else { + offset += length; + } + } + } + if mime == "image/webp" + && b.len() >= 25 + && &b[..4] == b"RIFF" + && u64::from(le32(b, 4)) + 8 == b.len() as u64 + && &b[8..12] == b"WEBP" + { + let mut offset = 12; + let mut size = None; + let mut canvas = None; + while offset + 8 <= b.len() { + let chunk = &b[offset..offset + 4]; + let length = le32(b, offset + 4) as usize; + let start = offset + 8; + let end = start + length; + if end + length % 2 > b.len() { + return Err(image_error()); + } + if chunk == b"VP8X" { + if offset != 12 || length != 10 { + return Err(image_error()); + } + canvas = Some((le24(b, start + 4) + 1, le24(b, start + 7) + 1)); + } + if chunk == b"VP8L" || chunk == b"VP8 " { + if size.is_some() { + return Err(image_error()); + } + if chunk == b"VP8L" && length > 5 && b[start] == 47 { + let bits = le32(b, start + 1); + size = Some(((bits & 0x3fff) + 1, ((bits >> 14) & 0x3fff) + 1)); + } else if chunk == b"VP8 " && length > 10 && b[start + 3..start + 6] == [157, 1, 42] + { + size = Some(( + u32::from(u16::from_le_bytes([b[start + 6], b[start + 7]])) & 0x3fff, + u32::from(u16::from_le_bytes([b[start + 8], b[start + 9]])) & 0x3fff, + )); + } else { + return Err(image_error()); + } + } + offset = end + length % 2; + } + if offset == b.len() && size.is_some() && (canvas.is_none() || canvas == size) { + return size.ok_or_else(image_error); + } + } + Err(image_error()) +} + +pub fn check_images(images: &Value) -> Result<()> { + let list = images + .as_array() + .filter(|v| v.len() <= 4) + .ok_or_else(image_error)?; + let mut total = 0usize; + for image in list { + let (mime, encoded) = if let Some(url) = image.as_str() { + if url.len() > MAX_IMAGE.div_ceil(3) * 4 + 32 { + return Err(image_error()); + } + let (prefix, encoded) = url.split_once(',').ok_or_else(image_error)?; + let lower = prefix.to_ascii_lowercase(); + let mime = match lower.as_str() { + "data:image/png;base64" => "image/png", + "data:image/jpeg;base64" => "image/jpeg", + "data:image/webp;base64" => "image/webp", + _ => return Err(image_error()), + }; + (mime, encoded) + } else { + keys(image, &["content_type", "base64"])?; + let mime = image["content_type"].as_str().ok_or_else(image_error)?; + if !matches!(mime, "image/png" | "image/jpeg" | "image/webp") { + return Err(image_error()); + } + (mime, image["base64"].as_str().ok_or_else(image_error)?) + }; + let bytes = decode(encoded)?; + total += bytes.len(); + if total > MAX_TOTAL { + return Err(image_error()); + } + let (w, h) = dimensions(&bytes, mime)?; + if w == 0 || h == 0 || u64::from(w) * u64::from(h) > 16_000_000 { + return Err(image_error()); + } + } + Ok(()) +} + +pub fn request( + model: &str, + state: &Value, + questions: &Value, + images: Option<&Value>, +) -> Result { + if !matches!(model, "clef" | "clef-flash") { + return Err(Error::invalid("Invalid Clef model")); + } + crate::decisions::check_questions(questions, "Clef questions")?; + for (id, q) in object(questions)? { + if id.is_empty() + || id.len() > 100 + || !id + .bytes() + .all(|c| c.is_ascii_alphanumeric() || b"_.-".contains(&c)) + || q["instructions"] + .as_str() + .is_none_or(|v| v.trim().is_empty()) + { + return Err(Error::invalid("Invalid Clef question ID or instructions")); + } + let n = q["criteria"] + .as_object() + .map_or_else(|| q["criteria"].as_array().map_or(0, Vec::len), Map::len); + if (q["type"] == "choice" && n < 2) || (q["type"] == "score" && !(2..=10).contains(&n)) { + return Err(Error::invalid( + "Clef choice requires at least two options; score requires 2..10 levels", + )); + } + } + if canonical(state)?.len() > 262_144 { + return Err(Error::limit("Clef state exceeds 262144 bytes")); + } + let mut body = json!({"model":model,"state":state,"questions":questions}); + if let Some(images) = images { + check_images(images)?; + body["images"] = images.clone(); + } + if canonical(&body)?.len() > MAX_BODY { + return Err(Error::limit("Clef request exceeds 13 MiB")); + } + Ok(body) +} + +fn probability(v: &Value) -> Result { + v.as_f64() + .filter(|n| n.is_finite() && (0.0..=1.0).contains(n)) + .ok_or_else(|| Error::invalid("probability")) +} + +fn score_endpoint(bounds: &[(f64, f64)], lower_total: f64, maximum: bool) -> f64 { + let mut mean: f64 = bounds.iter().enumerate().map(|(i, b)| i as f64 * b.0).sum(); + let mut remaining = (1.0 - lower_total).max(0.0); + for position in 0..bounds.len() { + let i = if maximum { + bounds.len() - 1 - position + } else { + position + }; + let added = remaining.min(bounds[i].1 - bounds[i].0); + mean += i as f64 * added; + remaining -= added; + if remaining <= 0.0 { + break; + } + } + mean +} + +pub fn response(value: &Value, model: &str, questions: &Value) -> Result<(Value, Value)> { + parse_response(value, model, questions).map_err(|_| { + Error::new( + "EFFECT_UNPARSEABLE", + "Invalid Cloudflare Clef response; body withheld", + ) + }) +} + +fn parse_response(value: &Value, model: &str, questions: &Value) -> Result<(Value, Value)> { + keys(value, &["success", "result", "errors", "messages"])?; + if value["success"] != true + || value + .get("errors") + .is_some_and(|v| v.as_array().is_none_or(|v| !v.is_empty())) + || value + .get("messages") + .is_some_and(|v| v.as_array().is_none_or(|v| v.len() > 64)) + { + return Err(Error::invalid("envelope")); + } + let raw = &value["result"]; + keys(raw, &["model", "answers", "usage"])?; + if raw["model"] != model { + return Err(Error::invalid("model")); + } + let declared = object(&raw["answers"])?; + let qs = object(questions)?; + if declared.len() != qs.len() || declared.keys().any(|id| !qs.contains_key(id)) { + return Err(Error::invalid("answer IDs")); + } + let mut answers = Map::new(); + for (id, q) in qs { + let a = &declared[id]; + if a["type"] != q["type"] { + return Err(Error::invalid("answer type")); + } + if q["type"] == "noul" { + keys(a, &["type", "noul"])?; + probability(&a["noul"])?; + answers.insert(id.clone(), a.clone()); + continue; + } + if q["type"] == "choice" { + keys(a, &["type", "choice", "confidence", "probabilities"])?; + } else { + keys( + a, + &["type", "score", "legend", "confidence", "probabilities"], + )?; + } + probability(&a["confidence"])?; + let expected: Vec = if q["type"] == "choice" { + object(&q["criteria"])?.keys().cloned().collect() + } else { + q["criteria"] + .as_array() + .ok_or_else(|| Error::invalid("criteria"))? + .iter() + .enumerate() + .map(|(i, _)| i.to_string()) + .collect() + }; + let p = object(&a["probabilities"])?; + if p.len() != expected.len() || expected.iter().any(|k| !p.contains_key(k)) { + return Err(Error::invalid("probability IDs")); + } + let mut sum = 0.0; + let mut bounds = Vec::with_capacity(expected.len()); + for k in &expected { + let value = probability(&p[k])?; + sum += value; + bounds.push(((value - 0.0005).max(0.0), (value + 0.0005).min(1.0))); + } + let lower_total: f64 = bounds.iter().map(|b| b.0).sum(); + let upper_total: f64 = bounds.iter().map(|b| b.1).sum(); + if sum <= 0.0 || lower_total > 1.0 + 1e-9 || upper_total < 1.0 - 1e-9 { + return Err(Error::invalid("probability mass")); + } + if q["type"] == "choice" { + let choice = a["choice"] + .as_str() + .ok_or_else(|| Error::invalid("choice"))?; + if !p.contains_key(choice) + || p.values().any(|v| { + v.as_f64().unwrap_or(2.0) > p[choice].as_f64().unwrap_or(-1.0) + 0.000001 + }) + { + return Err(Error::invalid("choice")); + } + } else { + let levels = q["criteria"] + .as_array() + .ok_or_else(|| Error::invalid("criteria"))?; + let legend: Map = levels + .iter() + .enumerate() + .map(|(i, v)| (i.to_string(), v.clone())) + .collect(); + let score = a["score"] + .as_f64() + .filter(|n| n.is_finite()) + .ok_or_else(|| Error::invalid("score"))?; + if a["legend"] != Value::Object(legend) + || score < 0.0 + || score > (levels.len() - 1) as f64 + || score + 0.0005 < score_endpoint(&bounds, lower_total, false) - 1e-9 + || score - 0.0005 > score_endpoint(&bounds, lower_total, true) + 1e-9 + { + return Err(Error::invalid("score or legend")); + } + } + answers.insert(id.clone(), a.clone()); + } + keys(&raw["usage"], &["input_tokens", "output_tokens"])?; + let mut usage = json!({"model":model}); + for (wire, ours) in [("input_tokens", "tokensIn"), ("output_tokens", "tokensOut")] { + let n = raw["usage"][wire] + .as_u64() + .filter(|n| *n <= 1_000_000_000) + .ok_or_else(|| Error::invalid("usage"))?; + usage[ours] = json!(n); + } + Ok((json!({"answers":answers}), json!({"usage":usage}))) +} + +pub fn prepare( + model: &str, + request_value: &Value, + images: Option<&Value>, +) -> Result<(Value, bool)> { + let single = match request_value["kind"].as_str() { + Some("classifier") => true, + Some("decide") => false, + Some("gate") => { + return Err(Error::new( + "EFFECT_UNBOUND", + "approval gates require a host approval executor, not a model", + )); + } + _ => { + return Err(Error::new( + "EFFECT_UNPARSEABLE", + "Clef serves typed decisions only", + )); + } + }; + let questions = if single { + crate::decisions::choice_question(request_value)? + } else { + request_value["questions"].clone() + }; + let state = json!({"prompt":request_value["prompt"],"context":request_value["context"]}); + let supplied = request_value.get("images"); + if supplied.is_some() && images.is_some() { + return Err(Error::invalid("images must have one source")); + } + Ok(( + request(model, &state, &questions, supplied.or(images))?, + single, + )) +} + +pub async fn ask( + account: &str, + credential: &str, + body: &Value, + deadline_ms: u64, +) -> Result<(Value, Value)> { + let model = body["model"] + .as_str() + .ok_or_else(|| Error::invalid("model"))?; + let body = request( + model, + &body["state"], + &body["questions"], + body.get("images"), + )?; + let url = endpoint(account, model)?; + crate::credentials::check_shape(credential, "Cloudflare credential")?; + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_millis(deadline_ms.clamp(1, 120_000))) + .build() + .map_err(|_| Error::new("EFFECT_FAILED", "HTTP client initialization failed"))?; + let mut res = client + .post(url) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(canonical(&body)?) + .bearer_auth(credential) + .send() + .await + .map_err(|_| { + Error::new( + "EFFECT_FAILED", + "Clef request failed or timed out; external completion uncertain", + ) + .uncertain() + })?; + if !res.status().is_success() { + return Err(Error::new( + "EFFECT_FAILED", + format!( + "Clef returned HTTP {}; redirects and response body withheld", + res.status().as_u16() + ), + )); + } + if res + .content_length() + .is_some_and(|n| n > MAX_RESPONSE as u64) + { + return Err(Error::limit("Clef response bytes").uncertain()); + } + let mut bytes = Vec::new(); + while let Some(chunk) = res.chunk().await.map_err(|_| { + Error::new( + "EFFECT_FAILED", + "Clef response read failed; external completion uncertain", + ) + .uncertain() + })? { + if bytes.len().saturating_add(chunk.len()) > MAX_RESPONSE { + return Err(Error::limit("Clef response bytes").uncertain()); + } + bytes.extend_from_slice(&chunk); + } + let raw: Value = serde_json::from_slice(&bytes).map_err(|_| { + Error::new( + "EFFECT_UNPARSEABLE", + "Invalid Cloudflare Clef response; body withheld", + ) + })?; + response(&raw, model, &body["questions"]) +} + +pub fn executor_id(model: &str) -> String { + if model == DEFAULT_MODEL { + "clef".into() + } else { + format!("clef:{model}") + } +} +pub fn cache_identity(account: &str, model: &str, images: Option<&Value>) -> Result { + digest( + &json!({"provider":"cloudflare","baseUrl":endpoint(account,model)?,"model":model,"images":images}), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recorded_synthetic_scores_allow_rounding_but_reject_inconsistent_scores() { + let questions = json!({"severity":{"type":"score","instructions":"How severe is the customer impact?","criteria":["No impact","Minor","Major","Critical"]}}); + for (model, score, confidence, probabilities) in [ + ( + "clef", + 2.931, + 0.8612, + json!({"0":0.0047,"1":0.0051,"2":0.0448,"3":0.9454}), + ), + ( + "clef-flash", + 2.7378, + 0.5316, + json!({"0":0.0149,"1":0.0157,"2":0.186,"3":0.7834}), + ), + ] { + let mut envelope = json!({"success":true,"result":{"model":model,"usage":{"input_tokens":319,"output_tokens":0},"answers":{"severity":{"type":"score","score":score,"confidence":confidence,"probabilities":probabilities,"legend":{"0":"No impact","1":"Minor","2":"Major","3":"Critical"}}}}}); + let (output, _) = response(&envelope, model, &questions).unwrap(); + assert_eq!(output["answers"]["severity"]["score"], json!(score)); + envelope["result"]["answers"]["severity"]["score"] = json!(score - 0.01); + assert!(response(&envelope, model, &questions).is_err()); + } + } + + #[test] + fn probability_rounding_requires_a_feasible_normalized_distribution() { + let questions = json!({"label":{"type":"choice","instructions":"Choose","criteria":{"a":null,"b":null,"c":null}}}); + for (p, valid) in [(0.333, true), (0.0, false), (0.332, false), (0.334, false)] { + let envelope = json!({"success":true,"result":{"model":"clef","usage":{"input_tokens":12,"output_tokens":0},"answers":{"label":{"type":"choice","choice":"a","confidence":0.2,"probabilities":{"a":p,"b":p,"c":p}}}}}); + assert_eq!(response(&envelope, "clef", &questions).is_ok(), valid); + } + } + + #[test] + fn image_size_padding_and_pixel_limits_are_checked() { + for value in ["====", "AA=A", "AB==", "AAAA=", "private"] { + assert!(decode(value).is_err()); + } + assert!(decode(&"A".repeat((MAX_IMAGE + 1).div_ceil(3) * 4)).is_err()); + let mut png = [0u8; 33]; + png[..8].copy_from_slice(&[137, 80, 78, 71, 13, 10, 26, 10]); + png[8..12].copy_from_slice(&13u32.to_be_bytes()); + png[12..16].copy_from_slice(b"IHDR"); + png[16..20].copy_from_slice(&4001u32.to_be_bytes()); + png[20..24].copy_from_slice(&4001u32.to_be_bytes()); + assert!(dimensions(&png, "image/png").is_err()); + assert!(dimensions(&png, "image/jpeg").is_err()); + assert!(check_images(&json!([{ "content_type":"image/png","base64":"iVBORw0KGgoAAAANSUhEUgAAD6EAAA+hCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+j1ioAAAAASUVORK5CYII=" }])).is_err()); + let large = format!( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwC{}", + "A".repeat(4 * 1024 * 1024 - 44) + ); + assert!( + check_images(&json!([{ "content_type":"image/png","base64":large.clone() }])).is_err() + ); + assert!(check_images(&json!([{ "content_type":"image/png","base64":large.clone() },{ "content_type":"image/png","base64":large.clone() },{ "content_type":"image/png","base64":large }])).is_err()); + } + + #[test] + fn jpeg_and_webp_require_image_payload_and_complete_containers() { + let header = [255, 216, 255, 192, 0, 8, 8, 0, 1, 0, 1, 1, 255, 217]; + assert!(dimensions(&header, "image/jpeg").is_err()); + let jpeg = [ + 255, 216, 255, 192, 0, 11, 8, 0, 1, 0, 1, 1, 1, 17, 0, 255, 218, 0, 8, 1, 1, 0, 0, 63, + 0, 1, 255, 217, + ]; + assert_eq!(dimensions(&jpeg, "image/jpeg").unwrap(), (1, 1)); + assert!(dimensions(&jpeg[..jpeg.len() - 1], "image/jpeg").is_err()); + let mut header = [0u8; 30]; + header[..4].copy_from_slice(b"RIFF"); + header[4..8].copy_from_slice(&22u32.to_le_bytes()); + header[8..16].copy_from_slice(b"WEBPVP8X"); + header[16..20].copy_from_slice(&10u32.to_le_bytes()); + assert!(dimensions(&header, "image/webp").is_err()); + let webp = decode("UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AAAAAA").unwrap(); + assert_eq!(dimensions(&webp, "image/webp").unwrap(), (1, 1)); + assert!(dimensions(&webp[..webp.len() - 1], "image/webp").is_err()); + } +} diff --git a/crates/algal/src/credentials.rs b/crates/algal/src/credentials.rs index 82ccee3f..27f1a8ad 100644 --- a/crates/algal/src/credentials.rs +++ b/crates/algal/src/credentials.rs @@ -24,6 +24,11 @@ pub struct ProviderSpec { pub fn spec(provider: &str) -> Result { match provider { + "clef" => Ok(ProviderSpec { + env: "CLOUDFLARE_API_TOKEN", + service: "algal.clef", + account: "cloudflare", + }), "jev" => Ok(ProviderSpec { env: "TYPESAFE_API_KEY", service: "algal.jev", @@ -507,6 +512,15 @@ pub fn resolve(provider: &str, explicit: Option<&str>) -> Result) -> Result Result<(&'static str, String)> { + if provider == "clef" { + return Err(Error::invalid( + "Clef credentials are environment-only; set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID", + )); + } check_shape(key, &format!("{provider} credential"))?; let spec = spec(provider)?; if vault_detail().is_some() { @@ -538,6 +557,11 @@ pub fn store(provider: &str, key: &str) -> Result<(&'static str, String)> { /// Remove the credential from every local store. Returns the sources that /// yielded a removal. pub fn forget(provider: &str) -> Result> { + if provider == "clef" { + return Err(Error::invalid( + "Clef credentials are environment-only; remove the token from your environment", + )); + } let spec = spec(provider)?; let mut removed = Vec::new(); if vault_forget(provider, &spec)? { diff --git a/crates/algal/src/decisions.rs b/crates/algal/src/decisions.rs index 4cf84b41..4ae40283 100644 --- a/crates/algal/src/decisions.rs +++ b/crates/algal/src/decisions.rs @@ -18,7 +18,7 @@ pub const DEFAULT_MODEL: &str = "jev-latest"; pub const CREDENTIAL_ENV: &str = "TYPESAFE_API_KEY"; const MAX_QUESTIONS: usize = 64; -const MAX_NAME: usize = 64; +const MAX_NAME: usize = 100; const MAX_INSTRUCTIONS: usize = 4096; const MAX_CRITERIA: usize = 32; const MAX_CRITERION: usize = 512; @@ -358,6 +358,11 @@ pub async fn serve( request: &Value, deadline_ms: u64, ) -> Result<(Value, Value)> { + if request.get("images").is_some() { + return Err(Error::invalid( + "Legacy Jev does not support embedded images; use Clef", + )); + } let kind = request["kind"].as_str().unwrap_or(""); let (questions, single) = match kind { "decide" => { diff --git a/crates/algal/src/effects.rs b/crates/algal/src/effects.rs index 91ad2dcc..76cc7d8a 100644 --- a/crates/algal/src/effects.rs +++ b/crates/algal/src/effects.rs @@ -33,6 +33,13 @@ fn jev_model_default() -> String { crate::decisions::DEFAULT_MODEL.to_owned() } +fn clef_model_default() -> String { + crate::clef::DEFAULT_MODEL.to_owned() +} +fn clef_account_default() -> String { + std::env::var("CLOUDFLARE_ACCOUNT_ID").unwrap_or_default() +} + fn recall_embedder_default() -> String { "local".to_owned() } @@ -66,6 +73,14 @@ pub enum Backend { #[serde(default)] credential_env: Option, }, + Clef { + #[serde(default = "clef_model_default")] + model: String, + #[serde(default = "clef_account_default")] + account_id: String, + #[serde(default)] + images: Option, + }, Recall { dir: PathBuf, #[serde(default = "recall_embedder_default")] @@ -118,7 +133,7 @@ impl Backend { Self::Gateway { .. } | Self::Openai { .. } | Self::Apple { .. } => { matches!(kind, "agent" | "classifier") } - Self::Jev { .. } => matches!(kind, "classifier" | "decide"), + Self::Jev { .. } | Self::Clef { .. } => matches!(kind, "classifier" | "decide"), Self::Recall { .. } => kind == "recall", Self::Acp { .. } | Self::Xcb { .. } => kind == "agent", } @@ -150,6 +165,11 @@ impl Backend { Self::Scripted { responses } => { digest(&json!({"kind":"scripted","id":id,"responses":responses})) } + Self::Clef { + model, + account_id, + images, + } => crate::clef::cache_identity(account_id, model, images.as_ref()), _ => Ok(id.to_owned()), } } @@ -173,6 +193,16 @@ impl Backend { check_env(env)?; } } + Self::Clef { + model, + account_id, + images, + } => { + crate::clef::endpoint(account_id, model)?; + if let Some(images) = images { + crate::clef::check_images(images)?; + } + } Self::Recall { dir, embedder } => { if dir.as_os_str().is_empty() || dir.to_string_lossy().len() > 8_192 { return Err(Error::invalid("recall index directory")); @@ -1103,6 +1133,29 @@ impl Host { meta["executor"] = json!(crate::decisions::executor_id(&model)); Ok((out, meta)) } + Backend::Clef { + model, + account_id, + images, + } => { + let (body, single) = crate::clef::prepare(model, request, images.as_ref())?; + let credential = crate::credentials::resolve("clef", None)? + .ok_or_else(|| { + Error::new( + "EFFECT_UNBOUND", + "Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID for Clef", + ) + })? + .0; + let (out, mut meta) = + crate::clef::ask(account_id, &credential, &body, deadline).await?; + meta["executor"] = json!(crate::clef::executor_id(model)); + if single { + Ok((out["answers"]["answer"]["choice"].clone(), meta)) + } else { + Ok((out, meta)) + } + } Backend::Recall { dir, embedder } => { if request["kind"] != "recall" || !request["recall"].is_object() { return Err(Error::new( diff --git a/crates/algal/src/lib.rs b/crates/algal/src/lib.rs index d3ffa95e..10d06a84 100644 --- a/crates/algal/src/lib.rs +++ b/crates/algal/src/lib.rs @@ -26,6 +26,7 @@ pub mod build_info; pub mod canonical; pub mod capabilities; pub mod civilization; +pub mod clef; pub mod context; pub mod context_history; pub mod context_history_access; diff --git a/crates/algal/src/main.rs b/crates/algal/src/main.rs index b38fa68f..5074474a 100644 --- a/crates/algal/src/main.rs +++ b/crates/algal/src/main.rs @@ -111,8 +111,16 @@ struct Execution { #[arg(long)] gateway_model: Option, /// TypeSafe Jev decision executor; bare `--jev` uses `jev-latest`. - #[arg(long, num_args = 0..=1, default_missing_value = "jev-latest")] + #[arg(long, num_args = 0..=1, default_missing_value = "jev-latest", help = "Legacy TypeSafe Jev executor; use --clef for current Cloudflare decisions")] jev: Option, + #[arg(long, num_args = 0..=1, default_missing_value = "clef", help = "Cloudflare Clef decision executor (clef or clef-flash; paid, opt-in)", long_help = "Cloudflare Clef decision executor (clef or clef-flash; paid, opt-in). Set CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN; CLOUDFLARE_AUTH_TOKEN is a token alias.")] + clef: Option, + #[arg( + long, + requires = "clef", + help = "JSON array of embedded PNG/JPEG/WebP images; max 4, 4 MiB each, 8 MiB total" + )] + images: Option, /// Derived-index recall executor; embedder is `local` or `gateway[:]`. #[arg(long, num_args = 0..=1, default_missing_value = "local")] recall: Option, @@ -448,6 +456,9 @@ enum Commands { /// Vault a provider credential locally; never echoes the key. Auth { /// Credential provider (`jev`). + #[arg( + help = "Credential provider: clef (environment-only status) or jev (legacy local vault)" + )] provider: String, /// Report the credential's redacted status and exit. #[arg(long)] @@ -470,6 +481,8 @@ enum Commands { /// TypeSafe Jev availability: credential status plus a live probe. #[arg(long)] jev: bool, + #[arg(long, help = "Check Cloudflare Clef configuration without an API call")] + clef: bool, /// Print the machine-readable report (the default when not at a terminal). #[arg(long)] json: bool, @@ -1161,6 +1174,7 @@ fn host(options: &Execution, dir: &Path) -> Result { + usize::from(options.executor_cmd.is_some()) + usize::from(options.gateway_model.is_some()) + usize::from(options.jev.is_some()) + + usize::from(options.clef.is_some()) + usize::from(options.recall.is_some()) + usize::from(options.base_url.is_some()) + usize::from(options.apple) @@ -1227,6 +1241,16 @@ fn host(options: &Execution, dir: &Path) -> Result { Some(Backend::Gateway { model: model.clone(), }) + } else if let Some(model) = &options.clef { + Some(Backend::Clef { + model: model.clone(), + account_id: std::env::var("CLOUDFLARE_ACCOUNT_ID").unwrap_or_default(), + images: options + .images + .as_ref() + .map(|path| load(path, algal::clef::MAX_BODY)) + .transpose()?, + }) } else if let Some(model) = &options.jev { Some(Backend::Jev { model: model.clone(), @@ -1261,7 +1285,11 @@ fn host(options: &Execution, dir: &Path) -> Result { let mut host = Host::default(); if let Some(backend) = backend { backend.validate()?; - host.entries.push(("default".into(), backend)); + let id = match &backend { + Backend::Clef { model, .. } => algal::clef::executor_id(model), + _ => "default".into(), + }; + host.entries.push((id, backend)); } host }; @@ -1505,6 +1533,7 @@ async fn execute(cli: Cli) -> Result { && options.host.is_none() && options.gateway_model.is_none() && options.jev.is_none() + && options.clef.is_none() && options.recall.is_none() && options.base_url.is_none() && !options.apple @@ -3347,6 +3376,11 @@ async fn execute(cli: Cli) -> Result { emit(&algal::credentials::status(&provider)?)?; return Ok(true); } + if provider == "clef" { + return Err(Error::invalid( + "Clef credentials are environment-only; set CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN (CLOUDFLARE_AUTH_TOKEN is also accepted)", + )); + } if forget { let removed = algal::credentials::forget(&provider)?; emit(&json!({"provider":provider,"removed":removed}))?; @@ -3382,6 +3416,7 @@ async fn execute(cli: Cli) -> Result { apple, apple_bridge, jev, + clef, json, } => { // The report is data on stdout: a pipe keeps the JSON object unless @@ -3391,6 +3426,19 @@ async fn execute(cli: Cli) -> Result { && (io::stdout().is_terminal() || cli_style::explicit_audience(&cli_style::process_env) == Some(cli_style::Audience::Human)); + if clef { + let credential = algal::credentials::status("clef")?; + let account_configured = algal::clef::endpoint( + &std::env::var("CLOUDFLARE_ACCOUNT_ID").unwrap_or_default(), + "clef", + ) + .is_ok(); + let configured = account_configured && credential["configured"] == true; + emit( + &json!({"provider":"clef","configured":configured,"accountConfigured":account_configured,"credential":credential,"liveChecked":false}), + )?; + return Ok(configured); + } if jev { let status = tokio::task::spawn_blocking(|| algal::credentials::status("jev")) .await @@ -3476,7 +3524,9 @@ async fn execute(cli: Cli) -> Result { } Ok(diagnosis.available()) } else if human { - let jev_env = std::env::var("TYPESAFE_API_KEY").is_ok_and(|key| !key.is_empty()); + let clef_env = std::env::var("CLOUDFLARE_API_TOKEN") + .or_else(|_| std::env::var("CLOUDFLARE_AUTH_TOKEN")) + .is_ok_and(|key| !key.is_empty()); print_doctor( &[ ( @@ -3487,16 +3537,16 @@ async fn execute(cli: Cli) -> Result { platform_name() ), ), - if jev_env { + if clef_env { ( cli_style::Symbol::Ok, - "Jev key found in TYPESAFE_API_KEY. Test it: algal doctor --jev" + "Cloudflare token found. Check configuration: algal doctor --clef" .into(), ) } else { ( cli_style::Symbol::Skip, - "Jev decisions: not checked. Test a saved key: algal doctor --jev" + "Clef decisions: not configured. Set CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN." .into(), ) }, diff --git a/crates/algal/tests/clef.rs b/crates/algal/tests/clef.rs new file mode 100644 index 00000000..a1b1a9f0 --- /dev/null +++ b/crates/algal/tests/clef.rs @@ -0,0 +1,213 @@ +use algal::{clef, effects::Backend}; +use serde_json::{Value, json}; + +fn image() -> Value { + json!({"content_type":"image/png","base64":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+j1ioAAAAASUVORK5CYII="}) +} +fn questions() -> Value { + json!({"keep":{"type":"noul","instructions":"Keep this?"},"label":{"type":"choice","instructions":"Choose","criteria":{"yes":null,"no":null}},"rate":{"type":"score","instructions":"Rate","criteria":["low","high"]}}) +} +fn envelope() -> Value { + json!({"success":true,"errors":[],"messages":[],"result":{"model":"clef","answers":{"keep":{"type":"noul","noul":0.8},"label":{"type":"choice","choice":"yes","confidence":0.9,"probabilities":{"yes":0.9,"no":0.1}},"rate":{"type":"score","score":0.7,"confidence":0.7,"probabilities":{"0":0.3,"1":0.7},"legend":{"0":"low","1":"high"}}},"usage":{"input_tokens":12,"output_tokens":0}}}) +} + +#[test] +fn captured_cloudflare_contract_preserves_images_and_legends() { + let account = "a".repeat(32); + assert_eq!( + clef::endpoint(&account, "clef").unwrap(), + format!( + "https://api.cloudflare.com/client/v4/accounts/{account}/ai/run/@cf/cloudflare/clef" + ) + ); + assert!(clef::endpoint("../other", "clef").is_err()); + assert!(clef::endpoint(&account, "jev").is_err()); + let images = json!([image()]); + let body = clef::request( + "clef", + &json!({"text":"state"}), + &questions(), + Some(&images), + ) + .unwrap(); + assert_eq!( + body, + json!({"model":"clef","state":{"text":"state"},"questions":questions(),"images":images}) + ); + let (out, meta) = clef::response(&envelope(), "clef", &questions()).unwrap(); + assert_eq!( + out["answers"]["rate"]["legend"], + json!({"0":"low","1":"high"}) + ); + assert_eq!( + meta, + json!({"usage":{"model":"clef","tokensIn":12,"tokensOut":0}}) + ); +} + +#[test] +fn mismatched_or_invalid_responses_are_private_errors() { + for (path, value) in [ + ("/success", json!(false)), + ("/result/model", json!("clef-flash")), + ("/result/usage/input_tokens", json!(-1)), + ("/result/answers/keep/noul", json!(2)), + ("/result/answers/label/confidence", json!(-0.1)), + ("/result/answers/label/choice", json!("other")), + ("/result/answers/label/probabilities/yes", json!(0.8)), + ("/result/answers/rate/legend/0", json!("PRIVATE")), + ("/result/answers/rate/score", json!(1.8)), + ] { + let mut bad = envelope(); + *bad.pointer_mut(path).unwrap() = value; + let error = clef::response(&bad, "clef", &questions()).unwrap_err(); + assert_eq!(error.code, "EFFECT_UNPARSEABLE"); + assert!(!error.message.contains("PRIVATE")); + } + let mut extra = envelope(); + extra["result"]["answers"]["extra"] = json!({"type":"noul","noul":0.5}); + assert!(clef::response(&extra, "clef", &questions()).is_err()); + assert!(clef::response(&envelope()["result"], "clef", &questions()).is_err()); +} + +#[test] +fn preflight_rejects_unsafe_images_and_question_variants() { + for images in [ + json!(["https://example.com/a.png"]), + json!([{ "content_type":"image/svg+xml","base64":"AAAA" }]), + json!([{ "content_type":"image/png","base64":"broken" }]), + json!([image(), image(), image(), image(), image()]), + ] { + assert!(clef::request("clef", &json!({}), &questions(), Some(&images)).is_err()); + } + let url = format!( + "data:image/png;base64,{}", + image()["base64"].as_str().unwrap() + ); + assert!(clef::check_images(&json!([url])).is_ok()); + assert!( + clef::request( + "clef", + &json!({}), + &json!({"bad id":{"type":"noul","instructions":"q"}}), + None + ) + .is_err() + ); + assert!( + clef::request( + "clef", + &json!({}), + &json!({"q":{"type":"choice","instructions":"q","criteria":{"one":null}}}), + None + ) + .is_err() + ); + let name = "a".repeat(100); + assert!( + clef::request( + "clef", + &json!({}), + &json!({name:{"type":"noul","instructions":"q"}}), + None + ) + .is_ok() + ); + let effect = json!({"kind":"decide","prompt":"Evaluate","context":{},"questions":questions(),"images":[image()]}); + assert_eq!( + clef::prepare("clef", &effect, None).unwrap().0["images"], + effect["images"] + ); + assert!(clef::prepare("clef", &json!({"kind":"gate"}), None).is_err()); +} + +#[test] +fn canonical_backend_is_explicit_and_does_not_accept_old_credentials() { + let backend: Backend = + serde_json::from_value(json!({"kind":"clef","accountId":"a".repeat(32)})).unwrap(); + backend.validate().unwrap(); + assert!(backend.supports("classifier")); + assert!(backend.supports("decide")); + assert!(!backend.supports("agent")); + assert!(!backend.supports("gate")); + assert_eq!(clef::executor_id("clef"), "clef"); + assert_eq!(clef::executor_id("clef-flash"), "clef:clef-flash"); + assert!( + serde_json::from_value::( + json!({"kind":"clef","accountId":"a".repeat(32),"credentialEnv":"TYPESAFE_API_KEY"}) + ) + .is_err() + ); + assert!(algal::credentials::store("clef", "fake-test-token").is_err()); + assert_eq!( + algal::credentials::spec("clef").unwrap().env, + "CLOUDFLARE_API_TOKEN" + ); +} + +#[test] +fn inline_clef_images_share_the_unchanged_one_mib_host_limit() { + let mut config = json!({"contract":"algal.host.v1","executors":{"judge":{"kind":"clef","accountId":"a".repeat(32),"images":[image()]}}}); + let host = algal::effects::Host::from_config(&config).unwrap(); + assert!(host.entries[0].1.supports("decide")); + config["executors"]["judge"]["images"][0]["base64"] = json!("A".repeat(1_048_576)); + let error = match algal::effects::Host::from_config(&config) { + Ok(_) => panic!("oversized host accepted"), + Err(error) => error, + }; + assert_eq!(error.message, "host config bytes"); +} + +#[test] +fn cli_configuration_checks_do_not_contact_either_provider() { + let run = |args: &[&str], env: &[(&str, &str)]| { + let mut command = std::process::Command::new(env!("CARGO_BIN_EXE_algal")); + command + .arg("--no-update") + .args(args) + .env_clear() + .stdin(std::process::Stdio::null()); + for (key, value) in env { + command.env(key, value); + } + command.output().unwrap() + }; + let help = run(&["run", "--help"], &[]); + let text = String::from_utf8(help.stdout).unwrap(); + assert!(text.contains("--clef")); + assert!(text.contains("--images")); + let status = run( + &["auth", "clef", "--status"], + &[("TYPESAFE_API_KEY", "fake-legacy-token")], + ); + assert_eq!( + serde_json::from_slice::(&status.stdout).unwrap(), + json!({"provider":"clef","configured":false}) + ); + let account = "a".repeat(32); + let checked = run( + &["doctor", "--clef"], + &[ + ("CLOUDFLARE_ACCOUNT_ID", &account), + ("CLOUDFLARE_AUTH_TOKEN", "fake-cloudflare-token"), + ], + ); + assert!(checked.status.success()); + let report: Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["configured"], true); + assert_eq!(report["liveChecked"], false); + assert!( + !String::from_utf8(checked.stdout) + .unwrap() + .contains("fake-cloudflare-token") + ); + let missing = run(&["doctor", "--clef"], &[]); + assert_eq!(missing.status.code(), Some(1)); + let save = run(&["auth", "clef"], &[]); + assert!(!save.status.success()); + assert!( + String::from_utf8(save.stderr) + .unwrap() + .contains("environment-only") + ); +} diff --git a/crates/algal/tests/cli_help.rs b/crates/algal/tests/cli_help.rs index d855bfde..6e1509f5 100644 --- a/crates/algal/tests/cli_help.rs +++ b/crates/algal/tests/cli_help.rs @@ -164,7 +164,7 @@ fn doctor_prints_sentences_for_a_person_and_json_for_scripts() { )), "{stdout}" ); - assert!(stdout.contains("– Jev decisions: not checked.")); + assert!(stdout.contains("– Clef decisions: not configured.")); assert_eq!(text(&human.stderr), "Next: algal demo start ./my-review\n"); let piped = algal(&["doctor"], &[]); diff --git a/docs/cumulative-skill-experiment.md b/docs/cumulative-skill-experiment.md index bb64bd52..865c32c4 100644 --- a/docs/cumulative-skill-experiment.md +++ b/docs/cumulative-skill-experiment.md @@ -6,6 +6,13 @@ composing earlier procedures makes the system better at later work than equally resourced alternatives. It preserves the original design and reports the exploratory studies below. The broader claim remains open. +For new decision runs, use the Cloudflare Clef variants in +`experiments/cumulative-skill/pipeline/record-triage-clef.algal.json` and +`record-classify-clef.algal.json`, with an explicitly attached Clef executor. +Set `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`; paid calls are opt-in. +The Jev study descriptions and saved results below remain historical, not +measurements of Clef. + ## Claim under test Under a fixed model and a controlled budget, retaining and composing earlier diff --git a/docs/executors.md b/docs/executors.md index efcd526a..a0b9874b 100644 --- a/docs/executors.md +++ b/docs/executors.md @@ -1,9 +1,9 @@ # Executors -Agent, classifier, and gate cells never reach a provider directly. Each -activation emits a `algal.effect.v1` request; the host's executor answers -it. ALGAL owns the schedule, the bounds, and the receipt — the executor -owns provider access and anything the provider does. +Agent, classifier, decide, recall, and gate cells never reach a provider +directly. Each activation emits an `algal.effect.v1` request; the host's +executor answers it. ALGAL controls the schedule, limits and saved run; +the executor handles provider access. ## The wire shape @@ -28,9 +28,10 @@ stdout fails the cell. } ``` -`kind` is `agent`, `classifier`, or `gate`. A gate is an approval point — -route those requests to a human or a policy check, not a model. `route` is a -hint carried through verbatim; honoring it is the executor's business. +`kind` is `agent`, `classifier`, `decide`, `recall`, or `gate`. A `decide` +request carries `questions`; `recall` carries a derived query. A gate is an +approval point: route it to a human or policy check, not a model. `route` +selects a compatible host executor by provider or preset name. `context.cells` appears only when the cell's `view.cells` declares ancestor cells; each entry is the ancestor's committed record (`status`, `outputs?`) @@ -122,6 +123,92 @@ responseFormat?, timeoutMs?, maxResponseBytes? })` from `@hraness/algal`. `--recall local` selects deterministic trigram embeddings, not a local language model; model generation uses the endpoint options above. +## Cloudflare Clef + +Clef serves `decide` and `classifier` cells, never generated agent output +or approval gates. Both CLIs select it with `--clef`, which defaults to +`clef`; use `--clef clef-flash` for the other supported model. Hosted calls +are paid and opt-in. ALGAL has not measured Clef quality or latency for +its examples; historical Jev measurements do not apply to Clef. + +Set `CLOUDFLARE_ACCOUNT_ID` to 32 lowercase hexadecimal characters and +`CLOUDFLARE_API_TOKEN` to your token. `CLOUDFLARE_AUTH_TOKEN` is a token +alias; the nonempty primary token takes precedence. Both CLIs resolve +the token only from the environment, never a TypeSafe key or local vault. +The account may also be set in a native host's `accountId` field. +`algal auth clef --status` reports redacted credential status and +`algal doctor --clef` checks configuration without an API call. + +```sh +bun cli.ts run examples/model-router.algal.json \ + --args examples/model-router.args.json --clef +algal run examples/model-router.algal.json \ + --args examples/model-router.args.json --clef clef-flash +``` + +The Bun `--executors` map and both bench executor maps accept `clef` or +`clef:clef-flash`. A native `--host` file uses `algal.host.v1`: + +```json +{ + "contract": "algal.host.v1", + "executors": { "judge": { "kind": "clef", "model": "clef" } }, + "defaultExecutor": "judge" +} +``` + +The native backend accepts optional `accountId` (defaulting to the +account environment variable) and `images`. Library hosts use +`clefExecutor({ accountId?, model?, images?, credential? })` or +`clefAsker` from `@hraness/algal`; `credential` may be an in-memory token +or an async resolver. A custom `asker` cannot be combined with configured +images. Credentials never enter configuration, cache or saved-run identity. +The account, model and configured image bytes do: changing any of them +changes the cache identity and saved `configurationDigest`. + +### Images and limits + +`--clef --images images.json` attaches the JSON file's image array to each +Clef request. Use embedded PNG/JPEG/WebP data URLs or +`{ "content_type": "image/png", "base64": "..." }` objects. Remote URLs +are rejected. The limits are 4 images, 4 MiB per image and 8 MiB total after +base64 decoding into file bytes, 16 megapixels per image, and 13 MiB for +the image JSON file and final request body. Inline native host images +share the separate 1 MiB limit for the entire `algal.host.v1` configuration, +including base64 text. Use the separate `--images` file for the larger budget. + +Normal manifest runs receive only host-configured images. There is no +manifest image field, implicit file read, or source-code attachment. +Direct low-level calls may use `clefAsker.ask(state, questions, signal, +images)`, TypeScript `EffectRequest.images`, or native `clef::prepare`; +supply them before computing the request digest. Only one image source is +accepted. Container structure, signatures and dimensions are checked +before credential resolution and dispatch. The adapter does not decode +pixel streams or verify PNG checksums, so it cannot guarantee that every +payload will decode. Header-only and truncated containers are rejected; +animated WebP without a top-level image payload is not supported. + +Text state is limited to 262144 canonical JSON bytes. Question IDs match +`[A-Za-z0-9_.-]{1,100}`; there are at most 64 questions, with nonempty +instructions up to 4096 UTF-8 bytes. Choices require 2..32 options; scores +require 2..10 ordered levels. Responses must match the model, IDs, types +and legends. Probability and score rounding follow the +[decision specification](../spec/v1/organism.md#decide-cells), preserving +the reported values. Default response size is 1 MiB and the adapter's +maximum request duration is 120 seconds, further limited by the cell's +deadline. The fixed Cloudflare account/model endpoint rejects redirects; +transport or response-read failure after dispatch is uncertain and cannot +be automatically retried. Errors withhold credentials and response bodies. +Saved runs record model identity, token usage and configuration digest and +replay without either provider. + +## Legacy Jev + +`--jev`, `jevAsker`, `jevExecutor` and native `kind:"jev"` are explicit +legacy adapters. They use `TYPESAFE_API_KEY` or the legacy credential vault, +never Cloudflare credentials, and reject images. Historical fixtures and +study results keep their original Jev identities and replay offline. + ## Apple Intelligence The native CLI's `algal run --apple` (and `algal civ --live --apple`) routes cells to Apple's @@ -250,8 +337,10 @@ described under "Tool registries". ## Rules of thumb - Keep secrets in the command's environment or config, never in manifests. -- The request is the whole job: prompt, view-selected context, output - contract, budgets. Do not read other files to answer it. +- The request contains the prompt, view-selected context, output contract + and budgets. Provider-specific host configuration may also supply fixed + inputs such as Clef images; bind those inputs to configuration and cache + identity. Do not implicitly read unrelated files to answer a request. - Print only the output value on stdout — logs go to stderr. - Respect `budget.maxOutputBytes`; oversized output fails the cell. - Errors count: a thrown effect is recorded on the receipt as diff --git a/docs/model-router.md b/docs/model-router.md index 525481a8..a50253d2 100644 --- a/docs/model-router.md +++ b/docs/model-router.md @@ -4,12 +4,17 @@ `frontier` or `standard`: the binary capability tier a coding-agent model picker decides at session start. It is small, fast, and replayable: two `expr` cells compute deterministic shape features and the routing score; -one `decide` cell asks a typed-decision provider (Jev) for six scores. +one `decide` cell asks a host-selected decision provider for six scores. +The manifest is provider-neutral; Cloudflare Clef is the recommended +adapter for new runs. Hosted execution is paid and opt-in; set +`CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` before +using `--clef`. The default model is `clef`; `--clef clef-flash` selects the +other supported model. ```sh bun cli.ts run examples/model-router.algal.json \ - --args '{"src":{"task":"migrate the session store and update every caller"}}' \ - --jev + --args examples/model-router.args.json \ + --clef ``` Output: `{"route": "frontier"|"standard", "score": , @@ -50,6 +55,10 @@ capability probe does not need one. ## Provenance and limits +The fitted head uses historical Jev scores. The weights and measured results +below have not been evaluated with Clef; selecting Clef does not establish +the same routing quality. + The head has two generations, both fit on one operator's local Codex, Claude Code, and Devin CLI stores and labeled by the model that served the first turn: `claude-fable-*` and `gpt-6-astra*` count as @@ -100,13 +109,15 @@ generations are published at hraness.com/prompting/data.json. ## Cost and speed -One routing decision costs one bounded `decide` call (~1k tokens in, -~0.2k out with `jev-latest`) plus two fueled `expr` evaluations — no -agent cell, no generated text. All six questions ride in a single ask. +One routing decision uses one `decide` call plus two fueled `expr` +evaluations, with no agent cell or generated text. All six questions ride in +a single request. Historical `jev-latest` runs used about 1k input tokens and +0.2k output tokens per decision. Clef cost and latency have not been measured +for this example. ## Reuse -Any typed-decision executor can serve the `decide` cell — `--jev`, +Any typed-decision executor can serve the `decide` cell, including `--clef`, `--executor-cmd`, or a `route`/`preset` executor. Retrain the head by re-fitting the nine weights on a new labeled sample and editing the `combine` program; the cell boundaries do not change. diff --git a/examples/model-router.algal.json b/examples/model-router.algal.json index c5f5fa80..ca4451eb 100644 --- a/examples/model-router.algal.json +++ b/examples/model-router.algal.json @@ -2,7 +2,7 @@ "contract": "algal.organism.v1", "key": "organism:model-router", "name": "Model router", - "note": "Fast frontier-vs-standard router for first-session task prompts. One expr cell computes deterministic shape features (word count, imperative open, resume/takeover markers, action-verb density); one bounded decide call asks a typed-decision provider (Jev) for difficulty, scope, ambiguity, stakes, kind, and a frontier-worthiness probability; a second expr cell applies the fitted linear head and kind gates. Generation 1 coefficients: a prior-anchored logistic update of the September 22 head (306 prompts) on 84 first prompts from September 2026, the window in which the frontier choice existed (see docs/model-router.md); every feature is task-intrinsic, not operator identity. Route output: frontier | standard.", + "note": "Fast frontier-vs-standard router for first-session task prompts. One expr cell computes deterministic shape features (word count, imperative open, resume/takeover markers, action-verb density); one bounded decide call asks a host-selected typed-decision provider for difficulty, scope, ambiguity, stakes, kind, and a frontier-worthiness probability; a second expr cell applies the fitted linear head and kind gates. Generation 1 coefficients: a prior-anchored logistic update of the September 22 head (306 prompts) on 84 first prompts from September 2026, the window in which the frontier choice existed (see docs/model-router.md); every feature is task-intrinsic, not operator identity. The fitted weights and measured results are historical Jev results, not evaluated on Clef. Route output: frontier | standard.", "budgets": { "maxSteps": 16, "maxAgentCalls": 1, diff --git a/experiments/cumulative-skill/README.md b/experiments/cumulative-skill/README.md index e61a2e2f..952f2ae6 100644 --- a/experiments/cumulative-skill/README.md +++ b/experiments/cumulative-skill/README.md @@ -30,13 +30,20 @@ pipeline/ record-triage.algal.json reference pipeline (agent + expr cells) record-triage.responses.json scripted classify responses for the acquisition batches, keyed by request digest - record-triage-jev.algal.json per-record classify variant: `each` maps - records through a child classifier whose - `labelsExpr` resolves the task taxonomy - record-classify.algal.json the child: one classifier effect routed to - `provider: "jev"`, emitting {recordId, label} + record-triage-clef.algal.json current per-record classify variant: `each` + maps records through the Clef child + record-classify-clef.algal.json classifier routed to `provider: "clef"`, + with task taxonomy resolved by `labelsExpr` + record-triage-jev.algal.json historical Jev pipeline + record-classify.algal.json historical Jev child ``` +The current hosted decision variant uses Cloudflare Clef. Attach `--clef` +(or a named native `kind:"clef"` host), set `CLOUDFLARE_ACCOUNT_ID` and +`CLOUDFLARE_API_TOKEN`, and load the Clef child from `pipeline/`. Calls are +paid and opt-in. The Jev variants and completed study results remain +historical; their accuracy and cost numbers are not Clef measurements. + ## Task spec: `algal.experiment-task.v1` A task spec is a closed object with exactly these keys. Unknown keys at any diff --git a/experiments/cumulative-skill/pipeline/record-classify-clef.algal.json b/experiments/cumulative-skill/pipeline/record-classify-clef.algal.json new file mode 100644 index 00000000..a8255c3f --- /dev/null +++ b/experiments/cumulative-skill/pipeline/record-classify-clef.algal.json @@ -0,0 +1,33 @@ +{ + "contract": "algal.organism.v1", + "key": "organism:experiment-record-classify-clef", + "name": "Record classify child with Clef", + "note": "The per-record child for the Cloudflare Clef classify arm. Each item runs one classifier effect; labelsExpr derives the choice set from the task spec. The emit cell pairs the record id with the chosen label for deterministic recombination. Hosted execution is paid and opt-in; historical Jev results do not measure Clef quality.", + "budgets": { "maxSteps": 8, "maxAgentCalls": 2, "maxWork": 50000 }, + "interface": { + "inputs": { "record": { "cell": "in", "port": "record" }, "spec": { "cell": "in", "port": "spec" } }, + "outputs": { "pair": { "cell": "emit", "port": "out" } } + }, + "cells": [ + { "id": "in", "kind": "input", "outputs": { "record": "json", "spec": "json" } }, + { + "id": "cls", "kind": "classifier", "inputs": { "record": "json", "spec": "json" }, + "prompt": "Assign the record its issue class. The spec's taxonomy.classes list the valid class ids; each class's about text describes it. Judge from the record's subject and body alone; the pipeline applies the declared rules afterward.", + "view": { "inputs": "*" }, + "output": { "kind": "choice", "labelsExpr": { "contract": "algal.expr.v1", "program": ["map", ["get", "spec", "taxonomy", "classes"], "c", ["get", "c", "id"]] } }, + "route": { "provider": "clef" }, + "budget": { "maxContextBytes": 65536, "maxOutputBytes": 1024, "maxEffectMs": 120000 } + }, + { + "id": "emit", "kind": "expr", "inputs": { "record": "json", "label": "json" }, + "expr": { "contract": "algal.expr.v1", "program": { "recordId": ["get", "record", "id"], "label": ["get", "label"] } }, + "output": { "kind": "json", "schema": { "type": "object" } } + } + ], + "edges": [ + { "from": { "cell": "in", "port": "record" }, "to": { "cell": "cls", "port": "record" } }, + { "from": { "cell": "in", "port": "spec" }, "to": { "cell": "cls", "port": "spec" } }, + { "from": { "cell": "cls", "port": "out" }, "to": { "cell": "emit", "port": "label" } }, + { "from": { "cell": "in", "port": "record" }, "to": { "cell": "emit", "port": "record" } } + ] +} diff --git a/experiments/cumulative-skill/pipeline/record-triage-clef.algal.json b/experiments/cumulative-skill/pipeline/record-triage-clef.algal.json new file mode 100644 index 00000000..8e4b4fde --- /dev/null +++ b/experiments/cumulative-skill/pipeline/record-triage-clef.algal.json @@ -0,0 +1,70 @@ +{ + "contract": "algal.organism.v1", + "key": "organism:experiment-record-triage-clef", + "name": "Record triage task pipeline with Clef", + "note": "Each record is classified through the Cloudflare Clef child, with the task taxonomy resolved by labelsExpr. Deterministic expressions apply rules, summarize and pack results. Hosted execution is paid and opt-in. Historical Jev results are preserved separately and do not measure this variant.", + "budgets": { "maxSteps": 256, "maxAgentCalls": 64, "maxWork": 500000 }, + "interface": { + "inputs": { "records": { "cell": "in", "port": "records" }, "spec": { "cell": "in", "port": "spec" } }, + "outputs": { "out": { "cell": "pack", "port": "out" } } + }, + "cells": [ + { "id": "in", "kind": "input", "outputs": { "records": "json", "spec": "json" } }, + { "id": "per", "kind": "each", "manifest": "sha256:938551cf60a8b733053776da2509053ebc14a1b29894730e5c6d5dbd429ed5f1", "over": "record", "maxItems": 64 }, + { + "id": "apply", "kind": "expr", "inputs": { "records": "json", "spec": "json", "pairs": "json" }, + "expr": { + "contract": "algal.expr.v1", + "program": [ + "map", ["get", "records"], "r", + ["let", "lab", + ["fold", ["get", "pairs"], null, "acc", "p", ["if", ["eq", ["get", "p", "recordId"], ["get", "r", "id"]], ["get", "p", "label"], ["get", "acc"]]], + ["merge", + { "recordId": ["get", "r", "id"], "label": ["get", "lab"] }, + ["fold", ["get", "spec", "rules"], {}, "acc", "rule", + ["if", + ["fold", ["keys", ["get", "rule", "when"]], true, "ok", "slot", + ["and", ["get", "ok"], + ["let", "c", ["get", "rule", "when", ["get", "slot"]], + ["let", "sv", ["if", ["eq", ["get", "slot"], "label"], ["get", "lab"], ["get", "r", ["get", "slot"]]], + ["if", ["isMap", ["get", "c"]], + ["if", ["has", ["get", "c"], "in"], + ["contains", ["get", "c", "in"], ["get", "sv"]], + ["and", + ["if", ["has", ["get", "c"], "min"], ["gte", ["get", "sv"], ["get", "c", "min"]], true], + ["if", ["has", ["get", "c"], "max"], ["lte", ["get", "sv"], ["get", "c", "max"]], true], + ["if", ["has", ["get", "c"], "eq"], ["eq", ["get", "sv"], ["get", "c", "eq"]], true]]], + ["eq", ["get", "c"], ["get", "sv"]]]]]]], + ["merge", ["get", "acc"], ["get", "rule", "then"]], ["get", "acc"]]]]]] + }, + "output": { "kind": "json", "schema": { "type": "array" } } + }, + { + "id": "summarize", "kind": "expr", "inputs": { "results": "json", "spec": "json" }, + "expr": { + "contract": "algal.expr.v1", + "program": [ + "map", ["get", "spec", "outputFormat", "summaries"], "sname", + ["if", ["eq", ["get", "sname"], "total"], + { "name": "total", "counts": ["list", { "key": "records", "count": ["len", ["get", "results"]] }] }, + ["if", ["eq", ["get", "sname"], "perLabel"], + { "name": "perLabel", "counts": ["map", ["get", "spec", "taxonomy", "classes"], "c", { "key": ["get", "c", "id"], "count": ["len", ["filter", ["get", "results"], "r", ["eq", ["get", "r", "label"], ["get", "c", "id"]]]] }] }, + ["let", "fld", ["if", ["eq", ["get", "sname"], "perDecision"], "decision", ["if", ["eq", ["get", "sname"], "perPriority"], "priority", "queue"]], + { "name": ["get", "sname"], "counts": ["map", ["get", "spec", "outputFormat", "fields", ["get", "fld"]], "L", { "key": ["get", "L"], "count": ["len", ["filter", ["get", "results"], "r", ["eq", ["get", "r", ["get", "fld"]], ["get", "L"]]]] }] }]]]] + }, + "output": { "kind": "json", "schema": { "type": "array" } } + }, + { "id": "pack", "kind": "expr", "inputs": { "results": "json", "summary": "json" }, "expr": { "contract": "algal.expr.v1", "program": { "results": ["get", "results"], "summary": ["get", "summary"] } }, "output": { "kind": "json", "schema": { "type": "object" } } } + ], + "edges": [ + { "from": { "cell": "in", "port": "records" }, "to": { "cell": "per", "port": "record" } }, + { "from": { "cell": "in", "port": "spec" }, "to": { "cell": "per", "port": "spec" } }, + { "from": { "cell": "in", "port": "records" }, "to": { "cell": "apply", "port": "records" } }, + { "from": { "cell": "in", "port": "spec" }, "to": { "cell": "apply", "port": "spec" } }, + { "from": { "cell": "per", "port": "pair" }, "to": { "cell": "apply", "port": "pairs" } }, + { "from": { "cell": "apply", "port": "out" }, "to": { "cell": "summarize", "port": "results" } }, + { "from": { "cell": "in", "port": "spec" }, "to": { "cell": "summarize", "port": "spec" } }, + { "from": { "cell": "apply", "port": "out" }, "to": { "cell": "pack", "port": "results" } }, + { "from": { "cell": "summarize", "port": "out" }, "to": { "cell": "pack", "port": "summary" } } + ] +} diff --git a/index.ts b/index.ts index 53607738..fe23a99f 100644 --- a/index.ts +++ b/index.ts @@ -236,6 +236,8 @@ export { jevExecutor, } from "./src/jev"; export type { JevAskerOptions, JevExecutorOptions } from "./src/jev"; +export { CLEF_DEFAULT_MODEL, CLEF_MODELS, CLEF_IMAGE_LIMITS, clefAsker, clefExecutor, clefEndpoint, clefRequest, parseClefResponse, validateClefImages } from "./src/clef"; +export type { ClefModel, ClefImage, ClefAskerOptions, ClefExecutorOptions } from "./src/clef"; export { algalHome, diff --git a/package.json b/package.json index b6b610a1..583ae699 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "./store-memory": "./src/store-memory.ts", "./verify": "./src/verify.ts", "./decisions": "./src/decisions.ts", + "./clef": "./src/clef.ts", "./effects": "./src/effects.ts", "./digest": "./src/digest.ts", "./values": "./src/values.ts", diff --git a/scripts/clef-fixture.ts b/scripts/clef-fixture.ts new file mode 100644 index 00000000..9bd4040d --- /dev/null +++ b/scripts/clef-fixture.ts @@ -0,0 +1,63 @@ +import { readFileSync } from "node:fs"; +import { clefExecutor, clefRequest, parseClefResponse, validateClefImages, type ClefImage, type ClefModel } from "../src/clef"; +import { effectRequestDigest, replayExecutor, type EffectRequest } from "../src/effects"; +import type { DecisionQuestions } from "../src/decisions"; +import { digestCanonical } from "../src/digest"; +import { AlgalError } from "../src/errors"; +import { asJsonValue, type JsonObject, type JsonValue } from "../src/values"; + +export const fixture = JSON.parse(readFileSync(new URL("./fixtures/clef.json", import.meta.url), "utf8")) as { + accountId: string; questions: DecisionQuestions; images: ClefImage[]; response: JsonObject; + cases: { name: string; model: ClefModel; source: string; imageIndices: number[]; patch: Record; error?: string }[]; + invalidImages: unknown[]; +}; + +export async function clefFixture(): Promise { + const cases: JsonValue[] = []; + for (const item of fixture.cases) { + const envelope = structuredClone(fixture.response); + for (const [path, value] of Object.entries(item.patch)) { + const parts = path.slice(1).split("/"); + let target: JsonObject = envelope; + for (const part of parts.slice(0, -1)) target = target[part] as JsonObject; + target[parts.at(-1)!] = value; + } + const images = item.imageIndices.map(index => fixture.images[index]!); + const configured = item.source === "configured" ? images : undefined; + const request: EffectRequest = { contract: "algal.effect.v1", cellId: "probe", kind: "decide", prompt: "Evaluate the observation.", context: { inputs: { text: "東京 observation" }, turn: 0 }, output: { kind: "json", schema: {} }, budget: { maxContextBytes: 4096, maxOutputBytes: 4096 }, questions: fixture.questions, ...(item.source === "request" ? { images } : {}) }; + const state = { prompt: request.prompt, context: request.context }; + const body = clefRequest(item.model, asJsonValue(state, "state"), fixture.questions, request.images ?? configured); + const executor = clefExecutor({ accountId: fixture.accountId, model: item.model, ...(configured ? { images: configured } : {}), credential: "fake-offline-token", fetch: async (_url, init) => { + if (String(init?.body) !== JSON.stringify(body) && digestCanonical(JSON.parse(String(init?.body))) !== digestCanonical(body)) throw new Error("captured body differs"); + return Response.json(envelope); + } }); + const configuration = { kind: "clef", accountId: fixture.accountId, model: item.model, images: configured ?? null }; + const requestDigest = effectRequestDigest(request); + const receipt: JsonObject = { requestDigest, executor: executor.id, ...asJsonValue(await executor.receiptFor!(request), "metadata") as JsonObject }; + let parsed: JsonValue; + try { + parsed = asJsonValue(parseClefResponse(envelope, item.model, fixture.questions), "parsed"); + const execution = await executor.executeEffect!(request); + receipt.output = execution.output; + Object.assign(receipt, execution.metadata); + if (item.error) throw new Error(`${item.name} accepted an invalid envelope`); + } catch (error) { + if (!(error instanceof AlgalError) || error.code !== item.error) throw error; + parsed = { error: { code: error.code, message: error.message } }; + receipt.error = { code: error.code, message: error.message }; + } + const replay = replayExecutor([receipt as unknown as Parameters[0][number]]); + const replayReceipt: JsonObject = { requestDigest, ...asJsonValue(await replay.receiptFor!(request), "replay metadata") as JsonObject }; + try { replayReceipt.output = await replay.execute(request); } + catch (error) { + if (!(error instanceof AlgalError)) throw error; + replayReceipt.error = { code: error.code, message: error.message }; + } + cases.push(asJsonValue({ name: item.name, request, body, parsed, configuration, configurationDigest: digestCanonical(configuration), cacheIdentity: executor.cacheIdentity, imageDigest: digestCanonical(images), receipt, replay: replayReceipt }, "case")); + } + const invalidImages = fixture.invalidImages.map(images => { + try { validateClefImages(images); return false; } catch { return true; } + }); + if (invalidImages.some(rejected => !rejected)) throw new Error("invalid image fixture was accepted"); + return { cases, invalidImages }; +} diff --git a/scripts/clef-parity.ts b/scripts/clef-parity.ts new file mode 100644 index 00000000..4f3891ea --- /dev/null +++ b/scripts/clef-parity.ts @@ -0,0 +1,74 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { resolve, join } from "node:path"; +import { clefFixture, fixture } from "./clef-fixture"; +import { canonicalize } from "../src/values"; + +const root = resolve(import.meta.dir, ".."); +const binary = resolve(root, "target/debug/examples/clef_parity"); +const native = resolve(root, "target/debug/algal"); +const env = { PATH: process.env.PATH ?? "", CLOUDFLARE_ACCOUNT_ID: fixture.accountId, HRANESS_TELEMETRY: "off", HRANESS_SUPPORT: "off", ALGAL_NO_UPDATE: "1" }; +async function command(argv: string[]) { + const child = Bun.spawn(argv, { cwd: root, env, stdin: "ignore", stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); + return { stdout, stderr, code }; +} +const expected = canonicalize(await clefFixture()); +const result = await command([binary]); +if (result.code !== 0) throw new Error(`native Clef fixture failed: ${result.stderr}`); +const actual = canonicalize(JSON.parse(result.stdout)); +if (expected !== actual) { + const reference = JSON.parse(expected), compiled = JSON.parse(actual); + for (let i = 0; i < reference.cases.length; i++) { + if (canonicalize(reference.cases[i]) !== canonicalize(compiled.cases[i])) throw new Error(`Clef parity mismatch in ${reference.cases[i].name}: Bun ${canonicalize(reference.cases[i])}; native ${canonicalize(compiled.cases[i])}`); + } + throw new Error("Clef invalid-image parity mismatch"); +} + +function pngOfSize(size: number) { + const image = fixture.images[0] as { content_type: "image/png"; base64: string }; + const seed = Buffer.from(image.base64, "base64"), filler = Buffer.alloc(size - seed.length); + filler.writeUInt32BE(filler.length - 12, 0); filler.write("paDd", 4); + let crc = 0xffffffff; + for (const byte of filler.subarray(4, -4)) { + crc ^= byte; + for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ ((crc & 1) ? 0xedb88320 : 0); + } + filler.writeUInt32BE((crc ^ 0xffffffff) >>> 0, filler.length - 4); + return { ...image, base64: Buffer.concat([seed.subarray(0, 33), filler, seed.subarray(33)]).toString("base64") }; +} +const directory = await mkdtemp(join(tmpdir(), "algal-clef-parity-")); +let cliChecks = 0; +try { + const args = join(directory, "args.json"), images = join(directory, "images.json"), host = join(directory, "host.json"); + await writeFile(args, JSON.stringify({ src: { value: "alpha" } })); + const large = pngOfSize(4 * 1024 * 1024); + const runtimes = [[process.execPath, "cli.ts"], [native, "--no-update"]]; + for (const runtime of runtimes) { + const base = [...runtime, "run", "examples/foundry-constant.algal.json", "--args", args, "--dir", join(directory, "store")]; + for (const [value, accepted] of [ + [[large, large], true], + [[large, large, fixture.images[0]], false], + [[pngOfSize(4 * 1024 * 1024 + 1)], false], + [Array(5).fill(fixture.images[0]), false], + ] as const) { + await writeFile(images, JSON.stringify(value)); + const output = await command([...base, "--clef", "--images", images]); + if ((output.code === 0) !== accepted) throw new Error(`Clef image CLI capacity failed (${runtime[0]}, accepted=${accepted}): ${output.stderr} ${output.stdout}`); + cliChecks++; + } + await writeFile(images, " ".repeat(13 * 1024 * 1024 + 1)); + if ((await command([...base, "--clef", "--images", images])).code === 0) throw new Error("oversized image JSON file accepted"); + cliChecks++; + } + const config = (value: unknown) => ({ contract: "algal.host.v1", executors: { clef: { kind: "clef", accountId: fixture.accountId, images: value } } }); + const base = [native, "--no-update", "run", "examples/foundry-constant.algal.json", "--args", args, "--dir", join(directory, "store"), "--host", host]; + await writeFile(host, JSON.stringify(config([fixture.images[0]]))); + if ((await command(base)).code !== 0) throw new Error("small inline Clef host image rejected"); + cliChecks++; + await writeFile(host, JSON.stringify(config([large]))); + const rejected = await command(base); + if (rejected.code === 0 || !rejected.stderr.includes("bytes")) throw new Error(`1 MiB host limit not preserved: ${rejected.stderr}`); + cliChecks++; +} finally { await rm(directory, { recursive: true, force: true }); } +console.log(`Clef offline parity: ${fixture.cases.length} request/parser/cache/receipt/replay cases, ${fixture.invalidImages.length} invalid image cases, ${cliChecks} CLI capacity checks passed; no provider calls or credentials.`); diff --git a/scripts/fixtures/clef.json b/scripts/fixtures/clef.json new file mode 100644 index 00000000..40a7a86c --- /dev/null +++ b/scripts/fixtures/clef.json @@ -0,0 +1,51 @@ +{ + "accountId": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "questions": { + "keep": { "type": "noul", "instructions": "Keep the observation?" }, + "label": { "type": "choice", "instructions": "Choose a label.", "criteria": { "a": null, "b": "Other" } }, + "severity": { "type": "score", "instructions": "How severe is the customer impact?", "criteria": ["No impact", "Minor", "Major", "Critical"] } + }, + "images": [ + { "content_type": "image/png", "base64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+j1ioAAAAASUVORK5CYII=" }, + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+j1ioAAAAASUVORK5CYII=", + { "content_type": "image/webp", "base64": "UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AAAAAA" }, + { "content_type": "image/jpeg", "base64": "/9j/wAALCAABAAEBAREA/9oACAEBAAA/AAH/2Q==" } + ], + "response": { + "success": true, + "errors": [], + "messages": [], + "result": { + "model": "clef", + "usage": { "input_tokens": 319, "output_tokens": 0 }, + "answers": { + "keep": { "type": "noul", "noul": 0.8 }, + "label": { "type": "choice", "choice": "a", "confidence": 0.75, "probabilities": { "a": 0.75, "b": 0.25 } }, + "severity": { "type": "score", "score": 2.931, "confidence": 0.8612, "probabilities": { "0": 0.0047, "1": 0.0051, "2": 0.0448, "3": 0.9454 }, "legend": { "0": "No impact", "1": "Minor", "2": "Major", "3": "Critical" } } + } + } + }, + "cases": [ + { "name": "text", "model": "clef", "source": "none", "imageIndices": [], "patch": {} }, + { "name": "configured-png", "model": "clef", "source": "configured", "imageIndices": [0], "patch": {} }, + { "name": "direct-png", "model": "clef", "source": "request", "imageIndices": [0], "patch": {} }, + { "name": "configured-data-url", "model": "clef", "source": "configured", "imageIndices": [1], "patch": {} }, + { "name": "flash-all-formats", "model": "clef-flash", "source": "configured", "imageIndices": [0, 1, 2, 3], "patch": { "/result/model": "clef-flash", "/result/answers/severity/score": 2.7378, "/result/answers/severity/confidence": 0.5316, "/result/answers/severity/probabilities": { "0": 0.0149, "1": 0.0157, "2": 0.186, "3": 0.7834 } } }, + { "name": "feasible-three-decimal-choice", "model": "clef", "source": "none", "imageIndices": [], "patch": { "/result/answers/label/probabilities": { "a": 0.75, "b": 0.249 } } }, + { "name": "inconsistent-score", "model": "clef", "source": "none", "imageIndices": [], "patch": { "/result/answers/severity/score": 2.921 }, "error": "EFFECT_UNPARSEABLE" }, + { "name": "infeasible-probabilities", "model": "clef", "source": "none", "imageIndices": [], "patch": { "/result/answers/label/probabilities": { "a": 0.7, "b": 0.2 } }, "error": "EFFECT_UNPARSEABLE" }, + { "name": "wrong-legend", "model": "clef", "source": "none", "imageIndices": [], "patch": { "/result/answers/severity/legend/0": "PRIVATE" }, "error": "EFFECT_UNPARSEABLE" }, + { "name": "wrong-model", "model": "clef", "source": "none", "imageIndices": [], "patch": { "/result/model": "clef-flash" }, "error": "EFFECT_UNPARSEABLE" }, + { "name": "failed-envelope", "model": "clef", "source": "none", "imageIndices": [], "patch": { "/success": false, "/errors": [{ "message": "PRIVATE" }] }, "error": "EFFECT_UNPARSEABLE" }, + { "name": "unknown-answer", "model": "clef", "source": "none", "imageIndices": [], "patch": { "/result/answers/extra": { "type": "noul", "noul": 0.5 } }, "error": "EFFECT_UNPARSEABLE" }, + { "name": "negative-usage", "model": "clef", "source": "none", "imageIndices": [], "patch": { "/result/usage/input_tokens": -1 }, "error": "EFFECT_UNPARSEABLE" } + ], + "invalidImages": [ + ["https://example.com/private.png"], + [{ "content_type": "image/png", "base64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwC" }], + [{ "content_type": "image/jpeg", "base64": "/9j/wAAICAABAAEB/9k=" }], + [{ "content_type": "image/webp", "base64": "UklGRhYAAABXRUJQVlA4WAoAAAAAAAAAAAAAAAAA" }], + [{ "content_type": "image/png", "base64": "AB==" }], + [{ "content_type": "image/svg+xml", "base64": "AAAA" }] + ] +} diff --git a/spec/v1/organism.md b/spec/v1/organism.md index 5dd04f69..69ef1647 100644 --- a/spec/v1/organism.md +++ b/spec/v1/organism.md @@ -396,10 +396,10 @@ A runtime that knows only version 2 rejects `schemaVersion: 3` with "instructions": "Pick the closest lane.", "criteria": { "bug": null, "feature": null, "chore": null } }, - "risk": { "type": "score", "instructions": "Rate breakage risk." } + "risk": { "type": "score", "instructions": "Rate breakage risk.", "criteria": ["Low", "High"] } }, "view": { "inputs": "*" }, - "route": { "provider": "jev" }, + "route": { "provider": "clef" }, "budget": { "maxContextBytes": 65536 }, "retry": { "attempts": 2 } } @@ -410,9 +410,11 @@ A `decide` cell declares a bounded question map (≤ 64 names, each `criteria`). It declares **no** output contract — the contract is derived from the question map, an `{"answers": {: }}` record: -- `noul` answers `{noul: number}` — a keep/relevance probability. -- `choice` answers `{choice: "", confidence, probabilities}`. -- `score` answers `{score: number, confidence, probabilities}`. +- `noul` answers `{type:"noul", noul: number}`, a keep/relevance probability. +- `choice` answers `{type:"choice", choice: "", confidence, probabilities}`. +- `score` answers `{type:"score", score: number, confidence, probabilities, legend?}`. + Clef requires and preserves a legend mapping zero-based level indices to + the declared criteria; its score is on 0..(number of levels − 1). The cell activates like an agent with one turn and no tools: the effect request carries `questions` alongside `context`, and a *decision provider* @@ -426,6 +428,47 @@ reject `agent` and `gate` requests — approval stays human/policy-routed. decision provider serves them by synthesizing one `choice` question from the declared labels. +Cloudflare Clef is an opt-in decision backend (`--clef`, or native +`kind:"clef"`). It defaults to model `clef`; `clef-flash` is also supported. +The CLIs resolve the account ID from `CLOUDFLARE_ACCOUNT_ID` and the bearer +token from `CLOUDFLARE_API_TOKEN` or `CLOUDFLARE_AUTH_TOKEN`, without a local +credential vault. Library hosts may provide an in-memory credential. +Requests use the fixed Cloudflare account/model REST URL with redirects +forbidden. Responses must use a successful REST envelope, match the model +and question IDs, and provide probabilities and confidence in 0..1, +matching score legends, and nonnegative token counts. Each probability has +a clipped ±0.0005 interval for three-decimal rounding. Those intervals must +contain a distribution summing to 1; a score's ±0.0005 interval must also +intersect the achievable weighted mean of that same distribution. Reported +values are preserved, not renormalized. Score legends are saved with +the decision output. Clef question IDs match `[A-Za-z0-9_.-]{1,100}`; +instructions must be nonempty, choices require 2..32 options and scores +require 2..10 levels. ALGAL rejects oversized state or question evidence +instead of truncating it; the provider's context limits also apply. + +Ordinary manifest runs receive only host-configured images, through +`--clef --images `, the TypeScript executor's `images` option, or the +native Clef backend's `images` field. There is no manifest image field or +implicit file/source-code attachment. TypeScript `EffectRequest.images` +and native `clef::prepare` accept images only for direct low-level adapter +calls; they are not an image hook in the scheduler. Only one image source +is accepted, and TypeScript rejects a custom `asker` with configured images. + +The array accepts embedded PNG/JPEG/WebP data URLs or +`{content_type,base64}` objects. It is limited to 4 images, 4 MiB each and +8 MiB total after base64 decoding into file bytes, 16 megapixels each, and +a 13 MiB JSON file/request body. Native `algal.host.v1` files keep their +separate 1 MiB limit for the entire configuration, including inline base64; +`--clef --images` supports the larger image budget without changing that +host limit. Remote URLs, invalid MIME/base64, missing image payloads, +truncated containers and exceeded limits fail before credential resolution +or provider access. Container structure and dimensions are checked, but +pixel streams and PNG checksums are not decoded or verified. These checks +do not guarantee a decodable image payload. Direct effect-request images +participate in the request digest; executor-configured images participate +in the configuration/cache identity. Credentials never participate in +those identities. Explicit legacy Jev adapters reject images. + `budget.maxTurns` is fixed at 1 for `decide`; other budget fields, `view`, `route`, and `retry` behave as on agent cells. @@ -443,7 +486,7 @@ the declared labels. "k": 8, "embedder": "local", "route": { "provider": "recall" }, - "rerank": { "route": { "provider": "jev" }, "take": 4 }, + "rerank": { "route": { "provider": "clef" }, "take": 4 }, "budget": { "maxContextBytes": 65536, "maxOutputBytes": 65536 }, "retry": { "attempts": 2 } } diff --git a/src/clef-cli.test.ts b/src/clef-cli.test.ts new file mode 100644 index 00000000..6cf4516a --- /dev/null +++ b/src/clef-cli.test.ts @@ -0,0 +1,50 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const root = resolve(import.meta.dir, ".."); +async function cli(args: string[], env: Record = {}) { + const child = Bun.spawn([process.execPath, "cli.ts", ...args], { cwd: root, env: { PATH: process.env.PATH ?? "", HRANESS_TELEMETRY: "off", HRANESS_SUPPORT: "off", ...env }, stdin: "ignore", stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]); + return { stdout, stderr, code }; +} + +test("Clef CLI documents both models and images, and only checks environment configuration", async () => { + expect((await cli(["run", "--help"])).stdout).toContain("--clef [clef|clef-flash]"); + expect((await cli(["run", "--help"])).stdout).toContain("--images "); + const status = await cli(["auth", "clef", "--status"], { TYPESAFE_API_KEY: "fake-legacy-token" }); + expect(JSON.parse(status.stdout)).toEqual({ provider: "clef", configured: false }); + const check = await cli(["doctor", "--clef"], { CLOUDFLARE_ACCOUNT_ID: "a".repeat(32), CLOUDFLARE_AUTH_TOKEN: "fake-cloudflare-token" }); + expect(check.code).toBe(0); + expect(JSON.parse(check.stdout)).toMatchObject({ provider: "clef", configured: true, accountConfigured: true, liveChecked: false }); + expect(check.stdout).not.toContain("fake-cloudflare-token"); + const missing = await cli(["doctor", "--clef"]); + expect(missing.code).toBe(1); + expect(JSON.parse(missing.stdout)).toMatchObject({ configured: false, liveChecked: false }); + const save = await cli(["auth", "clef", "--stdin"]); + expect(save.code).toBe(2); + expect(save.stderr).toContain("environment-only"); +}); + +test("invalid model and image inputs fail before any Cloudflare request", async () => { + const home = await mkdtemp(join(tmpdir(), "algal-clef-cli-")); + try { + const env = { HOME: home, ALGAL_HOME: home, CLOUDFLARE_ACCOUNT_ID: "a".repeat(32) }; + const common = ["run", "examples/model-router.algal.json", "--args", "examples/model-router.args.json", "--dir", join(home, "store")]; + const invalidModel = await cli([...common, "--clef", "jev-latest"], env); + expect(invalidModel.code).toBe(2); + expect(invalidModel.stderr).toContain("clef or clef-flash"); + const images = join(home, "images.json"); + await writeFile(images, JSON.stringify(["https://example.com/private.png"])); + const invalidImages = await cli([...common, "--clef", "--images", images], env); + expect(invalidImages.code).not.toBe(0); + expect(invalidImages.stderr).toContain("embedded PNG"); + const noProvider = await cli([...common, "--images", images], env); + expect(noProvider.code).toBe(2); + expect(noProvider.stderr).toContain("requires --clef"); + const mixed = await cli([...common, "--clef", "--jev"], env); + expect(mixed.code).toBe(2); + expect(mixed.stderr).toContain("choose one default executor"); + } finally { await rm(home, { recursive: true, force: true }); } +}); diff --git a/src/clef-fixture.test.ts b/src/clef-fixture.test.ts new file mode 100644 index 00000000..2c2d261c --- /dev/null +++ b/src/clef-fixture.test.ts @@ -0,0 +1,19 @@ +import { expect, test } from "bun:test"; +import { clefFixture, fixture } from "../scripts/clef-fixture"; +import { asObject, canonicalize, type JsonValue } from "./values"; + +test("shared offline Clef fixture preserves request, parser and replay identities", async () => { + const result = asObject(await clefFixture(), "fixture"); + const cases = result.cases as JsonValue[]; + expect(cases).toHaveLength(fixture.cases.length); + expect(result.invalidImages).toEqual(fixture.invalidImages.map(() => true)); + for (const value of cases) { + const item = asObject(value, "case"); + expect(canonicalize(item.receipt!)).toBe(canonicalize(item.replay!)); + expect(asObject(item.receipt, "receipt").configurationDigest).toBe(item.configurationDigest); + } + const byName = Object.fromEntries(cases.map(value => { const item = asObject(value, "case"); return [String(item.name), item]; })); + expect(byName["configured-png"]!.cacheIdentity).not.toBe(byName.text!.cacheIdentity); + expect(byName["configured-png"]!.configurationDigest).not.toBe(byName["configured-data-url"]!.configurationDigest); + expect(asObject(byName["direct-png"]!.receipt, "receipt").requestDigest).not.toBe(asObject(byName["configured-png"]!.receipt, "receipt").requestDigest); +}); diff --git a/src/clef-image.ts b/src/clef-image.ts new file mode 100644 index 00000000..32b779cb --- /dev/null +++ b/src/clef-image.ts @@ -0,0 +1 @@ +export type ClefImage = string | { content_type: "image/png" | "image/jpeg" | "image/webp"; base64: string }; diff --git a/src/clef.test.ts b/src/clef.test.ts new file mode 100644 index 00000000..e56fd636 --- /dev/null +++ b/src/clef.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, test } from "bun:test"; +import { clefAsker, clefEndpoint, clefExecutor, parseClefResponse, validateClefImages } from "./clef"; +import { effectRequestDigest, replayExecutor, type EffectRequest } from "./effects"; +import { resolveCredential } from "./credentials"; + +const accountId = "a".repeat(32); +const credential = "fake-cloudflare-token"; +const questions = { keep: { type: "noul" as const, instructions: "Keep this?" } }; +const result = { model: "clef", answers: { keep: { type: "noul" as const, noul: 0.8 } }, usage: { input_tokens: 12, output_tokens: 0 } }; +const image = { content_type: "image/png" as const, base64: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+j1ioAAAAASUVORK5CYII=" }; + +describe("Cloudflare Clef", () => { + test("captures fixed-origin REST requests and unwraps matched answers", async () => { + const asker = clefAsker({ accountId, credential, fetch: async (url, init) => { + expect(url).toBe(clefEndpoint(accountId, "clef")); + expect(init?.redirect).toBe("error"); + expect(init?.headers).toMatchObject({ authorization: `Bearer ${credential}` }); + expect(JSON.parse(String(init?.body))).toEqual({ model: "clef", state: { text: "state" }, questions, images: [image] }); + return Response.json({ success: true, result, errors: [], messages: [] }); + }}); + expect(await asker.ask({ text: "state" }, questions, undefined, [image])).toEqual({ answers: result.answers, usage: { model: "clef", tokensIn: 12, tokensOut: 0 } }); + expect(() => clefEndpoint("../other", "clef")).toThrow(); + expect(() => clefAsker({ accountId, credential, baseUrl: "https://evil.example" })).toThrow(); + expect(() => clefAsker({ accountId, credential, baseUrl: `${clefEndpoint(accountId, "clef")}?redirect=other` })).toThrow(); + expect(() => clefAsker({ accountId, credential, baseUrl: clefEndpoint(accountId, "clef-flash") })).toThrow(); + expect(() => clefAsker({ accountId, credential, maxResponseBytes: 0 })).toThrow(); + }); + + test("preserves score legends, validates probability mass and question correspondence", () => { + const qs = { label: { type: "choice" as const, instructions: "Choose", criteria: { yes: null, no: null } }, rate: { type: "score" as const, instructions: "Rate", criteria: ["low", "high"] } }; + const value = { model: "clef", usage: result.usage, answers: { label: { type: "choice", choice: "yes", confidence: 0.9, probabilities: { yes: 0.9, no: 0.1 } }, rate: { type: "score", score: 0.7, confidence: 0.7, probabilities: { "0": 0.3, "1": 0.7 }, legend: { "0": "low", "1": "high" } } } }; + expect(parseClefResponse({ success: true, result: value }, "clef", qs).answers.rate).toMatchObject({ legend: { "0": "low", "1": "high" } }); + for (const bad of [ + { ...value, model: "clef-flash" }, { ...value, usage: { input_tokens: -1, output_tokens: 0 } }, + { ...value, answers: {} }, { ...value, answers: { ...value.answers, extra: result.answers.keep } }, + { ...value, answers: { ...value.answers, label: { ...value.answers.label, confidence: 2 } } }, + { ...value, answers: { ...value.answers, label: { ...value.answers.label, choice: "other" } } }, + { ...value, answers: { ...value.answers, label: { ...value.answers.label, probabilities: { yes: 0.8, no: 0.1 } } } }, + { ...value, answers: { ...value.answers, rate: { ...value.answers.rate, legend: { "0": "wrong", "1": "high" } } } }, + ]) expect(() => parseClefResponse({ success: true, result: bad }, "clef", qs)).toThrow("Invalid Cloudflare Clef response"); + expect(() => parseClefResponse({ success: false, result, errors: [{ message: "PRIVATE" }] }, "clef", questions)).toThrow("Invalid Cloudflare Clef response"); + expect(() => parseClefResponse(result, "clef", questions)).toThrow(); + }); + + test("accepts recorded synthetic Clef score rounding without accepting inconsistent scores", () => { + const qs = { severity: { type: "score" as const, instructions: "How severe is the customer impact?", criteria: ["No impact", "Minor", "Major", "Critical"] } }; + for (const sample of [ + { model: "clef" as const, score: 2.931, confidence: 0.8612, probabilities: { "0": 0.0047, "1": 0.0051, "2": 0.0448, "3": 0.9454 } }, + { model: "clef-flash" as const, score: 2.7378, confidence: 0.5316, probabilities: { "0": 0.0149, "1": 0.0157, "2": 0.186, "3": 0.7834 } }, + ]) { + const answer = { type: "score" as const, score: sample.score, confidence: sample.confidence, probabilities: sample.probabilities, legend: { "0": "No impact", "1": "Minor", "2": "Major", "3": "Critical" } }; + const envelope = { success: true, result: { model: sample.model, usage: { input_tokens: 319, output_tokens: 0 }, answers: { severity: answer } } }; + expect(parseClefResponse(envelope, sample.model, qs).answers.severity).toEqual(answer); + expect(() => parseClefResponse({ ...envelope, result: { ...envelope.result, answers: { severity: { ...answer, score: sample.score - 0.01 } } } }, sample.model, qs)).toThrow("Invalid Cloudflare Clef response"); + } + }); + + test("requires a normalized distribution within individual rounding intervals", () => { + const qs = { label: { type: "choice" as const, instructions: "Choose", criteria: { a: null, b: null, c: null } } }; + const envelope = (p: number) => ({ success: true, result: { model: "clef", usage: result.usage, answers: { label: { type: "choice", choice: "a", confidence: 0.2, probabilities: { a: p, b: p, c: p } } } } }); + expect(parseClefResponse(envelope(0.333), "clef", qs).answers.label).toMatchObject({ choice: "a" }); + for (const p of [0, 0.332, 0.334]) expect(() => parseClefResponse(envelope(p), "clef", qs)).toThrow("Invalid Cloudflare Clef response"); + }); + + test("rejects bad images and questions before credentials or provider access", async () => { + let calls = 0; + const asker = clefAsker({ accountId, credential: async () => { calls++; return credential; }, fetch: async () => { calls++; return Response.json({}); } }); + for (const images of [["https://example.com/a.png"], [{ ...image, base64: "broken" }], [{ ...image, content_type: "image/svg+xml" }], Array(5).fill(image), [{ ...image, base64: Buffer.alloc(4 * 1024 * 1024 + 1).toString("base64") }]]) { + await expect(asker.ask({}, questions, undefined, images)).rejects.toThrow(); + } + for (const qs of [{ "bad id": questions.keep }, { keep: { ...questions.keep, instructions: " " } }, { q: { type: "choice" as const, instructions: "choose", criteria: { one: null } } }]) await expect(asker.ask({}, qs)).rejects.toThrow(); + const bomb = Buffer.from(image.base64, "base64"); bomb.writeUInt32BE(4001, 16); bomb.writeUInt32BE(4001, 20); + expect(() => validateClefImages([{ ...image, base64: bomb.toString("base64") }])).toThrow(); + expect(calls).toBe(0); + }); + + test("images survive the decision executor and affect its private configuration identity", async () => { + const request: EffectRequest = { contract: "algal.effect.v1", cellId: "q", kind: "decide", prompt: "Evaluate", context: {}, output: { kind: "json", schema: {} }, budget: { maxContextBytes: 4096, maxOutputBytes: 4096 }, questions, images: [image] }; + const executor = clefExecutor({ accountId, credential, fetch: async (_url, init) => { expect(JSON.parse(String(init?.body)).images).toEqual([image]); return Response.json({ success: true, result }); } }); + expect(executor.id).toBe("clef"); + expect((await executor.execute(request) as { answers: unknown }).answers).toEqual(result.answers); + await expect(executor.execute({ ...request, kind: "gate" })).rejects.toThrow(); + await expect(executor.execute({ ...request, kind: "agent" })).rejects.toThrow(); + expect(clefExecutor({ accountId, credential: "another-fake-token" }).cacheIdentity).toBe(executor.cacheIdentity); + expect(clefExecutor({ accountId, credential, model: "clef-flash" }).id).toBe("clef:clef-flash"); + }); + + test("configured images cannot be silently dropped by a custom asker", () => { + let calls = 0; + const asker = { async ask() { calls++; return { answers: result.answers }; } }; + expect(() => clefExecutor({ accountId, images: [image], asker })).toThrow("custom asker cannot configure images"); + expect(calls).toBe(0); + }); + + test("image bytes bind replay; Clef and historical Jev receipts replay offline", async () => { + const request: EffectRequest = { contract: "algal.effect.v1", cellId: "q", kind: "decide", prompt: "Evaluate", context: {}, output: { kind: "json", schema: {} }, budget: { maxContextBytes: 4096, maxOutputBytes: 4096 }, questions, images: [image] }; + const requestDigest = effectRequestDigest(request); + expect(requestDigest).not.toBe(effectRequestDigest({ ...request, images: [] })); + for (const executor of ["clef", "jev"]) { + const output = { answers: result.answers }; + const replay = replayExecutor([{ requestDigest, output, executor, usage: { model: executor === "clef" ? "clef" : "jev-latest", tokensIn: 12, tokensOut: 0 } }]); + expect(await replay.execute(request)).toEqual(output); + } + }); + + test("aggregate bytes and JPEG/WebP container dimensions are checked", () => { + const png = Buffer.from(image.base64, "base64"); + const filler = Buffer.alloc(4 * 1024 * 1024 - png.length); + filler.writeUInt32BE(filler.length - 12, 0); filler.write("tEXt", 4); + const large = { ...image, base64: Buffer.concat([png.subarray(0, 33), filler, png.subarray(33)]).toString("base64") }; + expect(validateClefImages([large, large])).toHaveLength(2); + expect(() => validateClefImages([large, large, image])).toThrow(); + const jpeg = Buffer.from([255,216,255,192,0,11,8,0,1,0,1,1,1,17,0,255,218,0,8,1,1,0,0,63,0,1,255,217]); + expect(validateClefImages([{ content_type: "image/jpeg", base64: jpeg.toString("base64") }])).toHaveLength(1); + expect(() => validateClefImages([{ content_type: "image/png", base64: jpeg.toString("base64") }])).toThrow(); + jpeg.writeUInt16BE(4001, 7); jpeg.writeUInt16BE(4001, 9); + expect(() => validateClefImages([{ content_type: "image/jpeg", base64: jpeg.toString("base64") }])).toThrow(); + const webp = Buffer.from("UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AAAAAA", "base64"); + expect(validateClefImages([{ content_type: "image/webp", base64: webp.toString("base64") }])).toHaveLength(1); + webp.writeUInt16LE(4001, 26); webp.writeUInt16LE(4001, 28); + expect(() => validateClefImages([{ content_type: "image/webp", base64: webp.toString("base64") }])).toThrow(); + }); + + test("truncated or header-only containers fail before credential resolution", async () => { + const png = Buffer.from(image.base64, "base64"); + const webp = Buffer.alloc(30); webp.write("RIFF", 0); webp.writeUInt32LE(22, 4); webp.write("WEBPVP8X", 8); webp.writeUInt32LE(10, 16); + const invalid = [ + { ...image, base64: png.subarray(0, 33).toString("base64") }, + { ...image, base64: png.subarray(0, -1).toString("base64") }, + { ...image, base64: Buffer.concat([png, Buffer.from([0])]).toString("base64") }, + { content_type: "image/jpeg", base64: Buffer.from([255,216,255,192,0,8,8,0,1,0,1,1,255,217]).toString("base64") }, + { content_type: "image/webp", base64: webp.toString("base64") }, + ]; + let resolutions = 0, dispatches = 0; + const asker = clefAsker({ accountId, credential: async () => { resolutions++; return credential; }, fetch: async () => { dispatches++; return Response.json({}); } }); + for (const value of invalid) await expect(asker.ask({}, questions, undefined, [value])).rejects.toThrow("Invalid Clef image"); + expect(resolutions).toBe(0); expect(dispatches).toBe(0); + }); + + test("configured images bind the request, receipt and cache without accepting a second source", async () => { + let calls = 0; + const executor = clefExecutor({ accountId, credential, images: [image], fetch: async (_url, init) => { calls++; expect(JSON.parse(String(init?.body)).images).toEqual([image]); return Response.json({ success: true, result }); } }); + const request: EffectRequest = { contract: "algal.effect.v1", cellId: "q", kind: "decide", prompt: "Evaluate", context: {}, output: { kind: "json", schema: {} }, budget: { maxContextBytes: 4096, maxOutputBytes: 4096 }, questions }; + await executor.execute(request); + expect(executor.cacheIdentity).not.toBe(clefExecutor({ accountId, credential }).cacheIdentity); + expect(await executor.receiptFor?.(request)).not.toEqual(await clefExecutor({ accountId, credential }).receiptFor?.(request)); + await expect(executor.execute({ ...request, images: [] })).rejects.toThrow("images must have one source"); + expect(calls).toBe(1); + }); + + test("legacy TypeSafe environment values are never Clef credentials", async () => { + const saved = { ...process.env }; + try { + delete process.env.CLOUDFLARE_API_TOKEN; delete process.env.CLOUDFLARE_AUTH_TOKEN; + process.env.TYPESAFE_API_KEY = "legacy-private-token"; + expect(await resolveCredential("clef")).toBeUndefined(); + process.env.CLOUDFLARE_AUTH_TOKEN = credential; + expect(await resolveCredential("clef")).toEqual({ key: credential, source: "env" }); + process.env.CLOUDFLARE_API_TOKEN = "canonical-fake-token"; + expect((await resolveCredential("clef"))?.key).toBe("canonical-fake-token"); + } finally { process.env = saved; } + }); + + test("question byte limits reject oversized Unicode before dispatch", async () => { + const asker = clefAsker({ accountId, credential, fetch: async () => { throw new Error("must not dispatch"); } }); + await expect(asker.ask({}, { keep: { type: "noul", instructions: "😀".repeat(1100) } })).rejects.toMatchObject({ code: "PARSE_FAILED", uncertain: false }); + }); + + test("cancellation, transport uncertainty and error privacy remain intact", async () => { + let calls = 0; + const controller = new AbortController(); controller.abort(); + const asker = clefAsker({ accountId, credential: async () => { calls++; return credential; } }); + await expect(asker.ask({}, questions, controller.signal)).rejects.toMatchObject({ uncertain: false }); + expect(calls).toBe(0); + const failed = clefAsker({ accountId, credential, fetch: async () => { throw new Error("PRIVATE"); } }); + await expect(failed.ask({}, questions)).rejects.toMatchObject({ uncertain: true }); + const denied = clefAsker({ accountId, credential, fetch: async () => new Response("PRIVATE", { status: 401 }) }); + await expect(denied.ask({}, questions)).rejects.toThrow("body withheld"); + const limited = clefAsker({ accountId, credential, maxResponseBytes: 32, fetch: async () => new Response("x".repeat(33)) }); + await expect(limited.ask({}, questions)).rejects.toMatchObject({ uncertain: true }); + const redirect = clefAsker({ accountId, credential, fetch: async () => new Response("PRIVATE", { status: 302 }) }); + await expect(redirect.ask({}, questions)).rejects.toMatchObject({ code: "EFFECT_FAILED", uncertain: false }); + }); +}); diff --git a/src/clef.ts b/src/clef.ts new file mode 100644 index 00000000..f77ee501 --- /dev/null +++ b/src/clef.ts @@ -0,0 +1,253 @@ +import { decisionExecutor, parseDecisionQuestions, type DecisionAsker, type DecisionQuestions, type DecisionResponse, type DecisionState } from "./decisions"; +import type { Executor } from "./effects"; +import { AlgalError } from "./errors"; +import { boundedBytes } from "./io"; +import { digestCanonical } from "./digest"; +import { asJsonValue, asObject, canonicalBytes, canonicalize, noUnknownKeys, type JsonValue } from "./values"; +import { checkCredentialShape, credentialResolver } from "./credentials"; +import type { ClefImage } from "./clef-image"; + +export type { ClefImage }; +export const CLEF_DEFAULT_MODEL = "clef" as const; +export const CLEF_MODELS = ["clef", "clef-flash"] as const; +export type ClefModel = typeof CLEF_MODELS[number]; +export const CLEF_IMAGE_LIMITS = { maxImages: 4, maxImageBytes: 4 * 1024 * 1024, maxTotalImageBytes: 8 * 1024 * 1024, maxPixels: 16_000_000, maxBodyBytes: 13 * 1024 * 1024 } as const; + +function invalidImage(): never { throw new AlgalError("PARSE_FAILED", "Invalid Clef image: use embedded PNG, JPEG or WebP within the image limits"); } + +function dimensions(bytes: Buffer, mime: string): [number, number] { + if (mime === "image/png" && bytes.length >= 33 && bytes.subarray(0, 8).equals(Buffer.from([137,80,78,71,13,10,26,10]))) { + let offset = 8, data = 0; + let size: [number, number] | undefined; + while (offset + 12 <= bytes.length) { + const length = bytes.readUInt32BE(offset), end = offset + 12 + length; + if (end > bytes.length) return invalidImage(); + const chunk = bytes.toString("ascii", offset + 4, offset + 8); + if (offset === 8 && chunk !== "IHDR") return invalidImage(); + if (chunk === "IHDR") { + if (size || length !== 13 || bytes[offset + 18] !== 0 || bytes[offset + 19] !== 0 || bytes[offset + 20]! > 1) return invalidImage(); + const depths: Record = { 0: [1,2,4,8,16], 2: [8,16], 3: [1,2,4,8], 4: [8,16], 6: [8,16] }; + if (!depths[bytes[offset + 17]!]?.includes(bytes[offset + 16]!)) return invalidImage(); + size = [bytes.readUInt32BE(offset + 8), bytes.readUInt32BE(offset + 12)]; + } + if (chunk === "IDAT") data += length; + if (chunk === "IEND") return length === 0 && end === bytes.length && data > 0 && size ? size : invalidImage(); + offset = end; + } + } + if (mime === "image/jpeg" && bytes.length >= 4 && bytes[0] === 255 && bytes[1] === 216) { + let offset = 2, scans = 0; + let size: [number, number] | undefined; + while (offset + 2 <= bytes.length) { + if (bytes[offset++] !== 255) return invalidImage(); + while (bytes[offset] === 255) offset++; + const marker = bytes[offset++]; + if (marker === 217) return offset === bytes.length && scans > 0 && size ? size : invalidImage(); + if (marker === 1) continue; + if (marker === undefined || marker === 0 || marker === 216 || (marker >= 208 && marker <= 215) || offset + 2 > bytes.length) return invalidImage(); + const length = bytes.readUInt16BE(offset); + if (length < 2 || offset + length > bytes.length) return invalidImage(); + if ([192,193,194,195,197,198,199,201,202,203,205,206,207].includes(marker)) { + if (size || length < 11 || length !== 8 + 3 * bytes[offset + 7]!) return invalidImage(); + size = [bytes.readUInt16BE(offset + 5), bytes.readUInt16BE(offset + 3)]; + } + if (marker === 218) { + if (!size || length < 8 || length !== 6 + 2 * bytes[offset + 2]!) return invalidImage(); + offset += length; + const start = offset; + while (offset < bytes.length) { + if (bytes[offset] !== 255) { offset++; continue; } + if (bytes[offset + 1] === 0 || (bytes[offset + 1]! >= 208 && bytes[offset + 1]! <= 215)) { offset += 2; continue; } + break; + } + if (offset === start) return invalidImage(); + scans++; + } else offset += length; + } + } + if (mime === "image/webp" && bytes.length >= 25 && bytes.toString("ascii", 0, 4) === "RIFF" && bytes.readUInt32LE(4) + 8 === bytes.length && bytes.toString("ascii", 8, 12) === "WEBP") { + let offset = 12; + let size: [number, number] | undefined, canvas: [number, number] | undefined; + while (offset + 8 <= bytes.length) { + const chunk = bytes.toString("ascii", offset, offset + 4), length = bytes.readUInt32LE(offset + 4), start = offset + 8, end = start + length; + if (end + length % 2 > bytes.length) return invalidImage(); + if (chunk === "VP8X") { + if (offset !== 12 || length !== 10) return invalidImage(); + canvas = [1 + bytes.readUIntLE(start + 4, 3), 1 + bytes.readUIntLE(start + 7, 3)]; + } + if (chunk === "VP8L" || chunk === "VP8 ") { + if (size) return invalidImage(); + if (chunk === "VP8L" && length > 5 && bytes[start] === 47) { const bits = bytes.readUInt32LE(start + 1); size = [1 + (bits & 0x3fff), 1 + ((bits >>> 14) & 0x3fff)]; } + else if (chunk === "VP8 " && length > 10 && bytes[start + 3] === 157 && bytes[start + 4] === 1 && bytes[start + 5] === 42) size = [bytes.readUInt16LE(start + 6) & 0x3fff, bytes.readUInt16LE(start + 8) & 0x3fff]; + else return invalidImage(); + } + offset = end + length % 2; + } + if (offset === bytes.length && size && (!canvas || (canvas[0] === size[0] && canvas[1] === size[1]))) return size; + } + return invalidImage(); +} + +export function validateClefImages(value: unknown): ClefImage[] { + if (!Array.isArray(value) || value.length > CLEF_IMAGE_LIMITS.maxImages) return invalidImage(); + let total = 0; + return value.map((image: unknown) => { + let mime: string; + let encoded: string; + if (typeof image === "string") { + if (image.length > Math.ceil(CLEF_IMAGE_LIMITS.maxImageBytes / 3) * 4 + 32) return invalidImage(); + const match = /^data:(image\/(?:png|jpeg|webp));base64,(.+)$/i.exec(image); + if (!match) return invalidImage(); + mime = match[1]!.toLowerCase(); encoded = match[2]!; + } else { + const obj = asObject(image, "Clef image"); noUnknownKeys(obj, ["content_type", "base64"], "Clef image"); + if (typeof obj.content_type !== "string" || typeof obj.base64 !== "string") return invalidImage(); + mime = obj.content_type; encoded = obj.base64; + } + if (!["image/png", "image/jpeg", "image/webp"].includes(mime) || encoded.length < 4 || encoded.length > Math.ceil(CLEF_IMAGE_LIMITS.maxImageBytes / 3) * 4 || encoded.length % 4 !== 0) return invalidImage(); + for (let i = 0; i < encoded.length; i++) { + const c = encoded.charCodeAt(i); + if ((c >= 65 && c <= 90) || (c >= 97 && c <= 122) || (c >= 48 && c <= 57) || c === 43 || c === 47) continue; + if (c !== 61 || i < encoded.length - 2 || (i === encoded.length - 2 && encoded[i + 1] !== "=")) return invalidImage(); + } + const bytes = Buffer.from(encoded, "base64"); + if (bytes.toString("base64") !== encoded || bytes.length > CLEF_IMAGE_LIMITS.maxImageBytes) return invalidImage(); + total += bytes.length; + if (total > CLEF_IMAGE_LIMITS.maxTotalImageBytes) return invalidImage(); + const [width, height] = dimensions(bytes, mime); + if (width === 0 || height === 0 || width * height > CLEF_IMAGE_LIMITS.maxPixels) return invalidImage(); + return structuredClone(image) as ClefImage; + }); +} + +export function clefEndpoint(accountId: string, model: ClefModel = CLEF_DEFAULT_MODEL): string { + if (!/^[a-f0-9]{32}$/.test(accountId) || !CLEF_MODELS.includes(model)) throw new AlgalError("PARSE_FAILED", "Invalid Cloudflare Clef account or model"); + return `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run/@cf/cloudflare/${model}`; +} + +export function clefRequest(model: ClefModel, state: DecisionState, questions: DecisionQuestions, images?: unknown): JsonValue { + if (!CLEF_MODELS.includes(model)) throw new AlgalError("PARSE_FAILED", "Invalid Clef model"); + const parsed = parseDecisionQuestions(questions, "Clef questions"); + for (const [id, q] of Object.entries(parsed)) { + if (!/^[A-Za-z0-9_.-]{1,100}$/.test(id) || !q.instructions.trim()) throw new AlgalError("PARSE_FAILED", "Invalid Clef question ID or instructions"); + if (new TextEncoder().encode(q.instructions).byteLength > 4096 || (q.criteria !== undefined && Object.values(q.criteria).some(value => typeof value === "string" && new TextEncoder().encode(value).byteLength > 512))) throw new AlgalError("PARSE_FAILED", "Clef question exceeds instruction or criterion byte limit"); + const count = q.criteria === undefined ? 0 : Object.keys(q.criteria).length; + if ((q.type === "choice" && count < 2) || (q.type === "score" && (count < 2 || count > 10))) throw new AlgalError("PARSE_FAILED", "Clef choice requires at least two options; score requires 2..10 levels"); + } + asJsonValue(state, "Clef state"); + if (canonicalBytes(state) > 262_144) throw new AlgalError("BUDGET_EXHAUSTED", "Clef state exceeds 262144 bytes"); + const body: JsonValue = { model, state, questions: parsed as unknown as JsonValue, ...(images !== undefined ? { images: validateClefImages(images) as unknown as JsonValue } : {}) }; + if (canonicalBytes(body) > CLEF_IMAGE_LIMITS.maxBodyBytes) throw new AlgalError("BUDGET_EXHAUSTED", "Clef request exceeds 13 MiB"); + return body; +} + +export function parseClefResponse(value: unknown, model: ClefModel, questions: DecisionQuestions): DecisionResponse { + try { + const envelope = asObject(value, "Clef envelope"); + noUnknownKeys(envelope, ["success", "result", "errors", "messages"], "Clef envelope"); + if (envelope.success !== true || (envelope.errors !== undefined && (!Array.isArray(envelope.errors) || envelope.errors.length !== 0)) || (envelope.messages !== undefined && (!Array.isArray(envelope.messages) || envelope.messages.length > 64))) throw new Error(); + const raw = asObject(envelope.result, "Clef result"); noUnknownKeys(raw, ["model", "answers", "usage"], "Clef result"); + if (raw.model !== model) throw new Error(); + const declared = asObject(raw.answers, "Clef answers"); + if (Object.keys(declared).length !== Object.keys(questions).length || Object.keys(declared).some(id => !Object.hasOwn(questions, id))) throw new Error(); + const answers = Object.create(null) as DecisionResponse["answers"]; + const prob = (value: unknown): number => { if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) throw new Error(); return value; }; + for (const [id, q] of Object.entries(questions)) { + const a = asObject(declared[id], "Clef answer"); + if (a.type !== q.type) throw new Error(); + if (q.type === "noul") { noUnknownKeys(a, ["type", "noul"], "Clef answer"); answers[id] = { type: "noul", noul: prob(a.noul) }; continue; } + noUnknownKeys(a, q.type === "choice" ? ["type", "choice", "confidence", "probabilities"] : ["type", "score", "legend", "confidence", "probabilities"], "Clef answer"); + const p = asObject(a.probabilities, "Clef probabilities"); + const expected = q.type === "choice" ? Object.keys(q.criteria) : q.criteria.map((_, i) => String(i)); + if (Object.keys(p).length !== expected.length || expected.some(k => !Object.hasOwn(p, k))) throw new Error(); + const probabilities = Object.create(null) as Record; + let sum = 0; + for (const k of expected) { probabilities[k] = prob(p[k]); sum += probabilities[k]!; } + const bounds = expected.map(k => ({ lower: Math.max(0, probabilities[k]! - 0.0005), upper: Math.min(1, probabilities[k]! + 0.0005) })); + const lowerTotal = bounds.reduce((n, b) => n + b.lower, 0), upperTotal = bounds.reduce((n, b) => n + b.upper, 0); + if (sum <= 0 || lowerTotal > 1 + 1e-9 || upperTotal < 1 - 1e-9) throw new Error(); + const confidence = prob(a.confidence); + if (q.type === "choice") { + if (typeof a.choice !== "string" || !expected.includes(a.choice) || expected.some(k => probabilities[k]! > probabilities[a.choice as string]! + 0.000001)) throw new Error(); + answers[id] = { type: "choice", choice: a.choice, confidence, probabilities }; + } else { + const legend = asObject(a.legend, "Clef legend"); + const expectedLegend = Object.fromEntries(q.criteria.map((v, i) => [String(i), v])); + const endpoint = (maximum: boolean): number => { + let mean = bounds.reduce((n, b, i) => n + i * b.lower, 0), remaining = Math.max(0, 1 - lowerTotal); + for (let position = 0; position < bounds.length && remaining > 0; position++) { + const i = maximum ? bounds.length - 1 - position : position, b = bounds[i]!; + const added = Math.min(remaining, b.upper - b.lower); + mean += i * added; remaining -= added; + } + return mean; + }; + if (canonicalize(legend as JsonValue) !== canonicalize(expectedLegend) || typeof a.score !== "number" || !Number.isFinite(a.score) || a.score < 0 || a.score > q.criteria.length - 1 || a.score + 0.0005 < endpoint(false) - 1e-9 || a.score - 0.0005 > endpoint(true) + 1e-9) throw new Error(); + answers[id] = { type: "score", score: a.score, confidence, probabilities, legend: expectedLegend }; + } + } + const usage = asObject(raw.usage, "Clef usage"); noUnknownKeys(usage, ["input_tokens", "output_tokens"], "Clef usage"); + const tokens = (value: unknown): number => { if (!Number.isSafeInteger(value) || (value as number) < 0 || (value as number) > 1_000_000_000) throw new Error(); return value as number; }; + return { answers, usage: { model, tokensIn: tokens(usage.input_tokens), tokensOut: tokens(usage.output_tokens) } }; + } catch { throw new AlgalError("EFFECT_UNPARSEABLE", "Invalid Cloudflare Clef response; body withheld"); } +} + +export type ClefAskerOptions = { + credential?: string | (() => Promise); + accountId?: string; + model?: ClefModel; + baseUrl?: string; + fetch?: (input: Request | string | URL, init?: RequestInit) => Promise; + maxResponseBytes?: number; + images?: ClefImage[]; +}; + +export function clefAsker(options: ClefAskerOptions = {}): DecisionAsker { + const model = options.model ?? CLEF_DEFAULT_MODEL; + if (!CLEF_MODELS.includes(model)) throw new AlgalError("PARSE_FAILED", "Invalid Clef model"); + const max = options.maxResponseBytes ?? 1_048_576; + if (!Number.isSafeInteger(max) || max < 1 || max > 16_777_216) throw new AlgalError("PARSE_FAILED", "Invalid Clef response byte limit"); + const configuredAccount = options.accountId; + if (configuredAccount !== undefined) clefEndpoint(configuredAccount, model); + if (options.baseUrl !== undefined && options.baseUrl !== clefEndpoint(configuredAccount ?? process.env.CLOUDFLARE_ACCOUNT_ID ?? "", model)) throw new AlgalError("PARSE_FAILED", "Clef endpoint must be the fixed Cloudflare account/model endpoint"); + if (options.images !== undefined) validateClefImages(options.images); + const resolver = options.credential ?? credentialResolver("clef"); + return { + async ask(state, questions, signal, images) { + if (signal?.aborted) throw new AlgalError("BUDGET_EXHAUSTED", "Clef request cancelled before dispatch"); + if (images !== undefined && options.images !== undefined) throw new AlgalError("PARSE_FAILED", "images must have one source"); + signal = signal ? AbortSignal.any([signal, AbortSignal.timeout(120_000)]) : AbortSignal.timeout(120_000); + const request = clefRequest(model, state, questions, images ?? options.images) as { questions: JsonValue } & Record; + const body = canonicalize(request); + const endpoint = clefEndpoint(configuredAccount ?? process.env.CLOUDFLARE_ACCOUNT_ID ?? "", model); + const token = typeof resolver === "function" ? await resolver() : resolver; + checkCredentialShape(token, "Cloudflare credential"); + if (signal?.aborted) throw new AlgalError("BUDGET_EXHAUSTED", "Clef request cancelled before dispatch"); + let response: Response; + try { response = await (options.fetch ?? globalThis.fetch)(endpoint, { method: "POST", redirect: "error", headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body, ...(signal ? { signal } : {}) }); } + catch { throw new AlgalError(signal?.aborted ? "BUDGET_EXHAUSTED" : "EFFECT_FAILED", "Clef request transport failed; external completion uncertain", undefined, { uncertain: true }); } + if (!response.ok) { void response.body?.cancel().catch(() => {}); throw new AlgalError("EFFECT_FAILED", `Clef returned HTTP ${response.status}; redirects and response body withheld`); } + if (Number(response.headers.get("content-length")) > max) { void response.body?.cancel().catch(() => {}); throw new AlgalError("EFFECT_FAILED", "Clef response exceeds byte limit", undefined, { uncertain: true }); } + let bytes: Uint8Array; + try { bytes = await boundedBytes(response.body, max, "Clef response", signal); } + catch { throw new AlgalError(signal?.aborted ? "BUDGET_EXHAUSTED" : "EFFECT_FAILED", "Clef response read failed; external completion uncertain", undefined, { uncertain: true }); } + let raw: unknown; + try { raw = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); } + catch { throw new AlgalError("EFFECT_UNPARSEABLE", "Invalid Cloudflare Clef response; body withheld"); } + return parseClefResponse(raw, model, request.questions as unknown as DecisionQuestions); + }, + }; +} + +export type ClefExecutorOptions = ClefAskerOptions & { asker?: DecisionAsker }; + +export function clefExecutor(options: ClefExecutorOptions = {}): Executor { + if (options.asker !== undefined && options.images !== undefined) throw new AlgalError("PARSE_FAILED", "Clef custom asker cannot configure images; pass images through the low-level request instead"); + const model = options.model ?? CLEF_DEFAULT_MODEL; + const accountId = options.accountId ?? process.env.CLOUDFLARE_ACCOUNT_ID ?? ""; + const images = options.images !== undefined ? validateClefImages(options.images) : undefined; + const configuration = { kind: "clef", model, accountId, images: images ?? null } as unknown as JsonValue; + const endpoint = clefEndpoint(accountId, model); + const id = model === CLEF_DEFAULT_MODEL ? "clef" : `clef:${model}`; + return { ...decisionExecutor({ asker: options.asker ?? clefAsker({ ...options, model, accountId, ...(images !== undefined ? { images } : {}) }), id, cacheIdentity: digestCanonical({ provider: "cloudflare", baseUrl: endpoint, model, images: images ?? null } as unknown as JsonValue) }), receiptFor: () => ({ configurationDigest: digestCanonical(configuration) }) }; +} diff --git a/src/cli-golden/help.txt b/src/cli-golden/help.txt index 52d6c660..cf689474 100644 --- a/src/cli-golden/help.txt +++ b/src/cli-golden/help.txt @@ -34,7 +34,7 @@ Store and share manifests List saved programs Setup - auth jev Save a TypeSafe Jev key on this computer + auth clef --status Check environment-only Cloudflare credentials doctor Check that ALGAL is ready update Show this Bun runtime's manual update workflow diff --git a/src/cli-help.ts b/src/cli-help.ts index 89df3f2c..0d0789e4 100644 --- a/src/cli-help.ts +++ b/src/cli-help.ts @@ -57,7 +57,7 @@ const GROUPS: readonly Group[] = [ { title: "Setup", rows: [ - ["auth jev", "Save a TypeSafe Jev key on this computer"], + ["auth clef --status", "Check environment-only Cloudflare credentials"], ["doctor", "Check that ALGAL is ready"], ["update", "Show this Bun runtime's manual update workflow"], ], diff --git a/src/credentials.ts b/src/credentials.ts index 59f50663..05133885 100644 --- a/src/credentials.ts +++ b/src/credentials.ts @@ -15,7 +15,7 @@ import { homedir, platform } from "node:os"; import { join } from "node:path"; import { AlgalError } from "./errors"; -export type CredentialProvider = "jev"; +export type CredentialProvider = "clef" | "jev"; export type CredentialSource = | "option" @@ -42,6 +42,7 @@ type ProviderSpec = { }; const PROVIDERS: Record = { + clef: { env: "CLOUDFLARE_API_TOKEN", service: "algal.clef", account: "cloudflare" }, jev: { env: "TYPESAFE_API_KEY", service: "algal.jev", account: "typesafe" }, }; @@ -364,11 +365,12 @@ export async function resolveCredential( return { key: options.credential, source: "option" }; } const envName = options.env ?? PROVIDERS[provider].env; - const envValue = process.env[envName]; + const envValue = process.env[envName] || (provider === "clef" && options.env === undefined ? process.env.CLOUDFLARE_AUTH_TOKEN : undefined); if (typeof envValue === "string" && envValue.length > 0) { checkCredentialShape(envValue, `${provider} environment credential`); return { key: envValue, source: "env" }; } + if (provider === "clef") return undefined; const run = options.run ?? defaultRun; const backend = osBackend(provider); if (backend) { @@ -404,6 +406,7 @@ export async function storeCredential( key: string, options: { run?: CommandRunner } = {}, ): Promise<{ source: CredentialSource; location: string }> { + if (provider === "clef") throw new AlgalError("PARSE_FAILED", "Clef credentials are environment-only; set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID"); checkCredentialShape(key, `${provider} credential`); const run = options.run ?? defaultRun; const backend = osBackend(provider); @@ -425,6 +428,7 @@ export async function forgetCredential( provider: CredentialProvider, options: { run?: CommandRunner } = {}, ): Promise<{ removed: CredentialSource[] }> { + if (provider === "clef") throw new AlgalError("PARSE_FAILED", "Clef credentials are environment-only; remove the token from your environment"); const removed: CredentialSource[] = []; const run = options.run ?? defaultRun; const backend = osBackend(provider); diff --git a/src/decisions.ts b/src/decisions.ts index 6c2381fe..78b353e4 100644 --- a/src/decisions.ts +++ b/src/decisions.ts @@ -29,7 +29,7 @@ export type { DecisionQuestion, DecisionQuestions }; export const DECISION_BOUNDS = { maxQuestions: 64, - maxQuestionNameLen: 64, + maxQuestionNameLen: 100, maxInstructionsBytes: 4096, maxCriteria: 32, maxCriterionBytes: 512, @@ -49,6 +49,7 @@ export type DecisionAnswer = score: number; confidence: number; probabilities: Record; + legend?: Record; }; export type DecisionUsage = { @@ -72,6 +73,7 @@ export interface DecisionAsker { state: DecisionState, questions: DecisionQuestions, signal?: AbortSignal, + images?: unknown, ): Promise; } @@ -123,7 +125,7 @@ export function parseDecisionQuestion(u: unknown, at: string): DecisionQuestion `${at}.criteria requires 1..${DECISION_BOUNDS.maxCriteria} entries`, ); } - const out: Record = {}; + const out: Record = Object.create(null) as Record; for (const [k, v] of entries) { asString(k, `${at}.criteria key`, DECISION_BOUNDS.maxQuestionNameLen); out[k] = v === null ? null : asString(v, `${at}.criteria["${k}"]`, DECISION_BOUNDS.maxCriterionBytes); @@ -155,7 +157,7 @@ export function parseDecisionQuestions(u: unknown, at: string): DecisionQuestion `${at} requires 1..${DECISION_BOUNDS.maxQuestions} questions`, ); } - const out: DecisionQuestions = {}; + const out: DecisionQuestions = Object.create(null) as DecisionQuestions; for (const name of names) { asString(name, `${at} question name`, DECISION_BOUNDS.maxQuestionNameLen); out[name] = parseDecisionQuestion(obj[name], `${at}["${name}"]`); @@ -313,7 +315,7 @@ function choiceQuestion(request: EffectRequest): DecisionQuestions { `a decision executor cannot serve a ${request.output.kind} output — it answers typed decisions, not generated text`, ); } - const criteria: Record = {}; + const criteria = Object.create(null) as Record; for (const label of request.output.labels) criteria[label] = null; return { answer: { type: "choice", instructions: request.prompt, criteria }, @@ -377,7 +379,7 @@ export function decisionExecutor(options: DecisionExecutorOptions): Executor { prompt: request.prompt, context: request.context, } as unknown as JsonValue; - const response = await options.asker.ask(state, questions, signal); + const response = await options.asker.ask(state, questions, signal, request.images); const metadata = usageMeta(response); if (single !== undefined) { const answer = response.answers[single]; diff --git a/src/effects.ts b/src/effects.ts index 08f51441..1900b490 100644 --- a/src/effects.ts +++ b/src/effects.ts @@ -12,6 +12,7 @@ import { AlgalError, ERROR_CODES, type ErrorCode } from "./errors"; import { commandJson } from "./io-runtime"; import { asDigest, digestCanonical, type Digest } from "./digest"; import type { AgentOutput, Route } from "./contract"; +import type { ClefImage } from "./clef-image"; import { checkSchemaValueV2, checkSchemaValueV3, type SchemaVersion } from "./schema"; import type { Store } from "./store-contract"; import { @@ -67,6 +68,7 @@ export type EffectRequest = { * probes). Digested with the request, so a replayed run serves the same * questions to the same answers. */ questions?: DecisionQuestions; + images?: ClefImage[]; /** The derived semantic probe — present only on `kind:"recall"` requests: * the expr-evaluated query text, the hit cap, and the embedder spec whose * vectors the index must answer over. Digested with the request, so replay diff --git a/src/experiment-pipeline-clef.test.ts b/src/experiment-pipeline-clef.test.ts new file mode 100644 index 00000000..49b97830 --- /dev/null +++ b/src/experiment-pipeline-clef.test.ts @@ -0,0 +1,50 @@ +import { expect, test } from "bun:test"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { manifestToJson, parseOrganismManifest } from "./contract"; +import { digestCanonical } from "./digest"; +import { clefExecutor } from "./clef"; +import { gradeExperimentCase, parseExperimentTaskSpec, taskBatchArgs } from "./experiment-task"; +import { builtinRegistry } from "./registry"; +import { runOrganism } from "./run"; +import { MemoryStore } from "./store"; +import type { JsonObject, JsonValue } from "./values"; + +const directory = join(import.meta.dir, "..", "experiments", "cumulative-skill"); +const load = (file: string) => JSON.parse(readFileSync(join(directory, file), "utf8")) as JsonValue; +const manifest = parseOrganismManifest(load("pipeline/record-triage-clef.algal.json")); +const child = parseOrganismManifest(load("pipeline/record-classify-clef.algal.json")); + +test("Clef pipeline pins its separate child and preserves the taxonomy through the Cloudflare adapter", async () => { + expect((manifest.cells.find(cell => cell.kind === "each") as { manifest: string }).manifest).toBe(digestCanonical(manifestToJson(child))); + for (const file of readdirSync(join(directory, "tasks/acquisition")).filter(file => file.endsWith(".task.json")).sort()) { + const task = parseExperimentTaskSpec(load(`tasks/acquisition/${file}`)); + const store = new MemoryStore(); + await store.putManifest(child); + for (const batch of task.inputs) { + const labels = new Map(batch.expect.out.results.map(record => [record.recordId, record.label])); + let calls = 0; + const executor = clefExecutor({ accountId: "a".repeat(32), credential: "fake-offline-token", fetch: async (_url, init) => { + calls++; + const body = JSON.parse(String(init?.body)); + const criteria = body.questions.answer.criteria as Record; + expect(Object.keys(criteria).sort()).toEqual(task.taxonomy.classes.map(c => c.id).sort()); + const choice = labels.get(body.state.context.inputs.record.id)!; + expect(choice).toBeDefined(); + return Response.json({ success: true, result: { model: "clef", usage: { input_tokens: 12, output_tokens: 0 }, answers: { answer: { type: "choice", choice, confidence: 1, probabilities: Object.fromEntries(Object.keys(criteria).map(label => [label, label === choice ? 1 : 0])) } } } }); + } }); + const args: Record> = Object.create(null); + for (const [name, value] of Object.entries(taskBatchArgs(task, batch))) { + const target = manifest.interface!.inputs[name]!; + (args[target.cell] ??= Object.create(null))[target.port] = value; + } + const receipt = await runOrganism({ manifest, args, fns: builtinRegistry(), store, executors: [executor] }); + expect(receipt.outcome).toBe("complete"); + expect(calls).toBe(batch.records.length); + expect(receipt.effects.every(effect => effect.executor === "clef" && effect.configurationDigest !== undefined)).toBe(true); + const outputs: JsonObject = Object.create(null); + for (const [name, source] of Object.entries(manifest.interface!.outputs)) outputs[name] = receipt.cells[source.cell]!.outputs![source.port]!; + expect(gradeExperimentCase(task, batch, outputs).passed).toBe(true); + } + } +}); diff --git a/src/jev.ts b/src/jev.ts index 51c6eaf0..0ebd7f18 100644 --- a/src/jev.ts +++ b/src/jev.ts @@ -115,7 +115,9 @@ export function jevAsker(options: JevAskerOptions): DecisionAsker { state: DecisionState, questions: DecisionQuestions, signal?: AbortSignal, + images?: unknown, ): Promise { + if (images !== undefined) throw new AlgalError("PARSE_FAILED", "Legacy Jev does not support embedded images; use Clef"); if (signal?.aborted) throw new AlgalError("BUDGET_EXHAUSTED", "Jev request cancelled before dispatch"); const token = await key(); if (signal?.aborted) throw new AlgalError("BUDGET_EXHAUSTED", "Jev request cancelled before dispatch"); From 044ebc7ab503d017ba77f34046d06ba51a0e4042 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sun, 4 Oct 2026 23:43:53 -0400 Subject: [PATCH 2/2] Isolate browser bundle build in an owned child process The shard run wedges the bundler's process-global workers after child- spawning test files on the Linux runner; build in a spawned child with a SIGKILL timeout, matching the portable-application test's isolation. --- src/source-browser.test.ts | 50 ++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/source-browser.test.ts b/src/source-browser.test.ts index 30ff673b..2a9bcc35 100644 --- a/src/source-browser.test.ts +++ b/src/source-browser.test.ts @@ -16,17 +16,40 @@ const repository = join(import.meta.dir, ".."); // This consumer exists only in memory. It imports the compiler by package // name, so the bundle resolves the package.json export map as a host would. const consumer = join(repository, "browser-consumer.ts"); -let bundled: Promise | undefined; -function bundle(): Promise { - bundled ??= Bun.build({ - entrypoints: [consumer], - files: { [consumer]: [ - 'export * from "@hraness/algal/source";', - 'export * from "@hraness/algal/source-errors";', - 'export { setExprExports } from "@hraness/algal/expr";', - ].join("\n") }, - target: "browser", format: "esm", metafile: true, throw: false, - }); +interface Bundled { + logs: { level: string; message: string }[]; + success: boolean; + metafile?: { inputs: Record }; + output?: string; +} +// Keep the bundler's process-global workers out of the test process: the build +// runs in an owned child so a wedged worker fails on the timeout instead of +// stalling the shard. The same isolation as the portable-application test. +let bundled: Promise | undefined; +function bundle(): Promise { + bundled ??= (async () => { + const child = Bun.spawn([process.execPath, "--eval", ` + const consumer = ${JSON.stringify(consumer)}; + const result = await Bun.build({ + entrypoints: [consumer], + files: { [consumer]: [ + 'export * from "@hraness/algal/source";', + 'export * from "@hraness/algal/source-errors";', + 'export { setExprExports } from "@hraness/algal/expr";', + ].join("\\n") }, + target: "browser", format: "esm", metafile: true, throw: false, + }); + console.log(JSON.stringify({ + logs: result.logs.map(log => ({ level: log.level, message: log.message })), + success: result.success, + metafile: result.metafile, + output: result.outputs[0] ? await result.outputs[0].text() : undefined, + })); + `], { cwd: repository, stdin: "ignore", stdout: "pipe", stderr: "pipe", timeout: 30_000, killSignal: "SIGKILL" }); + const [exitCode, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); + if (exitCode !== 0) throw new Error(`browser bundle child exited ${exitCode}: ${stderr}`); + return JSON.parse(stdout) as Bundled; + })(); return bundled; } @@ -35,10 +58,11 @@ function bundle(): Promise { * and workers also provide. */ async function browserRealm(): Promise<{ host: BrowserHost; context: vm.Context }> { const result = await bundle(); - expect(result.outputs).toHaveLength(1); + expect(result.success).toBe(true); + expect(typeof result.output).toBe("string"); const context = vm.createContext({ TextDecoder, TextEncoder, URL, structuredClone }); const url = "https://example.invalid/app.js"; - const module = new vm.SourceTextModule(await result.outputs[0]!.text(), { context, identifier: url, initializeImportMeta: meta => { meta.url = url; } }); + const module = new vm.SourceTextModule(result.output!, { context, identifier: url, initializeImportMeta: meta => { meta.url = url; } }); await module.link(specifier => { throw new Error(`the browser bundle imports ${specifier}`); }); await module.evaluate(); return { host: module.namespace as BrowserHost, context };