diff --git a/Cargo.lock b/Cargo.lock index 6c947707..e863b01f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -328,6 +328,18 @@ version = "0.1.0" name = "gatos-policy" version = "0.1.0" +[[package]] +name = "gatos-privacy" +version = "0.1.0" +dependencies = [ + "anyhow", + "blake3", + "hex", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "gatos-wasm-bindings" version = "0.1.0" @@ -939,6 +951,26 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.1.9" diff --git a/Cargo.toml b/Cargo.toml index 67934114..9279a17a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/gatos-mind", "crates/gatos-echo", "crates/gatos-policy", + "crates/gatos-privacy", "crates/gatos-kv", "crates/gatosd", "bindings/wasm", diff --git a/Makefile b/Makefile index 5259946a..68acc3f3 100644 --- a/Makefile +++ b/Makefile @@ -12,14 +12,14 @@ diagrams: @bash -lc 'scripts/mermaid/generate_all.sh' lint-md: - @bash -lc 'if command -v node >/dev/null 2>&1; then \ + @bash -lc 'if command -v node >/dev/null 2>&1; then \ npx -y markdownlint-cli "**/*.md" --config .markdownlint.json; \ elif command -v docker >/dev/null 2>&1; then \ docker run --rm -v "$$PWD:/work" -w /work node:20 bash -lc "npx -y markdownlint-cli \"**/*.md\" --config .markdownlint.json"; \ else echo "Need Node.js or Docker" >&2; exit 1; fi' fix-md: - @bash -lc 'if command -v node >/dev/null 2>&1; then \ + @bash -lc 'if command -v node >/dev/null 2>&1; then \ npx -y markdownlint-cli "**/*.md" --fix --config .markdownlint.json; \ elif command -v docker >/dev/null 2>&1; then \ docker run --rm -v "$$PWD:/work" -w /work node:20 bash -lc "npx -y markdownlint-cli \"**/*.md\" --fix --config .markdownlint.json"; \ @@ -36,28 +36,55 @@ schema-compile: @bash -lc 'set -euo pipefail; \ if ! command -v node >/dev/null 2>&1; then \ echo "Node.js required (or run in CI)" >&2; exit 1; fi; \ - npx -y ajv-cli@5 ajv compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/job/job_manifest.schema.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/job/proof_of_execution_envelope.schema.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/proposal.schema.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/approval.schema.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/grant.schema.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/revocation.schema.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/proof_of_consensus_envelope.schema.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/policy/governance_policy.schema.json' + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/job/job_manifest.schema.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/job/proof_of_execution_envelope.schema.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/proposal.schema.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/approval.schema.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/grant.schema.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/revocation.schema.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/proof_of_consensus_envelope.schema.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/policy/governance_policy.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/privacy/opaque_pointer.schema.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/shiplog/event_envelope.schema.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/shiplog/consumer_checkpoint.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/shiplog/deployment_trailer.schema.json && \ + npx -y ajv-cli@5 compile --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/shiplog/anchor.schema.json' schema-validate: @bash -lc 'set -euo pipefail; \ if ! command -v node >/dev/null 2>&1; then \ echo "Node.js required (or run in CI)" >&2; exit 1; fi; \ - npx -y ajv-cli@5 ajv validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/job/job_manifest.schema.json -d examples/v1/job/manifest_min.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/job/proof_of_execution_envelope.schema.json -d examples/v1/job/poe_min.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/proposal.schema.json -d examples/v1/governance/proposal_min.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/approval.schema.json -d examples/v1/governance/approval_min.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/grant.schema.json -d examples/v1/governance/grant_min.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/revocation.schema.json -d examples/v1/governance/revocation_min.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/proof_of_consensus_envelope.schema.json -d examples/v1/governance/poc_envelope_min.json -r schemas/v1/common/ids.schema.json && \ - npx -y ajv-cli@5 ajv validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/policy/governance_policy.schema.json -d examples/v1/policy/governance_min.json' + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/job/job_manifest.schema.json -d examples/v1/job/manifest_min.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/job/proof_of_execution_envelope.schema.json -d examples/v1/job/poe_min.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/proposal.schema.json -d examples/v1/governance/proposal_min.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/approval.schema.json -d examples/v1/governance/approval_min.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/grant.schema.json -d examples/v1/governance/grant_min.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/revocation.schema.json -d examples/v1/governance/revocation_min.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/governance/proof_of_consensus_envelope.schema.json -d examples/v1/governance/poc_envelope_min.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/policy/governance_policy.schema.json -d examples/v1/policy/governance_min.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/privacy/opaque_pointer.schema.json -d examples/v1/privacy/opaque_pointer_min.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/shiplog/event_envelope.schema.json -d examples/v1/shiplog/event_min.json -r schemas/v1/common/ids.schema.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/shiplog/consumer_checkpoint.schema.json -d examples/v1/shiplog/checkpoint_min.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/shiplog/deployment_trailer.schema.json -d examples/v1/shiplog/trailer_min.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/shiplog/anchor.schema.json -d examples/v1/shiplog/anchor_min.json && \ + npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/privacy/opaque_pointer.schema.json -d examples/v1/privacy/pointer_low_entropy_min.json' + +schema-negative: + @bash -lc 'set -euo pipefail; \ + if ! command -v node >/dev/null 2>&1; then \ + echo "Node.js required (or run in CI)" >&2; exit 1; fi; \ + # Negative: checkpoint requires both fields + ! npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/shiplog/consumer_checkpoint.schema.json -d examples/v1/shiplog/checkpoint_ulid_only_invalid.json; \ + ! npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/shiplog/consumer_checkpoint.schema.json -d examples/v1/shiplog/checkpoint_commit_only_invalid.json; \ + # Negative: low-entropy pointer must not allow plaintext digest + ! npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/privacy/opaque_pointer.schema.json -d examples/v1/privacy/pointer_low_entropy_invalid.json' + +.PHONY: kill-check +kill-check: + @bash -lc 'scripts/killcheck/schema_headers.sh' + @bash -lc 'scripts/killcheck/ulid_reference.sh' + @bash -lc 'scripts/killcheck/error_casing.sh' schema-negative: @bash -lc 'set -euo pipefail; \ @@ -65,9 +92,9 @@ schema-negative: echo "Node.js required (or run in CI)" >&2; exit 1; fi; \ echo "{\"governance\":{\"x\":{\"ttl\":\"P\"}}}" > /tmp/bad1.json; \ echo "{\"governance\":{\"x\":{\"ttl\":\"PT\"}}}" > /tmp/bad2.json; \ - if npx -y ajv-cli@5 ajv validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/policy/governance_policy.schema.json -d /tmp/bad1.json; then \ + if npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/policy/governance_policy.schema.json -d /tmp/bad1.json; then \ echo "Should have rejected ttl=P" >&2; exit 1; else echo "Rejected ttl=P as expected"; fi; \ - if npx -y ajv-cli@5 ajv validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/policy/governance_policy.schema.json -d /tmp/bad2.json; then \ + if npx -y ajv-cli@5 validate --spec=draft2020 --strict=true -c ajv-formats -s schemas/v1/policy/governance_policy.schema.json -d /tmp/bad2.json; then \ echo "Should have rejected ttl=PT" >&2; exit 1; else echo "Rejected ttl=PT as expected"; fi' schemas: schema-compile schema-validate schema-negative diff --git a/crates/gatos-privacy/Cargo.toml b/crates/gatos-privacy/Cargo.toml new file mode 100644 index 00000000..f2b77003 --- /dev/null +++ b/crates/gatos-privacy/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "gatos-privacy" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +blake3 = { workspace = true } +hex = { workspace = true } +anyhow = { workspace = true } +thiserror = "1" diff --git a/crates/gatos-privacy/README.md b/crates/gatos-privacy/README.md new file mode 100644 index 00000000..8bfd99ad --- /dev/null +++ b/crates/gatos-privacy/README.md @@ -0,0 +1,33 @@ +# gatos-privacy + +Opaque Pointer types and helpers for the GATOS hybrid privacy model (ADR-0004). + +Key types +- `OpaquePointer`: JSON-facing struct that mirrors `schemas/v1/privacy/opaque_pointer.schema.json`. + - `digest: Option` — plaintext digest (may be omitted) + - `ciphertext_digest: Option` — ciphertext digest + - `extensions.class = "low-entropy"` implies `ciphertext_digest` MUST be present and `digest` MUST be absent. + +- `VerifiedOpaquePointer`: wrapper that enforces invariants during deserialization. + - Use this at trust boundaries to guarantee the low-entropy rules. + +Validation +- After deserializing `OpaquePointer`, call `pointer.validate()` to enforce: + - At least one of `digest` or `ciphertext_digest` is present. + - Low-entropy class requires `ciphertext_digest` and forbids `digest`. + +Examples +```rust +use gatos_privacy::{OpaquePointer, VerifiedOpaquePointer}; + +// 1) Verified wrapper enforces invariants automatically +let v: VerifiedOpaquePointer = serde_json::from_str(json)?; + +// 2) Manual validation on the plain struct +let p: OpaquePointer = serde_json::from_str(json)?; +p.validate()?; +``` + +Canonicalization +- When computing content IDs or digests, serialize JSON with RFC 8785 JCS (performed by higher layers). + diff --git a/crates/gatos-privacy/src/lib.rs b/crates/gatos-privacy/src/lib.rs new file mode 100644 index 00000000..3e7ae3f3 --- /dev/null +++ b/crates/gatos-privacy/src/lib.rs @@ -0,0 +1,110 @@ +//! gatos-privacy — Opaque Pointer types and helpers +//! +//! This crate defines the JSON-facing pointer envelope used by the +//! hybrid privacy model (ADR-0004). The struct mirrors the v1 schema +//! in `schemas/v1/privacy/opaque_pointer.schema.json`. +//! +//! Canonicalization: when computing content IDs or digests, callers +//! MUST serialize JSON using RFC 8785 JCS. This crate intentionally +//! does not take a dependency on a specific JCS implementation to +//! keep the workspace lean; higher layers may provide one. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OpaquePointer { + pub kind: Kind, + pub algo: Algo, + #[serde(skip_serializing_if = "Option::is_none")] + pub digest: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ciphertext_digest: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + pub location: String, + pub capability: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub extensions: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Kind { + OpaquePointer, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Algo { + Blake3, +} + +impl OpaquePointer { + /// Validate invariants beyond serde schema mapping. + pub fn validate(&self) -> Result<(), PointerError> { + let has_plain = self.digest.as_ref().map(|s| !s.is_empty()).unwrap_or(false); + let has_cipher = self + .ciphertext_digest + .as_ref() + .map(|s| !s.is_empty()) + .unwrap_or(false); + if !(has_plain || has_cipher) { + return Err(PointerError::MissingDigest); + } + let low_entropy = self + .extensions + .as_ref() + .and_then(|v| v.get("class")) + .and_then(|c| c.as_str()) + .map(|s| s == "low-entropy") + .unwrap_or(false); + if low_entropy { + if !has_cipher { + return Err(PointerError::LowEntropyNeedsCiphertextDigest); + } + if has_plain { + return Err(PointerError::LowEntropyForbidsPlainDigest); + } + } + Ok(()) + } +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum PointerError { + #[error("at least one of digest or ciphertext_digest is required")] + MissingDigest, + #[error("low-entropy class requires ciphertext_digest")] + LowEntropyNeedsCiphertextDigest, + #[error("low-entropy class forbids plaintext digest")] + LowEntropyForbidsPlainDigest, +} + +/// A validated wrapper that enforces `OpaquePointer::validate()` during +/// deserialization. Prefer this type when accepting pointers from untrusted +/// inputs; it guarantees schema-level invariants at the boundary. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(transparent)] +pub struct VerifiedOpaquePointer(pub OpaquePointer); + +impl<'de> Deserialize<'de> for VerifiedOpaquePointer { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let inner = OpaquePointer::deserialize(deserializer)?; + inner + .validate() + .map_err(serde::de::Error::custom)?; + Ok(Self(inner)) + } +} + +impl core::ops::Deref for VerifiedOpaquePointer { + type Target = OpaquePointer; + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/crates/gatos-privacy/tests/pointer_schema.rs b/crates/gatos-privacy/tests/pointer_schema.rs new file mode 100644 index 00000000..077b01a4 --- /dev/null +++ b/crates/gatos-privacy/tests/pointer_schema.rs @@ -0,0 +1,23 @@ +use gatos_privacy::OpaquePointer; + +fn read_example(rel: &str) -> String { + let dir = env!("CARGO_MANIFEST_DIR"); + std::fs::read_to_string(format!("{}/../../examples/v1/{}", dir, rel)).unwrap() +} + +#[test] +fn ciphertext_only_pointer_should_deserialize() { + // This example omits plaintext digest by design (low-entropy class) + let json = read_example("privacy/pointer_low_entropy_min.json"); + let ptr: Result = serde_json::from_str(&json); + assert!(ptr.is_ok(), "ciphertext-only opaque pointer must deserialize"); +} + +#[test] +fn both_digests_allowed_when_not_low_entropy() { + let json = read_example("privacy/opaque_pointer_min.json"); + let ptr: OpaquePointer = serde_json::from_str(&json).unwrap(); + let has_digest = ptr.digest.as_ref().map(|s| !s.is_empty()).unwrap_or(false); + assert!(has_digest); + assert!(ptr.ciphertext_digest.is_some()); +} diff --git a/crates/gatos-privacy/tests/verified.rs b/crates/gatos-privacy/tests/verified.rs new file mode 100644 index 00000000..b0343aa1 --- /dev/null +++ b/crates/gatos-privacy/tests/verified.rs @@ -0,0 +1,22 @@ +use gatos_privacy::{OpaquePointer, VerifiedOpaquePointer}; + +fn read_example(rel: &str) -> String { + let dir = env!("CARGO_MANIFEST_DIR"); + std::fs::read_to_string(format!("{}/../../examples/v1/{}", dir, rel)).unwrap() +} + +#[test] +fn verified_accepts_ciphertext_only_low_entropy() { + let json = read_example("privacy/pointer_low_entropy_min.json"); + let v: VerifiedOpaquePointer = serde_json::from_str(&json).expect("verified deserialize"); + assert!(v.ciphertext_digest.is_some()); + assert!(v.digest.is_none()); +} + +#[test] +fn verified_rejects_low_entropy_with_plain_digest() { + let json = read_example("privacy/pointer_low_entropy_invalid.json"); + let v: Result = serde_json::from_str(&json); + assert!(v.is_err(), "should reject invalid low-entropy pointer"); +} + diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 9b655fc2..fafd7571 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -141,29 +141,6 @@ Each feature includes user stories per relevant stakeholders (format requested), --- -## F6 — Opaque Pointers & CAS - -### F6-US-DML - -| | | -|--|--| -| **As a...** | Data/ML Engineer | -| **I want..** | encrypted artifacts with verifiable pointers | -| **So that...** | I can ship models across untrusted storage | - -#### Acceptance Criteria - -- [ ] Pointer includes plaintext hash, ciphertext hash, cipher meta -- [ ] Rekey operation available - -#### Test Plan - -- [ ] Golden: decrypt with correct key → match plaintext hash -- [ ] Edge: wrong bytes → hash mismatch -- [ ] Failure: rekey without authorization → deny - ---- - ## F7 — Epochs & Compaction ### F7-US-PENG @@ -208,3 +185,36 @@ Each feature includes user stories per relevant stakeholders (format requested), - [ ] Golden: metrics show non-zero counters post workload - [ ] Edge: cache stale → doctor recommends rebuild - [ ] Failure: FF-only violation → doctor flags critical + +--- + +## F9 — Hybrid Privacy Model + +See also: [ADR-0004](./decisions/ADR-0004/DECISION.md). + +### F9-US-DEV + +### F9-US-SEC + +| | | +|--|--| +| **As a...** | Security/Compliance | +| **I want..** | to audit the separation of public and private data | +| **So that...** | I can verify that sensitive data is properly isolated and access is controlled | + +#### Acceptance Criteria + +- [ ] Opaque Pointer resolution fails without a valid capability. +- [ ] Private blob digest matches the digest in the public pointer. +- [ ] Commit trailers (`Privacy-Redactions`, `Privacy-Pointers`) accurately report the number of redactions/pointers. + +#### Test Plan + +- [ ] Golden: project a unified state, resolve pointer, and verify content matches original. +- [ ] Edge: attempt to resolve a pointer with an invalid capability URI → DENY. +- [ ] Failure: tamper with a private blob → digest mismatch on resolution. + +## F6 — Privacy Opaque Pointers (ADR‑0004) + +- See ADR‑0004 for the normative pointer envelope and privacy projection rules. +- Acceptance: pointers validate against schema; low‑entropy public pointers hide plaintext `digest` and include `ciphertext_digest`. diff --git a/docs/SPEC.md b/docs/SPEC.md index 041cf0bd..783a4df4 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -71,7 +71,7 @@ graph TD end subgraph "Job Plane" - Compute("gatos-compute"); + Compute("gatos-compute (planned)"); end subgraph "Ledger Plane" @@ -128,6 +128,7 @@ graph TD A1 --> B6(audit) A1 --> B7(cache) A1 --> B8(epoch) + A1 --> B9(private) C(notes) --> C1(gatos) end subgraph Workspace @@ -148,6 +149,8 @@ The normative layout is as follows: │ └── gatos/ │ ├── journal/ │ ├── state/ +│ ├── private/ +│ │ └── / # e.g., the actor's ed25519 public key │ ├── mbus/ │ ├── mbus-ack/ │ ├── jobs/ @@ -282,28 +285,87 @@ On **DENY**, the gate **MUST** append an audit decision to `refs/gatos/audit/pol --- -## 7. Blob Pointers & Opaque Storage +## 7. Privacy and Opaque Pointers -Large or sensitive data is stored out-of-band in a content-addressed store and referenced via pointers. +See also: [ADR‑0004](./decisions/ADR-0004/DECISION.md). + +GATOS supports a hybrid privacy model where state can be separated into a verifiable public projection and a confidential private overlay. This is achieved by applying a deterministic **Projection Functor** during the state fold process, which replaces sensitive or large data with **Opaque Pointers**. + +### 7.1 Projection Model + +The State Engine (`gatos-echo`) can be configured with privacy rules. When folding history, it first computes a `UnifiedState` containing all data. It then applies the privacy rules to produce a `PublicState` and a set of `PrivateBlobs`. + +- **`PublicState`**: Contains only public data and Opaque Pointers. This is committed to the public `refs/gatos/state/public/...` namespace and is globally verifiable. +- **`PrivateBlobs`**: The raw data that was redacted or pointerized. This data is stored in a separate, private store (e.g., a local directory, a private object store) and is addressed by its content hash. + +Any commit that is the result of a privacy projection **MUST** include trailers indicating the number of redactions and pointers created. + +```text +Privacy-Redactions: 5 +Privacy-Pointers: 2 +``` + +### 7.2 Opaque Pointers + +An Opaque Pointer is a canonical JSON object that acts as a verifiable, addressable link to a private blob. It replaces the sensitive data in the `PublicState`. ```mermaid classDiagram - class BlobPointer { - +String kind: "blobptr" - +String algo - +String hash - +Number size - } class OpaquePointer { - +String kind: "opaque" - +String algo - +String hash - +String ciphertext_hash - +Object cipher_meta + +string kind: "opaque_pointer" + +string algo: "blake3" + +string digest: "blake3:" // plaintext digest + +string ciphertext_digest: "blake3:" // optional + +int size // bytes; SHOULD be present + +string location + +string capability // MUST NOT embed secrets + +object extensions // forward-compatible } ``` -Pointers **MUST** refer to bytes in `gatos/objects//`. For opaque objects, no plaintext **MAY** be stored in Git. +- `digest`: The **REQUIRED** `blake3` hash of the plaintext. For low‑entropy privacy classes, the public pointer MUST NOT expose this value. +- `ciphertext_digest`: The `blake3` hash of the stored ciphertext. For low‑entropy privacy classes, this field MUST be present in the public pointer. +- `size`: The size of the private blob in bytes (RECOMMENDED). +- `location`: A **REQUIRED** stable URI indicating where the blob can be fetched (e.g., `gatos-node://ed25519:`, `s3://bucket/key`). Do not embed pre‑signed tokens. +- `capability`: A **REQUIRED** reference to the authn/z + decryption mechanism (e.g., `gatos-key://...`, `kms://...`). It MUST NOT embed secrets; resolution occurs at the policy layer. + +The pointer itself is canonicalized via RFC 8785 JCS and its `content_id` is `blake3(JCS(pointer_json))`. + +### 7.3 Pointer Resolution + +Endpoint and AuthN: + +- Clients MUST resolve via `POST /gatos/private/blobs/resolve` with body `{ "digest": "blake3:", "want": "plaintext"|"ciphertext" }` and `Authorization: Bearer `. +- Tokens MUST include standard claims (`sub`, `aud`, `method`, `path`, `exp`, `nbf`); skew tolerance ±300s. 401 for authn failures; 403 for policy denials. + +Verification Steps: + +1. Fetch the ciphertext blob from `location` via the node’s resolver endpoint. +2. Acquire the necessary keys via the `capability` reference (policy-driven; no secrets in the pointer). +3. Decrypt. Compute `blake3(ciphertext)` and compare with `ciphertext_digest` when present; compute `blake3(plaintext)` and compare with `digest` when exposed. Any mismatch MUST yield `DigestMismatch`. +4. Servers SHOULD return `X-BLAKE3-Digest` and `Digest: sha-256=…` headers for response integrity. + +Error Taxonomy: + +- `Unauthorized` (401), `Forbidden` (403), `NotFound` (404), `DigestMismatch` (422), `CapabilityUnavailable` (503), `PolicyDenied` (403). + +Optional HTTP Message Signatures profile (RFC 9421): + +- As an alternative to JWT, clients MAY sign `@method`, `@target-uri`, `date`, `host`, `content-digest` and send `Signature-Input`/`Signature` headers. Servers SHOULD still emit `Digest` and `X-BLAKE3-Digest` response headers. + +Pointer Rotation (Rekey): + +1) fetch ciphertext; 2) decrypt; 3) re‑encrypt per new capability; 4) store new ciphertext; 5) emit rotation event updating pointer fields (capability/location). `digest` (plaintext) MUST remain stable. Add trailer `Privacy-Pointer-Rotations: `. + +Namespacing: + +- `refs/gatos/private//…` holds private overlay indices/metadata only; workspace mirror is `gatos/private//…`. Blobs live in external stores keyed by digest. + +Canonicalization: + +- All JSON labeled as canonical MUST use RFC 8785 JCS; non‑JSON maps MUST be ordered lexicographically by lowercase UTF‑8 keys. + +This process guarantees that even though the data is stored privately, its integrity is verifiable against the public ledger. --- @@ -624,7 +686,7 @@ Proposal → Approvals (N‑of‑M) → Grant. Quorum groups (e.g., `@leads`) MU Proposal-Id: blake3: Approval-Id: blake3: Signer: ed25519: - Expires-At: # OPTIONAL + Expires-At: # OPTIONAL. If present, the approval is only valid until this time. It cannot extend the proposal's expiration. ``` - Grant (at `refs/gatos/grants/…`): @@ -640,10 +702,10 @@ Proposal → Approvals (N‑of‑M) → Grant. Quorum groups (e.g., `@leads`) MU `Proof-Of-Consensus` is the BLAKE3 of a canonical JSON envelope containing: - The canonical proposal envelope (by value or `Proposal-Id`). -- A sorted list (by `Signer`) of all valid approvals used to reach quorum (by value or `Approval-Id`). +- A lexicographically sorted list of approvals ordered by the lowercase ASCII of each approval's `Signer` value (the `ed25519:` string). Each approval is included by value or via `Approval-Id`. - The governance rule id (`Policy-Rule`) and effective quorum parameters. -PoC envelope SHOULD be stored canonically under `refs/gatos/audit/proofs/governance/`; the Grant’s `Proof-Of-Consensus` trailer MUST equal `blake3(envelope_bytes)`. +PoC envelope MUST be stored canonically under `refs/gatos/audit/proofs/governance/`; the Grant’s `Proof-Of-Consensus` trailer MUST equal `blake3(envelope_bytes)`. ### 20.4 Lifecycle States diff --git a/docs/TECH-SPEC.md b/docs/TECH-SPEC.md index 4f991b54..a31c35c7 100644 --- a/docs/TECH-SPEC.md +++ b/docs/TECH-SPEC.md @@ -101,10 +101,10 @@ graph TD | `gatos-ledger-git` | `std`-dependent storage backend using `libgit2`. | | `gatos-ledger` | Composes ledger components via feature flags. | | `gatos-mind` | Asynchronous, commit-backed message bus (pub/sub). | -| `gatos-echo` | Deterministic state engine for processing events ("folds"). | -| `gatos-policy` | Deterministic policy engine for executing compiled rules and managing the Consensus Governance lifecycle. | -| `gatos-kv` | Git-backed key-value state cache. | -| `gatosd` | Main binary for the CLI and the JSONL RPC daemon. | +| `gatos-echo` | Deterministic state engine for processing events ("folds"). Privacy projection logic. | +| `gatos-policy` | Deterministic policy engine for executing compiled rules, managing Consensus Governance, and privacy rule evaluation. | +| `gatos-kv` | Git-backed key-value state cache, used for materializing and indexing queryable views of folded state. | +| `gatosd` | Main binary for the CLI, JSONL RPC daemon, and Opaque Pointer resolution endpoint. | | `gatos-compute` | Worker that discovers and executes jobs from the Job Plane. | | `gatos-wasm-bindings`| WASM bindings for browser and Node.js environments. | | `gatos-ffi-bindings` | C-compatible FFI for integration with other languages. | @@ -160,22 +160,78 @@ sequenceDiagram --- -## 6. Opaque Pointers +## 6. Privacy Projection and Resolution -The `rekey` command allows updating the encryption key for an opaque blob. +See also: [ADR‑0004](./decisions/ADR-0004/DECISION.md). + +The implementation of the hybrid privacy model involves a coordinated effort between the state, policy, and daemon components. + +### 6.1 Projection Implementation + +The projection from a `UnifiedState` to a `PublicState` is handled by `gatos-echo` with rules supplied by `gatos-policy`. ```mermaid sequenceDiagram - participant User - participant GATOS - - User->>GATOS: gatos blob rekey --to - GATOS->>GATOS: Create new Opaque Pointer - GATOS->>GATOS: Encrypt data with new pubkey - GATOS->>GATOS: Store new ciphertext in CAS - GATOS->>GATOS: Atomically update references + participant gatos-echo + participant gatos-policy + participant gatos-ledger + participant "StorageBackend (Interface)" + + Echo->>Echo: 1. Fold event history to produce UnifiedState + Echo->>Policy: 2. Request privacy rules for the current context + Policy-->>Echo: 3. Return `select` and `action` rules +loop for each field path in the UnifiedState tree + gatos-echo->>gatos-echo: 4. Match field path against rules + alt rule matches (e.g., "pointerize") + Echo->>Echo: 5. Generate Opaque Pointer envelope + Echo->>PrivateStore: 6. Store original node value as private blob, keyed by its blake3 digest + Echo->>Echo: 7. Replace node in state tree with pointer + end + end + Echo->>Ledger: 8. Commit the final PublicState tree ``` +The `PrivateStore` is a pluggable trait, allowing for backends like a local filesystem, S3, or another GATOS node. + +#### 6.1.1 Encryption Profile (Normative) + +- AEAD algorithm: XChaCha20-Poly1305. +- Nonces: 24-byte (192-bit) nonces MUST be unique per key. Prefer deterministic nonces derived from the pointer digest via HKDF (domain-separated) or a crash-safe monotonic per-key counter stored in KMS. Random nonces are permitted only with a documented collision budget and monitoring. +- AAD: MUST include the pointer digest, actor id, and effective policy version (or policy_root) to bind authorization context to bytes at rest. +- Reuse is catastrophic and MUST be proven impossible by construction. + +### 6.2 Resolution Implementation + +The `gatosd` daemon exposes a secure endpoint for resolving Opaque Pointers. + +- Endpoint: `POST /gatos/private/blobs/resolve` +- Content-Type: `application/json` +- Request body (JCS canonical JSON): + + ```json + { "digest": "blake3:", "want": "plaintext" } + ``` + + - `want` OPTIONAL: `"plaintext" | "ciphertext"` (default `"plaintext"`). +- Authentication: `Authorization: Bearer ` + - Claims (example): `iss`, `sub` (ed25519:), `aud` ("gatos-node:"), `exp`, `nbf`, `jti`, `method` ("POST"), `path` ("/gatos/private/blobs/resolve"), `digest` (MUST match body.digest). + - Clock skew tolerance: ±300 seconds. +- Authorization: Node evaluates policy for `` on ``. +- Response (200 OK): + - Headers: `Digest: sha-256=`, `X-BLAKE3-Digest: blake3:` + - Body: requested bytes (ciphertext or plaintext). + +Errors: 401 Unauthorized, 403 Forbidden, 404 Not Found, 422 DigestMismatch, 503 CapabilityUnavailable. + +Optional profile (HTTP Message Signatures, RFC 9421): + +- Clients MAY authenticate by signing components: `@method`, `@target-uri`, `date`, `host`, `content-digest` (SHA-256 over request body) and sending `Signature-Input: sig1=...` and `Signature: sig1=::`. +- Servers STILL apply policy and SHOULD return `Digest` and `X-BLAKE3-Digest` headers. + +Pointer Rotation (Rekey): + +- Implement a rotation that: (1) fetches; (2) decrypts; (3) re‑encrypts; (4) stores; (5) emits an audit event updating pointer fields while keeping plaintext `digest` stable. Add trailer `Privacy-Pointer-Rotations: ` when a projection commit includes rotations. + --- ## 7. JSONL Protocol @@ -187,7 +243,7 @@ sequenceDiagram participant Client as Client (SDK/CLI) participant Daemon as gatosd - Client->>Daemon: {"type":"append_event", "id":"01A", "ns":"...", "event":{...}} + Client->>Daemon: {"type":"append_event", "id":"01A", "topic":"...", "event":{...}} Daemon-->>Client: {"ok":true, "id":"01A", "commit_id":"..."} Client->>Daemon: {"type":"bus.subscribe", "id":"01C", "topic":"..."} @@ -236,8 +292,11 @@ graph TD C --> C1(Golden Vectors); C --> C2(Torture Tests); C --> C3(Reconcile Harness); + C --> C4(Projection Determinism); ``` +- **Projection Determinism**: Verifies that applying the same privacy policy to the same `UnifiedState` on different platforms (Linux, macOS, Windows) produces a byte-for-byte identical `PublicState` and the same set of private blobs. + --- ## 10. Security @@ -265,9 +324,9 @@ sequenceDiagram Examples ```json -{"type":"append_event","id":"01A","ns":"finance","event":{}} +{"type":"append_event","id":"01A","topic":"finance","event":{}} {"type":"bus.subscribe","id":"01C","topic":"gatos.jobs.pending"} -{"type":"fold_state","id":"01D","ns":"finance","channel":"table","spec":"folds/invoices.yaml"} +{"type":"fold_state","id":"01D","topic":"finance","channel":"table","spec":"folds/invoices.yaml"} {"type":"governance.proposal.new","id":"02A","action":"publish.artifact","target":"gatos://assets/model.bin","quorum":"2-of-3@leads"} {"type":"governance.approval.add","id":"02B","proposal":""} {"type":"governance.grant.verify","id":"02C","grant":""} @@ -281,7 +340,7 @@ Tuning batch size is a trade-off between latency and commit churn. ```mermaid xychart-beta - title "Batch Size Trade-off" + title "Batch Size Trade-off (Illustrative)" x-axis "Batch Size" y-axis "Metric" line "Latency" [50, 40, 35, 32, 30] diff --git a/docs/USE-CASES.md b/docs/USE-CASES.md index f3c52cb7..7b56a6d7 100644 --- a/docs/USE-CASES.md +++ b/docs/USE-CASES.md @@ -83,3 +83,13 @@ This document illustrates practical scenarios where GATOS provides unique value. |**Goal** | Signed toggles with audit and rollbacks. | | **How** | KV‑style events + index refs; push‑gate for enforcement. | | **Why GATOS** | Auditable configuration without a new database. | + +--- + +## 9) Verifiable, Compliant PII Management + +| | | +|---|---| +|**Goal** | Manage customer data (PII) in a way that is both auditable and privacy-preserving. | +| **How** | A privacy policy projects the unified state into a public state with PII replaced by Opaque Pointers. The private data lives in an actor-anchored, encrypted blob store. | +| **Why GATOS** | Provides a verifiable public audit trail ("a user's data was accessed") without ever exposing the private data ("the user's address is...") to the public ledger. Access is gated by cryptographic capabilities. | diff --git a/docs/decisions/ADR-0003/DECISION.md b/docs/decisions/ADR-0003/DECISION.md index 45f29beb..3e4683ee 100644 --- a/docs/decisions/ADR-0003/DECISION.md +++ b/docs/decisions/ADR-0003/DECISION.md @@ -86,10 +86,10 @@ Define a system for gating specific GATOS actions (e.g., locking a file, publish 7. Proof‑Of‑Consensus (normative) - The `Proof-Of-Consensus` digest MUST be the BLAKE3 of a canonical envelope that includes (see schema: [`schemas/v1/governance/proof_of_consensus_envelope.schema.json`](../../../schemas/v1/governance/proof_of_consensus_envelope.schema.json)): - The canonical proposal envelope (by value or by `Proposal-Id`). - - A sorted list (by `Signer`) of all valid approvals used to reach quorum (each by value or `Approval-Id`). + - A lexicographically sorted list of approvals by the lowercase ASCII of each approval's `Signer` value (the `ed25519:` string). Each approval is included by value or via `Approval-Id`. - The governance rule id (`Policy-Rule`) and effective quorum parameters. - Implementations MUST use canonical JSON (UTF‑8, sorted keys, no insignificant whitespace) to build this envelope before hashing. All hex encodings MUST be lowercase. Ordering by signer is an application‑level MUST; JSON Schema cannot enforce sort order. - - Storage: The canonical PoC envelope JSON SHOULD be persisted as a blob referenced under `refs/gatos/audit/proofs/governance/`; the `Proof-Of-Consensus` trailer MUST equal `blake3(envelope_bytes)`. + - Storage: The canonical PoC envelope JSON MUST be persisted as a blob referenced under `refs/gatos/audit/proofs/governance/`; the `Proof-Of-Consensus` trailer MUST equal `blake3(envelope_bytes)`. 8. Governance schema (policy integration) - Extend `.gatos/policy.yaml` to declare governance rules (JSON Schema: [`schemas/v1/policy/governance_policy.schema.json`](../../../schemas/v1/policy/governance_policy.schema.json)): diff --git a/docs/decisions/ADR-0004/DECISION.md b/docs/decisions/ADR-0004/DECISION.md new file mode 100644 index 00000000..c3611072 --- /dev/null +++ b/docs/decisions/ADR-0004/DECISION.md @@ -0,0 +1,312 @@ +--- +Status: Accepted +Date: 2025-11-10 +ADR: ADR-0004 +Authors: [flyingrobots, gemini-agent] +Requires: [ADR-0001] +Related: [ADR-0002, ADR-0003] +Tags: [Privacy, Projection, Opaque Pointers, Morphology Calculus] +Schemas: + - schemas/v1/privacy/opaque_pointer.schema.json +--- + +# ADR‑0004: Hybrid Privacy Model (Public Projection + Private Overlay) + +## Scope + +This ADR defines a **hybrid privacy model** for the GATOS operating surface. It formalizes the separation of state into a public, verifiable component and a private, actor-anchored overlay. This is achieved by introducing a **Projection Functor** that transforms a unified state into a public projection, leaving sensitive data in a private store referenced by **Opaque Pointers**. + +## Rationale + +GATOS's core value proposition is its verifiable, deterministic public ledger. However, many real-world applications require storing sensitive or large data (PII, secrets, large binaries) without committing it to the public history. The previous ad-hoc approach of using local, out-of-repo storage lacks the formal guarantees required by the GATOS Morphology Calculus. + +This ADR makes the hybrid model **normative, deterministic, and provable**. It ensures that public state remains globally verifiable while private data is securely addressable, auditable, and tied to the GATOS identity and policy model. + +## Mathematical Foundation (Morphology Calculus) + +This model is a direct application of the GATOS Morphology Calculus. + +1. **Shape Categories**: We define three categories of shapes: + * `Sh_Unified`: The category of shapes containing both public and private data. + * `Sh_Public`: The category of shapes containing only public data and opaque pointers. + * `Sh_Private`: The category of shapes containing only the private data blobs. + +2. **Projection as a Functor**: The privacy model is implemented as a functor, `Proj`, which maps shapes and morphisms from the unified category to the public category. + `Proj: Sh_Unified -> Sh_Public` + + This functor applies the privacy policy rules (`redact`, `pointerize`) to transform a unified shape into its public projection. The private data is extracted into `Sh_Private` during this process. + + ```mermaid + graph TD + subgraph Sh_Unified + U1("Unified Shape 1") + U2("Unified Shape 2") + U1 -- "Commit c" --> U2 + end + + subgraph Sh_Public + P1("Public Shape 1") + P2("Public Shape 2") + P1 -- "Proj(c)" --> P2 + end + + subgraph Sh_Private + B1("Private Blobs 1") + B2("Private Blobs 2") + end + + U1 -- "Proj" --> P1 + U2 -- "Proj" --> P2 + + U1 -- "Extract" --> B1 + U2 -- "Extract" --> B2 + + style P1 fill:#cde,stroke:#333 + style P2 fill:#cde,stroke:#333 + ``` + +This ensures that the transformation is structure-preserving and that the public history remains a valid, deterministic projection of the complete history. + +## Decision + +### 1. Actor-Anchored Private Namespace (Normative) + +Private data overlays are fundamentally tied to an actor's identity, not an ephemeral session. This anchors private data within the GATOS trust graph. + +* **Actor ID:** The canonical identifier for an actor, e.g., `ed25519:`. +* **Private Refs:** Private data is stored under refs namespaced by the actor ID. + + ``` + refs/gatos/private/// + ``` + +### 2. Encryption Algorithm & Nonce Discipline (Normative) + +* AEAD algorithm: Implementations MUST use XChaCha20-Poly1305 for encrypting private blobs referenced by Opaque Pointers. This is the only normative algorithm for the privacy overlay. It supersedes any prior mentions of AES‑256‑GCM as a default. +* Nonces: 24-byte (192-bit) nonces MUST be unique per key. Implementations SHOULD use deterministic, domain‑separated nonces derived via HKDF from the pointer digest and context, or a crash‑safe, monotonic per‑key counter stored in KMS. Random nonces are permitted only with a documented collision budget and active monitoring. +* Catastrophic reuse: Nonce reuse under the same key is catastrophic and MUST be proven impossible by construction (deterministic derivation) or by counter invariants. +* AAD binding: AEAD AAD MUST bind the pointer digest (not a separate content_id), the requester actor id, and the effective policy version so that verifiers can validate context and detect misuse. +* **Public Refs:** The corresponding public projection lives in the main state namespace. + + ``` + refs/gatos/state/public// + ``` + +### 2. Opaque Pointers (Normative) + +When private data is elided from the `PublicState`, a canonical JSON **Opaque Pointer** envelope is inserted in its place. + +```mermaid +classDiagram + class OpaquePointer { + +string kind: "opaque_pointer" + +string algo: "blake3" + +string digest: "blake3:" // plaintext digest + +string ciphertext_digest: "blake3:" // MAY be present + +int size // SHOULD be present (bytes) + +string location + +string capability // MUST NOT embed secrets + +object extensions // forward-compatible + } +``` + +* **`digest`**: The content-address of the private plaintext (`blake3(plaintext_bytes)`). This is the immutable link between the public and private worlds. +* **`ciphertext_digest`**: The content-address of the stored ciphertext (`blake3(ciphertext_bytes)`). For low‑entropy privacy classes (see Policy Hooks), the public pointer **MUST** include `ciphertext_digest` and policy **MUST NOT** expose the plaintext digest publicly. +* **`location`**: A URI indicating where to resolve the blob. Supported schemes include: + * `gatos-node://ed25519:`: Resolve via the GATOS trust graph. + * `https://...`, `s3://...`, `ipfs://...`: Standard distributed storage. + * `file:///...`: For local development and testing. +* **`capability`**: A reference identifying the authorization and decryption mechanism required to access the blob. It **MUST NOT** embed secrets or pre‑signed tokens. It SHOULD be a stable identifier (e.g., `gatos-key://v1/aes-256-gcm/` or `kms://...`) that can be resolved privately at the policy layer. + * Pointers MAY publish a non‑sensitive label and keep resolver details private via policy. Implementations MAY also place auxiliary hints inside `extensions`. + +The canonical `content_id` of the pointer itself is `blake3(JCS(pointer_json))`, where `JCS(…)` denotes RFC 8785 JSON Canonicalization Scheme applied to UTF‑8 bytes. This rule is normative for all canonical JSON in GATOS (pointers, governance envelopes, any JSON state snapshots). + +**Schema:** `schemas/v1/privacy/opaque_pointer.schema.json` + +#### AAD Components (Example) + +When encrypting a private blob referenced by an Opaque Pointer, the AEAD AAD MUST bind all of the following, in order, as UTF‑8 bytes: + +```text +1) pointer.digest +2) requester.actor_id (e.g., user:alice | service:policy) +3) policy.version (e.g., v1.2.3) +``` + +Implementations MAY structure these as concatenated bytes with clear domain separation (e.g., length‑prefixing) prior to supplying them as AAD to the AEAD algorithm. + +> Implementation note (non‑normative): When ingesting Opaque Pointers from untrusted JSON, implementations SHOULD validate the invariants at parse time (e.g., verify that low‑entropy pointers include `ciphertext_digest` and omit `digest`). In this repository, the `gatos-privacy` crate exposes `VerifiedOpaquePointer` which enforces these rules during deserialization; alternatively, callers can deserialize `OpaquePointer` and invoke `validate()` explicitly. + +### 3. The Projection Function (Normative) + +The State Engine (`gatos-echo`) is responsible for executing the projection. + +1. It computes a **UnifiedState** by folding the complete event history. +2. It consults the **Privacy Policy** (`.gatos/policy.yaml`). +3. It traverses the `UnifiedState` tree, applying `redact` or `pointerize` rules. + * `redact`: The field is removed from the public state. + * `pointerize`: The field's value is stored as a private blob, and an Opaque Pointer is substituted in the public state. +4. The resulting `PublicState` is committed to the public refs, and the `Private Blobs` are persisted to their specified `location`. + +Determinism Requirements: + +* All JSON artifacts produced during projection (including Opaque Pointers) MUST be canonicalized with RFC 8785 JCS prior to hashing. +* When non‑JSON maps are materialized (e.g., Git tree entries), keys MUST be ordered lexicographically by their lowercase UTF‑8 bytes. + +```mermaid +sequenceDiagram + participant E as State Engine (gatos-echo) + participant Pol as Policy Engine + participant L as Ledger (Git) + participant PS as Private Store + + E->>E: 1. Fold history into UnifiedState + E->>Pol: 2. Fetch privacy rules + Pol-->>E: 3. Return rules (redact/pointerize) + E->>E: 4. Apply rules to create PublicState + PrivateBlobs + E->>L: 5. Commit PublicState to public refs + E->>PS: 6. Store PrivateBlobs by digest +``` + +### 4. Pointer Resolution Protocol (Normative) + +Authentication semantics are aligned with HTTP. We adopt a simple, interoperable model (JWT default; HTTP Message Signatures optional): + +* **Endpoint**: `POST /gatos/private/blobs/resolve` +* **Request Body (application/json; JCS canonical form)**: + `{ "digest": "blake3:", "want": "plaintext"|"ciphertext" }` +* **Authorization**: `Authorization: Bearer ` + * Claims MUST include: `sub` (ed25519:), `aud` (node id or URL), `method` ("POST"), `path` ("/gatos/private/blobs/resolve"), `exp`, and `nbf`. + * Clock skew tolerance: ±300 seconds. + * On missing/invalid token: `401 Unauthorized`. On policy denial: `403 Forbidden`. + +A client resolving an Opaque Pointer **MUST** follow this protocol: + +1. **Parse Pointer**: Extract `digest`, optional `ciphertext_digest`, `location`, and `capability`. +2. **Fetch Blob**: + * If `gatos-node://`, resolve the actor's endpoint from the trust graph, then `POST /gatos/private/blobs/resolve` with the body above. + * The node **MUST** verify the bearer token and enforce policy before returning the blob. +3. **Acquire Capability**: + * Resolve the `capability` reference via the configured key system (KMS, key server). Secrets MUST NOT be embedded in the pointer. +4. **Decrypt and Verify**: + * Decrypt the fetched blob using the resolved key and AAD parameters (see Security Notes). + * Compute `blake3(plaintext)` and compare to `digest` if published; compute `blake3(ciphertext)` and compare to `ciphertext_digest` if published. A mismatch **MUST** produce `DigestMismatch`. + +Response headers on success: + +``` +Content-Type: application/octet-stream +X-BLAKE3-Digest: blake3: +Digest: sha-256= +``` + +Optional HTTP Message Signatures profile (RFC 9421): + +* Clients MAY authenticate by signing `@method`, `@target-uri`, `date`, `host`, `content-digest` (SHA‑256 of the JSON body) and sending `Signature-Input` and `Signature` headers. +* Servers SHOULD still return `Digest` and `X-BLAKE3-Digest` headers for response integrity. + +```mermaid +sequenceDiagram + participant C as Client + participant PN as Private GATOS Node + participant KMS as Key Management Service + + C->>C: 1. Read OpaquePointer + C->>PN: 2. POST /gatos/private/blobs/resolve (Authorization: Bearer ) + PN->>PN: 3. Check policy (is C allowed?) + alt Authorized + PN-->>C: 4. Return encrypted blob + C->>KMS: 5. Request key for {capability} + KMS-->>C: 6. Return decryption key + C->>C: 7. Decrypt blob + C->>C: 8. Verify blake3(decrypted) == digest + else Unauthorized + PN-->>C: 4. Return 401/403 + end +``` + +### 5. Policy Hooks (Normative) + +The privacy policy is defined in `.gatos/policy.yaml` and extends the policy engine's domain. + +```yaml +privacy: + classes: + pii_low_entropy: + min_entropy_bits: 40 + publish_plaintext_digest: false + require_ciphertext_digest: true + rules: + - select: "path.to.sensitive.data" + action: "pointerize" + class: "pii_low_entropy" + capability: "gatos-key://v1/aes-256-gcm/ops-key-01" + location: "gatos-node://ed25519:" + - select: "path.to.transient.data" + action: "redact" +``` + +The `select` syntax will use a simple path-matching language (e.g., glob patterns) defined by the policy engine. + +### 6. Auditability and Trailers (Normative) + +To make privacy operations transparent and auditable, any commit that creates a `PublicState` from a projection **MUST** include the following trailers: + +```text +Privacy-Redactions: 3 +Privacy-Pointers: 12 +Privacy-Pointer-Rotations: 1 +```text + +This provides a simple, top-level indicator that a projection has occurred, prompting auditors to look deeper if necessary. + +## Consequences + +### Pros + +* **Provable Privacy**: The model is grounded in the Morphology Calculus, making it verifiable. +* **Decoupled Storage**: Private data can live in any storage system (S3, IPFS, local disk) without affecting the public ledger's logic. +* **Integrated Auth/Authz**: By tying pointers to actor identities and capabilities, access to private data is governed by the existing GATOS trust and policy model. +* **Preserves Verifiability**: The `PublicState` remains globally verifiable, as pointers are just content-addressed links. + +### Cons + +* **Increased Complexity**: Resolution requires network requests and interaction with key management systems, adding latency and potential points of failure. +* **Operational Overhead**: Operators must manage the private blob stores and ensure their availability and security. + +## Feature Payoff + +* **Secure PII/Secret Storage**: Store sensitive data off-chain while retaining an auditable link to it. +* **Large Artifact Management**: Handle large binaries (ML models, videos) without bloating the Git repository. +* **Compliant Data Sharing**: Share a public, redacted dataset with third parties while retaining private access to the full, unified view. +* **Federated Learning**: Different actors can hold private models locally, referenced by pointers in a public "training plan" shape. + +--- + +## Namespacing and Storage (Normative) + +* Private overlays are actor‑anchored: `refs/gatos/private///` index metadata. The local workspace mirror is `gatos/private///`. +* Private blobs themselves are NOT stored under Git refs. They live in pluggable blob stores and are addressed by their `ciphertext_digest`/`digest`. + +## Security & Privacy Notes (Normative) + +* Capability references in pointers MUST NOT contain secrets or pre‑signed tokens. Use stable identifiers and resolve sensitive data via policy. +> Non‑normative interop profile: Some legacy deployments may use AES‑256‑GCM. If and only if such interop is required, deployments MAY support AES‑256‑GCM with AAD composed of (actor id, pointer digest, policy version) and 96‑bit nonces that are never reused per key. This profile is deprecated and MUST NOT be the default. +* Right‑to‑be‑forgotten: deleting private blobs breaks pointer resolution but does not remove the public pointer. Implement erasure as a tombstone event plus an audit record. + +### Algorithm variants (experimental; private attestations only) + +* Implementations MAY use a keyed BLAKE3 variant for private attestation envelopes (not for public Opaque Pointers): `algo = "blake3-keyed"` with parameters encoded in an envelope or pointer `extensions` field. +* Recommended KDF: `hkdf-sha256`; context string `"gatos:ptr:priv:"`; derive `key = HKDF(policy_key, salt = actor_pubkey, info = context)`. +* Public pointers MUST continue to use `algo = "blake3"` for third‑party verifiability. + +## Error Taxonomy (Normative) + +Implementations SHOULD use a stable set of error codes with JSON problem details: + +* `Unauthorized` (401) +* `Forbidden` (403) +* `NotFound` (404) +* `DigestMismatch` (422) +* `CapabilityUnavailable` (503) +* `PolicyDenied` (403) diff --git a/docs/decisions/ADR-0005/DECISION.md b/docs/decisions/ADR-0005/DECISION.md new file mode 100644 index 00000000..22593b6d --- /dev/null +++ b/docs/decisions/ADR-0005/DECISION.md @@ -0,0 +1,232 @@ +--- +Status: Proposed +Date: 2025-11-10 +ADR: ADR-0005 +Authors: [flyingrobots] +Requires: [ADR-0001, ADR-0004] +Related: [ADR-0002, ADR-0003] +Tags: [Shiplog, Event Stream, Consumers, JCS, ULID] +Schemas: + - ../../../schemas/v1/shiplog/event_envelope.schema.json + - ../../../schemas/v1/shiplog/consumer_checkpoint.schema.json + - ../../../schemas/v1/shiplog/deployment_trailer.schema.json + - ../../../schemas/v1/shiplog/anchor.schema.json + - ../../../schemas/v1/privacy/opaque_pointer.schema.json +Supersedes: [] +Superseded-By: [] +--- + +# ADR‑0005: Shiplog — A Parallel, Queryable Event Stream + +## Summary / Scope + +Introduce a first‑class, append‑only event stream ("Shiplog") that runs in parallel with deterministic state folds. The Shiplog provides per‑topic ordering, canonical event envelopes, consumer checkpoints, and query APIs. It is privacy‑aware (ADR‑0004) and deterministic (Morphology Calculus). + +## Context / Problem + +Many integrations require an append‑only stream rather than only snapshot state: analytics, external system replay, audit feeds, and incremental ETL. SPEC v0.3 defines append‑only journals conceptually but lacks a normative, queryable stream with consumer checkpoints and a canonical envelope format. This ADR makes the Shiplog normative. + +## Decision (Normative) + +### 1) Canonicalization and Identifiers + +- Envelope canonicalization: RFC 8785 JSON Canonicalization Scheme (JCS). The event Content‑Id is `blake3(JCS(envelope))`. +- ULID: 26‑char Crockford base32, uppercase, excluding I/L/O/U (`^[0-9A-HJKMNP-TV-Z]{26}$`). +- Hashes: content digests are `blake3:<64‑hex>` per `schemas/v1/common/ids.schema.json`. +- Numeric discipline: precision‑sensitive values (e.g., money/time) MUST be encoded as integers or strings. + +```mermaid +classDiagram + class EventEnvelope { + +string ulid + +string ns // topic namespace (e.g., "governance") + +string type // logical event type + +object payload // canonical JSON (JCS) + +map refs // OPTIONAL cross-refs + } +``` + +### 2) Namespaces and Ordering + +- Per‑namespace head ref (append‑only, linear): `refs/gatos/shiplog//head` +- Namespace naming: `^[a-z][a-z0-9._-]{0,63}$` (ASCII, lowercase start). +- Ordering per namespace is the Git parent chain. Appends MUST be fast‑forward (CAS on ref update). On a single node, ULIDs MUST increase strictly per namespace. + +```mermaid +graph TD + subgraph "Git Refs (sample)" + H1[refs/gatos/shiplog/demo/head]-->C1 + C1((e1))-->C2((e2))-->C3((e3)) + end + C1:::ev; C2:::ev; C3:::ev + classDef ev fill:#cde,stroke:#335; +``` + +### 3) Event Envelope (Schema) + +- Canonical JSON envelope at `schemas/v1/shiplog/event_envelope.schema.json` (draft‑2020‑12). +- Required fields: `ulid`, `ns`, `type`, `payload`. +- Optional `refs` (map) to link related state or IDs. +- Privacy (ADR‑0004): Payload MUST NOT embed private overlay data. Redacted values MUST be replaced by `OpaquePointer` envelopes per `schemas/v1/privacy/opaque_pointer.schema.json`. + +Numeric discipline: See §1 Canonicalization & Identifiers — precision‑sensitive values (e.g., monetary/time) MUST be encoded as integers or strings. + +### 4) Commit Message and Trailer + +Each Shiplog commit MUST include headers in the commit message (any order), followed by a single line containing three dashes `---` and then a JSON trailer object: + +``` +Event-Id: ulid: +Content-Id: blake3:<64-hex> +Namespace: +Envelope-Schema: https://gatos.dev/schemas/v1/shiplog/event_envelope.schema.json +Trailer-Schema: https://gatos.dev/schemas/v1/shiplog/deployment_trailer.schema.json +--- +{ "version": 1, + "env": "prod", + "who": { "name": "Jane Dev", "email": "jane@example.com" }, + "what": { "service": "web", "artifact": "ghcr.io/acme/web:1.2.3" }, + "where": { "region": "us-east-1", "cluster": "eks-a", "namespace": "prod" }, + "why": { "reason": "canary", "ticket": "OPS-123" }, + "how": { "pipeline": "gha", "run_url": "https://github.com/acme/repo/actions/runs/123456789" }, + "status": "success", + "when": { "start_ts": "2025-11-10T10:00:00Z", "end_ts": "2025-11-10T10:01:10Z", "dur_s": 70 }, + "seq": 42, + "journal_parent": "", + "trust_oid": "", + "previous_anchor": "", + "repo_head": "" +} +``` + +Trailer schema: `schemas/v1/shiplog/deployment_trailer.schema.json`. + +MUST: validate the trailer against this schema, and write the exact JCS bytes hashed for the envelope to `/gatos/shiplog//.json` (parse → JCS → hash → write → commit). The path is a logical path inside the Git commit tree: `/gatos/shiplog/...` is a blob stored in the tree referenced by the Shiplog commit, not a working‑directory file. Thus the object hash of the blob equals the canonical digest; verifiers MUST compare them byte‑for‑byte. + +Note: The trailer places `repo_head` as a top‑level field only. It MUST NOT appear inside nested objects such as `what` (or any nested object). Producers MUST ensure no duplication occurs. + +> [!IMPORTANT] +> Hashing Law — parse → JCS → hash → write → commit. The bytes you hash MUST be the exact JCS bytes you write and commit. + +### 5) Append Semantics + +Invariant: envelope.ns MUST equal the commit header `Namespace:` value and the per‑namespace ref segment. + +Append(`ns`, `envelope`): validate schema; compute `content_id = blake3(JCS(envelope))`; enforce monotone ULID per namespace on this node; create commit with headers + trailer; CAS update `refs/gatos/shiplog//head`; return `(commit_oid, ulid, content_id)`. + +ULID generation (normative): Implementations MUST use a monotonic ULID algorithm scoped per namespace (see ULID Spec §4.1 Monotonic Lexicographic Ordering). If the system clock moves backwards, the implementation MUST keep the last emitted millisecond timestamp and monotonically increase the randomness field; on overflow of the randomness field, the operation MUST fail with `TemporalOrder`. If two appends observe the same timestamp, the second MUST increase the randomness field compared to the previous append or fail with `AppendRejected` on CAS. Replayed appends MUST preserve the original ULID for that envelope; otherwise reject with `DigestMismatch`. + +Errors (normative): + +- 400 `InvalidEnvelope`; 409 `TemporalOrder`; 409 `AppendRejected`; 422 `DigestMismatch`. + +### 6) Query Semantics + +- `shiplog.read(ns, since_ulid, limit) -> [ (ulid, content_id, commit_oid, envelope) ]` (increasing ULID order). +- `shiplog.tail(namespaces[], limit_per_ns)` MAY multiplex without cross‑namespace causality guarantees. + +Tail fairness (normative): When multiplexing multiple namespaces, implementations MUST use fair scheduling (e.g., per‑namespace round‑robin) so that no namespace is starved under sustained load. The round‑robin order SHOULD be stable across restarts. Implementations SHOULD emit a per‑namespace watermark (last ULID included) to help consumers resume without duplication. Consumers restore by resuming from each namespace’s last watermark or their checkpoint, whichever is newer; duplicates MUST be tolerated by idempotent processing keyed by `(ns, ulid)`. + +### 7) Consumer Checkpoints + +- `refs/gatos/consumers//` points to the last processed Shiplog commit OID. Portable JSON (optional): `schemas/v1/shiplog/consumer_checkpoint.schema.json`. The `commit_oid` value MUST be lowercase hex. + +### 8) Privacy Interactions (ADR‑0004) + +AEAD algorithm is pinned by ADR‑0004 to XChaCha20‑Poly1305. Nonces MUST be unique per key; prefer deterministic HKDF‑derived nonces (domain-separated) or crash‑safe per‑key counters in KMS. Random nonces are permitted only with a documented collision budget and monitoring. AAD MUST bind the pointer digest (not a separate content_id), the actor id, and the policy version so verifiers can validate context. + +- Payloads MUST NOT embed private overlay data. Use Opaque Pointers per privacy schema. For low‑entropy classes, include `ciphertext_digest` and omit plaintext digest in public pointers. + +### 9) Governance and Ledger Interactions + +- Governance (ADR‑0003): Should emit Shiplog events under `ns="governance"`; envelopes carry `ns="governance"` and the commit header sets `Namespace: governance`. +- Ledger mirroring: MAY mirror ledger events; must preserve envelope determinism. + +### 10) Security Considerations + +- No secrets in commit messages or payloads. Use capability URIs; notes/logs may be private or pointerized. +- Idempotent appends; checkpoints are advisory. + +### 11) CLI Examples + +```bash +$ gatosd shiplog append --ns governance --file event.json +ok commit=8b1c1e4 content_id=blake3:2a6c… ulid=01HF4Y9Q1SM8Q7K9DK2R3V4AWB + +$ gatosd shiplog read --ns governance --since 01HF4Y9Q1SM8Q7K9DK2R3V4AWB --limit 2 +01HF4Y9Q1SM8Q7K9DK2R4V5CXD blake3:2a6c… 8b1c1e4 {"ulid":"01HF4Y9Q1SM8Q7K9DK2R4V5CXD","ns":"governance","type":"proposal.created","payload":{}} +01HF4Y9Q1SM8Q7K9DK2R4V5CXE blake3:c1d2… 9f0aa21 {"ulid":"01HF4Y9Q1SM8Q7K9DK2R4V5CXE","ns":"governance","type":"proposal.approved","payload":{}} + +$ gatosd shiplog checkpoint set --group analytics --ns governance --commit 8b1c1e4 +ok refs/gatos/consumers/analytics/governance -> 8b1c1e4 +``` + +### 12) Anchors + +Anchors are signed, portable snapshots of a Shiplog namespace head. An anchor document conforms to `schemas/v1/shiplog/anchor.schema.json` and records `(ulid, topic/ns, head)` plus optional metadata. Producers MAY write an anchor when rolling out a deployment, completing a batch, or before compaction. Consumers use anchors as stable restore points and for cross‑repo attestation. If signatures are used, they MUST bind the anchor JSON bytes (JCS) and the Git commit oid referenced by `head`. Anchors SHOULD be signed with the same key as the containing namespace’s governance identity to preserve a coherent audit chain. + +## Error Taxonomy (Normative) + +| Code | HTTP | Meaning | +|---|---:|---| +| AppendRejected | 409 | Not fast-forward (CAS failed) | +| TemporalOrder | 409 | ULID/timestamp monotonicity failure | +| DigestMismatch | 422 | Hash mismatch (body/envelope/JCS) | +| SigInvalid | 401/403 | Signature/attestation invalid | +| PolicyDenied | 403 | Policy decision denied | +| NotFound | 404 | Missing topic/checkpoint/anchor | +| CapabilityUnavailable | 503 | Dependent capability/KMS/blob store unavailable | + +## Consequences + +Clients SHOULD return a problem+json response with a stable `code` plus HTTP status. Example: + +```json +{ + "type": "https://gatos.dev/problems/append-rejected", + "title": "AppendRejected", + "status": 409, + "code": "AppendRejected", + "detail": "Not fast-forward (CAS failed)", + "instance": "urn:commit:8b1c1e4" +} +``` + +Pros: clean integration surface; deterministic envelopes; replay + analytics; explicit privacy. +Cons: additional refs to manage; potential duplication if mirroring ledger events. + +## Migration / Rollout + +1. Add schemas + CI wiring. +2. Implement gatos‑mind adapter and gatosd CLI/RPC. +3. Emit governance events. + +## Test Plan (Property + Integration) + +- Determinism; ordering; idempotence; query pagination; checkpoints; privacy envelopes. + +## Documentation Updates + +- SPEC and TECH‑SPEC sections updated; FEATURES include F6 — Shiplog Event Stream. + +## References + +- ADR‑0001, ADR‑0003, ADR‑0004. RFC 8785 JCS. + +--- + +## Compatibility Profile: `shiplog-compat` + +To interoperate with existing bash‑based producers (e.g., `git shiplog`), implementations MUST support a compatibility profile: + +- `ref_root = refs/_shiplog`; Journals: `journal/`; Anchors: `anchors/`; Notes: `notes/logs`; Consumers (optional mirror): `consumers//`. + +Commit body conventions are identical: header lines, a single `---` separator, then a JSON trailer object. Envelopes MAY be present in the commit tree for auditability. + +Canonicalization (ingestion): Content‑Id remains `blake3(JCS(envelope))`. If an existing producer created compact, key‑sorted JSON via `jq -cS .`, readers MUST parse and re‑canonicalize to JCS before hashing. Producers SHOULD emit JCS bytes. + +Anchors and Notes: Anchor commits MAY be written periodically to capture rollup points. Attachments/logs SHOULD be stored as Git notes; redact or pointerize as needed. + +Error taxonomy (aligned with Ledger‑Kernel): `AppendRejected`, `TemporalOrder`, `PolicyFail`, `SigInvalid`, `DigestMismatch`. + +Importer (recommended): mirror from `refs/_shiplog/*` to `refs/gatos/shiplog/*`, re‑canonicalizing to JCS and preserving commit authorship/timestamps. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 94b0994e..6425c593 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -20,3 +20,5 @@ Each ADR will have a status, typically one of the following: | [ADR-0001](./ADR-0001/DECISION.md) | Split gatos-ledger into no_std Core and std Backends | Accepted | 2025-11-08 | | [ADR-0002](./ADR-0002/DECISION.md) | Distributed Compute via a Job Plane | Accepted | 2025-11-08 | | [ADR-0003](./ADR-0003/DECISION.md) | Consensus Governance for Gated Actions | Accepted | 2025-11-08 | +| [ADR-0004](./ADR-0004/DECISION.md) | Hybrid Privacy Model (Public Projection + Private Overlay) | Accepted | 2025-11-10 | +| [ADR-0005](./ADR-0005/DECISION.md) | Shiplog — A Parallel, Queryable Event Stream | Proposed | 2025-11-10 | diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_1.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_1.svg new file mode 100644 index 00000000..f24f93ba --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_1.svg @@ -0,0 +1 @@ +
RFC 2119
Keywords
MUST
SHOULD
MAY
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_10.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_10.svg new file mode 100644 index 00000000..08b8917b --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_10.svg @@ -0,0 +1 @@ +
merge
undo
fork
undo
main
commit-a
commit-b
session-2
session-1
main
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_11.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_11.svg new file mode 100644 index 00000000..81877d3d --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_11.svg @@ -0,0 +1 @@ +
ProofEnvelope
+String type
+String ulid
+String inputs_root
+String output_root
+String policy_root
+String proof
+String sig
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_12.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_12.svg new file mode 100644 index 00000000..5de93309 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_12.svg @@ -0,0 +1 @@ +PeerBPeerAPeerBPeerAPeers are offline and make divergent changesalt[Policies are comparable][Policies areincomparable]Reconnect & Exchange EnvelopesValidate Signatures & Policy AncestryPrefer descendant policyAppend governance.conflict event diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_13.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_13.svg new file mode 100644 index 00000000..ef05eede --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_13.svg @@ -0,0 +1 @@ +
GATOS
local
push-gate
saas-hosted
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_14.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_14.svg new file mode 100644 index 00000000..09f7b59b --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_14.svg @@ -0,0 +1 @@ +
Exposes
Scrapes
Diagnoses
gatosd
/metrics
Prometheus
gatos doctor
Ref Invariants
Epoch Continuity
Cache Staleness
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_15.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_15.svg new file mode 100644 index 00000000..7d34e01c --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_15.svg @@ -0,0 +1 @@ +
Access Control
Requests access to
Links
Links
Evaluates request for
Resource
Actor
Capability Grant
Policy
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_16.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_16.svg new file mode 100644 index 00000000..030fa4e5 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_16.svg @@ -0,0 +1 @@ +
Triggers
Prunes
Epoch N
Epoch N+1 Anchor
Compaction
Unreferenced Blobs in Epoch N
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_17.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_17.svg new file mode 100644 index 00000000..cbe16f08 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_17.svg @@ -0,0 +1 @@ +
GATOS Implementation
Certification
Deterministic Fold
Exactly-Once Delivery
Offline Reconcile
Deny Audit
Blob Integrity
Consensus Integrity
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_18.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_18.svg new file mode 100644 index 00000000..9e54f7b0 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_18.svg @@ -0,0 +1 @@ +
git gatos
init
session
event
fold
bus
policy
trust
epoch
prove
doctor
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_19.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_19.svg new file mode 100644 index 00000000..70ebdf51 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_19.svg @@ -0,0 +1 @@ +Workergatos-echogatos-mindgatos-ledgergatosdClientWorkergatos-echogatos-mindgatos-ledgergatosdClientLater, a worker consumes the job...A fold process runs...1. Enqueue Job (Event)2. Append `jobs.enqueue` event3. Success4. Publish `gmb.msg` to topic5. Success6. Job Enqueued7. Subscribe to topic8. Delivers `gmb.msg`9. Report Result (Event)10. Append `jobs.result` event11. Success12. Write `gmb.ack`13. Result Recorded14. Read events from journal15. Compute new state (e.g., update queue view)16. Checkpoint new state diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_2.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_2.svg new file mode 100644 index 00000000..646b6118 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_2.svg @@ -0,0 +1 @@ +
GATOS System
User / Client
Policy Plane
State Plane
Message Plane
Job Plane
Ledger Plane
gatosd (Daemon)
gatos-ledger
gatos-compute (planned)
gatos-mind
gatos-echo
gatos-kv
gatos-policy
gatosd (CLI)
Client SDK
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_20.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_20.svg new file mode 100644 index 00000000..a241d1d0 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_20.svg @@ -0,0 +1 @@ +
Worker claims job (CAS)
Worker begins execution
`jobs.result` (ok)
`jobs.result` (fail)
Canceled by user/policy
Canceled by user/policy
pending
claimed
running
succeeded
failed
aborted
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_21.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_21.svg new file mode 100644 index 00000000..ff575db4 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_21.svg @@ -0,0 +1 @@ +
approval received
additional approvals
ttl elapsed
ttl elapsed
quorum satisfied
revocation committed
proposal
partial
expired
granted
revoked
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_3.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_3.svg new file mode 100644 index 00000000..7bfd07de --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_3.svg @@ -0,0 +1 @@ +
Workspace
policies
gatos
schema
folds
trust
objects
.git
gatos
refs
journal
state
mbus
jobs
sessions
audit
cache
epoch
private
notes
gatos
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_4.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_4.svg new file mode 100644 index 00000000..051c2670 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_4.svg @@ -0,0 +1 @@ +
issuer
1
1
subject
1
1
Grant
+String ulid
+String issuer
+String subject
+String[] caps
+Date exp
+String sig
«enumeration»
Actor
user
agent
service
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_5.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_5.svg new file mode 100644 index 00000000..4a0f3693 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_5.svg @@ -0,0 +1 @@ +
EventEnvelope
+String type
+String ulid
+String actor
+String[] caps
+Object payload
+String policy_root
+String sig
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_6.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_6.svg new file mode 100644 index 00000000..005f6e3d --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_6.svg @@ -0,0 +1 @@ +
Event Stream
Fold Function
Policy
State Root
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_7.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_7.svg new file mode 100644 index 00000000..9c16e272 --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_7.svg @@ -0,0 +1 @@ +PolicyGateGATOSClientPolicyGateGATOSClientalt[Action is Allowed][Action is Denied]Propose Action (Intent)Evaluate(Intent, Context)Decision: AllowBind policy_root to eventSuccessDecision: Deny(reason)Write Audit DecisionFailure(reason) diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_8.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_8.svg new file mode 100644 index 00000000..579db54a --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_8.svg @@ -0,0 +1 @@ +
OpaquePointer
+string kind: "opaque_pointer"
+string algo: "blake3"
+string digest: "blake3:" // plaintext digest
+string ciphertext_digest: "blake3:" // optional
+int size // bytes; SHOULD be present
+string location
+string capability // MUST NOT embed secrets
+object extensions // forward-compatible
 diff --git a/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_9.svg b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_9.svg new file mode 100644 index 00000000..6e2e4e8f --- /dev/null +++ b/docs/diagrams/generated/docs_SPEC__0679d036ea__mermaid_9.svg @@ -0,0 +1 @@ +ConsumerGATOSPublisherConsumerGATOSPublisherPublish Message (QoS: exactly_once)Deliver MessageProcess MessageSend AckObserve Ack QuorumCreate gmb.commit Event diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_1.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_1.svg new file mode 100644 index 00000000..2a48ce81 --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_1.svg @@ -0,0 +1 @@ +
gatos
gatos-ledger-core
crates
gatos-ledger-git
gatos-mind
gatos-echo
gatos-policy
gatos-kv
gatosd
gatos-compute
bindings
wasm
ffi
 diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_10.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_10.svg new file mode 100644 index 00000000..d41c9ca6 --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_10.svg @@ -0,0 +1 @@ +LibsodiumGATOSClientLibsodiumGATOSClientalt[Signature is Valid][Signature is Invalid]Submit Signed EventCanonicalize JSONed25519_verify(signature, payload, pubkey)OKProcess EventFailReject Event diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_11.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_11.svg new file mode 100644 index 00000000..77c3feba --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_11.svg @@ -0,0 +1 @@ +Batch Size Trade-off (Illustrative)11.522.533.544.55Batch Size16014012010080604020Metric diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_12.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_12.svg new file mode 100644 index 00000000..a3c613c3 --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_12.svg @@ -0,0 +1 @@ +
JSONL RPC
JSONL RPC
JSONL RPC
JSONL RPC
gatosd
Go SDK
Python SDK
Rust SDK
Node.js SDK
 diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_13.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_13.svg new file mode 100644 index 00000000..119564aa --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_13.svg @@ -0,0 +1 @@ +2025-01-052025-01-122025-01-192025-01-262025-02-022025-02-092025-02-162025-02-232025-03-022025-03-092025-03-162025-03-232025-03-302025-04-06Mirror Mode Shadow Consumers Canary (10%) Full Cutover Phase A: MirrorPhase B: ShadowPhase C: Dual-ReadPhase D: CutoverGATOS Migration Strategy diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_14.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_14.svg new file mode 100644 index 00000000..ee050bcd --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_14.svg @@ -0,0 +1 @@ +
Encodes
«Rust»
BincodeConfig
+standard()
Hash
+[u8; 32]
 diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_15.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_15.svg new file mode 100644 index 00000000..ad6fb484 --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_15.svg @@ -0,0 +1 @@ +WorkerBus (Message Plane)GATOS (Ledger)ClientWorkerBus (Message Plane)GATOS (Ledger)Client1. Create Job Commit2. Publish Job message3. Subscribe to job topic4. Receive Job message5. Atomically create Claim ref6. Claim successful7. Execute Job8. Create Result commit diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_16.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_16.svg new file mode 100644 index 00000000..4af2e2db --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_16.svg @@ -0,0 +1 @@ +Approver (via CLI)Message BusPolicy EngineGATOS (Ledger)ClientApprover (via CLI)Message BusPolicy EngineGATOS (Ledger)Clientloop[Approvals]alt[Quorum satisfied][Not yet satisfied]1. Create Proposal (Action, Target, Quorum)2. Validate proposal3. Accepted4. Publish proposal.created5. Create Approval (Signer, Proposal-Id)6. Verify signature + eligibility7. Approval valid8. Check quorum9. Create Grant (Proof-Of-Consensus)10. Publish grant.createdPending (partial) diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_2.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_2.svg new file mode 100644 index 00000000..12a405ec --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_2.svg @@ -0,0 +1 @@ +
GATOS System
User / Client
Policy Plane
State Plane
Message Plane
Job Plane
Ledger Plane
gatosd (Daemon)
gatos-ledger
gatos-compute
gatos-mind
gatos-echo
gatos-kv
gatos-policy
gatosd (CLI)
Client SDK
 diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_3.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_3.svg new file mode 100644 index 00000000..5d2eb6fc --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_3.svg @@ -0,0 +1 @@ +
uses
blake3
Canonical Events
FoldEngine
rmg-core
Canonical JSON Tree
state_root
 diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_4.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_4.svg new file mode 100644 index 00000000..43a4bc94 --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_4.svg @@ -0,0 +1 @@ +
folded by
produces
stored in
Journal Events
Indexer
Roaring Bitmap
refs/gatos/cache/
 diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_5.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_5.svg new file mode 100644 index 00000000..c92bfdd1 --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_5.svg @@ -0,0 +1 @@ +GATOSUserGATOSUsergatos epoch new <ns>Create new anchor at refs/gatos/epoch/<ns>/<epoch-id>Start CompactorWalk reachability from state_rootPrune unreferenced blobs diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_6.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_6.svg new file mode 100644 index 00000000..7298acaa --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_6.svg @@ -0,0 +1 @@ +LedgerPrivateStorePolicyEcho"StorageBackend (Interface)"gatos-ledgergatos-policygatos-echoLedgerPrivateStorePolicyEcho"StorageBackend (Interface)"gatos-ledgergatos-policygatos-echoalt[rule matches (e.g., "pointerize")]loop[for each field path in the UnifiedState tree]1. Fold event history to produce UnifiedState2. Request privacy rules for the current context3. Return `select` and `action` rules4. Match field path against rules5. Generate Opaque Pointer envelope6. Store original node value as private blob, keyed by its blake3 digest7. Replace node in state tree with pointer8. Commit the final PublicState tree diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_7.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_7.svg new file mode 100644 index 00000000..477574ad --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_7.svg @@ -0,0 +1 @@ +gatosdClient (SDK/CLI)gatosdClient (SDK/CLI)loop[Subscription Stream]{"type":"append_event", "id":"01A", "topic":"...", "event":{...}}{"ok":true, "id":"01A", "commit_id":"..."}{"type":"bus.subscribe", "id":"01C", "topic":"..."}{"ack":true, "id":"01C"}{"type":"bus.message", "id":"01C", "topic":"...", "payload":{...}} diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_8.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_8.svg new file mode 100644 index 00000000..cfed5e05 --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_8.svg @@ -0,0 +1 @@ +
Metrics
gatosd
gatos_journal_append_latency_ms
gatos_fold_latency_ms
gatos_bus_ack_lag
Journal
Fold Engine
Message Bus
 diff --git a/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_9.svg b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_9.svg new file mode 100644 index 00000000..a814a1e4 --- /dev/null +++ b/docs/diagrams/generated/docs_TECH-SPEC__15850d53f4__mermaid_9.svg @@ -0,0 +1 @@ +
CI Pipeline
Test Matrix
linux-amd64-glibc
macOS-arm64
Windows-amd64
Test Suites
Golden Vectors
Torture Tests
Reconcile Harness
Projection Determinism
 diff --git a/docs/diagrams/generated/docs_decisions_ADR-0001_DECISION__b1ba19490c__mermaid_1.svg b/docs/diagrams/generated/docs_decisions_ADR-0001_DECISION__b1ba19490c__mermaid_1.svg new file mode 100644 index 00000000..bc8be047 --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0001_DECISION__b1ba19490c__mermaid_1.svg @@ -0,0 +1 @@ +
feature: git2-backend
feature: core-only
Consumer Crate
gatos-ledger (meta-crate)
gatos-ledger-git (std)
gatos-ledger-core (no_std)
 diff --git a/docs/diagrams/generated/docs_decisions_ADR-0001_DECISION__b1ba19490c__mermaid_2.svg b/docs/diagrams/generated/docs_decisions_ADR-0001_DECISION__b1ba19490c__mermaid_2.svg new file mode 100644 index 00000000..5c5b303d --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0001_DECISION__b1ba19490c__mermaid_2.svg @@ -0,0 +1 @@ +ObjectStore Backendgatos-ledger-coreApplicationObjectStore Backendgatos-ledger-coreApplicationcreate_commit(parent, tree, signature)commit = Commit { ... }hash = compute_commit_id(&commit)put_object(hash, serialize(commit))(persists object)return hash diff --git a/docs/diagrams/generated/docs_decisions_ADR-0001_DECISION__b1ba19490c__mermaid_3.svg b/docs/diagrams/generated/docs_decisions_ADR-0001_DECISION__b1ba19490c__mermaid_3.svg new file mode 100644 index 00000000..27bb4633 --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0001_DECISION__b1ba19490c__mermaid_3.svg @@ -0,0 +1 @@ +
default-features = true
default-features = false,
features = [core-only]
Consumer Crate
uses gatos-ledger
Feature: 'git2-backend'
Includes `gatos-ledger-git`
Includes `gatos-ledger-core`
Provides `GitStore` impl of `ObjectStore`
Feature: 'core-only'
Includes `gatos-ledger-core` only
Provides `ObjectStore` trait and core types
 diff --git a/docs/diagrams/generated/docs_decisions_ADR-0002_DECISION__a21cc79f92__mermaid_1.svg b/docs/diagrams/generated/docs_decisions_ADR-0002_DECISION__a21cc79f92__mermaid_1.svg new file mode 100644 index 00000000..7b3f5060 --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0002_DECISION__a21cc79f92__mermaid_1.svg @@ -0,0 +1 @@ +
Worker discovers & claims job
Worker begins execution
Execution successful
Execution fails
Canceled by user/policy
Canceled by user/policy
pending
claimed
running
succeeded
failed
aborted
 diff --git a/docs/diagrams/generated/docs_decisions_ADR-0002_DECISION__a21cc79f92__mermaid_2.svg b/docs/diagrams/generated/docs_decisions_ADR-0002_DECISION__a21cc79f92__mermaid_2.svg new file mode 100644 index 00000000..df7a8c48 --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0002_DECISION__a21cc79f92__mermaid_2.svg @@ -0,0 +1 @@ +WorkerBus (Message Plane)GATOS (Ledger)ClientWorkerBus (Message Plane)GATOS (Ledger)Clientalt[Claim already exists by another worker][Claim created]1. Create Job Commit2. Publish Job message3. Subscribe to job topic4. Receive Job message5. Atomically create Claim ref (by job-id)6. Claim failed (CAS)7. Backoff and retry6. Claim successful7. Execute Job8. Create Result commit (with Job-Id trailer) diff --git a/docs/diagrams/generated/docs_decisions_ADR-0003_DECISION__3c4445d569__mermaid_1.svg b/docs/diagrams/generated/docs_decisions_ADR-0003_DECISION__3c4445d569__mermaid_1.svg new file mode 100644 index 00000000..ff575db4 --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0003_DECISION__3c4445d569__mermaid_1.svg @@ -0,0 +1 @@ +
approval received
additional approvals
ttl elapsed
ttl elapsed
quorum satisfied
revocation committed
proposal
partial
expired
granted
revoked
 diff --git a/docs/diagrams/generated/docs_decisions_ADR-0003_DECISION__3c4445d569__mermaid_2.svg b/docs/diagrams/generated/docs_decisions_ADR-0003_DECISION__3c4445d569__mermaid_2.svg new file mode 100644 index 00000000..eeb29207 --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0003_DECISION__3c4445d569__mermaid_2.svg @@ -0,0 +1 @@ +ApproverMessage BusPolicy EngineGATOS (Ledger)ClientApproverMessage BusPolicy EngineGATOS (Ledger)Clientloop[Approvals]alt[Quorum satisfied][Not yet satisfied]1. Create Proposal (Action, Target, Quorum)2. Validate proposal3. Accepted4. Publish proposal.created5. Create Approval (Signer, Proposal-Id)6. Verify signature + eligibility7. Approval valid8. Check quorum9. Create Grant (Proof-Of-Consensus)10. Publish grant.createdPending (partial) diff --git a/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_1.svg b/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_1.svg new file mode 100644 index 00000000..a4c30efe --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_1.svg @@ -0,0 +1 @@ +
Sh_Private
Sh_Public
Sh_Unified
Commit c
Proj(c)
Proj
Proj
Extract
Extract
Private Blobs 1
Private Blobs 2
Public Shape 1
Public Shape 2
Unified Shape 1
Unified Shape 2
 diff --git a/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_2.svg b/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_2.svg new file mode 100644 index 00000000..9faec20a --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_2.svg @@ -0,0 +1 @@ +
OpaquePointer
+string kind: "opaque_pointer"
+string algo: "blake3"
+string digest: "blake3:" // plaintext digest
+string ciphertext_digest: "blake3:" // MAY be present
+string location
+string capability // MUST NOT embed secrets
+object extensions // forward-compatible
+int size // SHOULD be present(bytes)
 diff --git a/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_3.svg b/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_3.svg new file mode 100644 index 00000000..0e9fcfc8 --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_3.svg @@ -0,0 +1 @@ +Private StoreLedger (Git)Policy EngineState Engine (gatos-echo)Private StoreLedger (Git)Policy EngineState Engine (gatos-echo)1. Fold history into UnifiedState2. Fetch privacy rules3. Return rules (redact/pointerize)4. Apply rules to create PublicState + PrivateBlobs5. Commit PublicState to public refs6. Store PrivateBlobs by digest diff --git a/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_4.svg b/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_4.svg new file mode 100644 index 00000000..df225c1c --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0004_DECISION__12d7f53080__mermaid_4.svg @@ -0,0 +1 @@ +Key Management ServicePrivate GATOS NodeClientKey Management ServicePrivate GATOS NodeClientalt[Authorized][Unauthorized]1. Read OpaquePointer2. POST /gatos/private/blobs/resolve (Authorization: Bearer <JWT>)3. Check policy (is C allowed?)4. Return encrypted blob5. Request key for {capability}6. Return decryption key7. Decrypt blob8. Verify blake3(decrypted) == digest4. Return 401/403 diff --git a/docs/diagrams/generated/docs_decisions_ADR-0005_DECISION__03ffbed862__mermaid_1.svg b/docs/diagrams/generated/docs_decisions_ADR-0005_DECISION__03ffbed862__mermaid_1.svg new file mode 100644 index 00000000..c5bf78ea --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0005_DECISION__03ffbed862__mermaid_1.svg @@ -0,0 +1 @@ +
EventEnvelope
+string ulid
+string type // logical event type
+map refs // OPTIONAL cross-refs
+string ns // topic namespace(e.g., "governance")
+object payload // canonical JSON(JCS)
 diff --git a/docs/diagrams/generated/docs_decisions_ADR-0005_DECISION__03ffbed862__mermaid_2.svg b/docs/diagrams/generated/docs_decisions_ADR-0005_DECISION__03ffbed862__mermaid_2.svg new file mode 100644 index 00000000..9fd28dcf --- /dev/null +++ b/docs/diagrams/generated/docs_decisions_ADR-0005_DECISION__03ffbed862__mermaid_2.svg @@ -0,0 +1 @@ +
Git Refs (sample)
e1
refs/gatos/shiplog/demo/head
e2
e3
 diff --git a/docs/diagrams/generated/docs_diagrams_api_endpoints__578bd81e4d__mermaid_1.svg b/docs/diagrams/generated/docs_diagrams_api_endpoints__578bd81e4d__mermaid_1.svg new file mode 100644 index 00000000..e0680fa8 --- /dev/null +++ b/docs/diagrams/generated/docs_diagrams_api_endpoints__578bd81e4d__mermaid_1.svg @@ -0,0 +1 @@ +gatosdClient (SDK/CLI)gatosdClient (SDK/CLI)loop[Subscription Stream]{"type":"append_event", "id":"01A", "ns":"...", "event":{...}}{"ok":true, "id":"01A", "commit_id":"..."}{"type":"bus.publish", "id":"01B", "topic":"...", "payload":{...}}{"ok":true, "id":"01B", "msg_id":"..."}{"type":"bus.subscribe", "id":"01C", "topic":"..."}{"ack":true, "id":"01C"}{"type":"bus.message", "id":"01C", "topic":"...", "payload":{...}}{"type":"fold_state", "id":"01D", "ns":"..."}{"ok":true, "id":"01D", "state_root":"..."} diff --git a/docs/diagrams/generated/docs_diagrams_api_endpoints__578bd81e4d__mermaid_2.svg b/docs/diagrams/generated/docs_diagrams_api_endpoints__578bd81e4d__mermaid_2.svg new file mode 100644 index 00000000..5fe476e7 --- /dev/null +++ b/docs/diagrams/generated/docs_diagrams_api_endpoints__578bd81e4d__mermaid_2.svg @@ -0,0 +1 @@ +gatosdClient (SDK/CLI)gatosdClient (SDK/CLI){"type":"append_event", "id":"02A", "ns":"invalid", "event":{...}}{"ok":false, "id":"02A", "error":{"code":"ERR_INVALID_NS", "message":"namespace not found"}} diff --git a/docs/diagrams/generated/docs_diagrams_architecture__105fc24d87__mermaid_1.svg b/docs/diagrams/generated/docs_diagrams_architecture__105fc24d87__mermaid_1.svg new file mode 100644 index 00000000..12a405ec --- /dev/null +++ b/docs/diagrams/generated/docs_diagrams_architecture__105fc24d87__mermaid_1.svg @@ -0,0 +1 @@ +
GATOS System
User / Client
Policy Plane
State Plane
Message Plane
Job Plane
Ledger Plane
gatosd (Daemon)
gatos-ledger
gatos-compute
gatos-mind
gatos-echo
gatos-kv
gatos-policy
gatosd (CLI)
Client SDK
 diff --git a/docs/diagrams/generated/docs_diagrams_data_flow__559f0c180d__mermaid_1.svg b/docs/diagrams/generated/docs_diagrams_data_flow__559f0c180d__mermaid_1.svg new file mode 100644 index 00000000..70ebdf51 --- /dev/null +++ b/docs/diagrams/generated/docs_diagrams_data_flow__559f0c180d__mermaid_1.svg @@ -0,0 +1 @@ +Workergatos-echogatos-mindgatos-ledgergatosdClientWorkergatos-echogatos-mindgatos-ledgergatosdClientLater, a worker consumes the job...A fold process runs...1. Enqueue Job (Event)2. Append `jobs.enqueue` event3. Success4. Publish `gmb.msg` to topic5. Success6. Job Enqueued7. Subscribe to topic8. Delivers `gmb.msg`9. Report Result (Event)10. Append `jobs.result` event11. Success12. Write `gmb.ack`13. Result Recorded14. Read events from journal15. Compute new state (e.g., update queue view)16. Checkpoint new state diff --git a/docs/diagrams/generated/docs_diagrams_state_management__86fd1ffb65__mermaid_1.svg b/docs/diagrams/generated/docs_diagrams_state_management__86fd1ffb65__mermaid_1.svg new file mode 100644 index 00000000..080036d4 --- /dev/null +++ b/docs/diagrams/generated/docs_diagrams_state_management__86fd1ffb65__mermaid_1.svg @@ -0,0 +1 @@ +
Worker consumes `bus.message`
`jobs.result` (ok) event recorded
`jobs.result` (fail) event recorded
`attempts` < max_retries
`attempts` >= max_retries
Job is re-published
Enqueued
Processing
Succeeded
Failed
Retrying
DeadLetterQueue
 diff --git a/examples/v1/policy/privacy_min.json b/examples/v1/policy/privacy_min.json new file mode 100644 index 00000000..c510c3b9 --- /dev/null +++ b/examples/v1/policy/privacy_min.json @@ -0,0 +1,21 @@ +{ + "privacy": { + "classes": { + "pii_low_entropy": { + "min_entropy_bits": 40, + "publish_plaintext_digest": false, + "require_ciphertext_digest": true + } + }, + "rules": [ + { + "select": "user.email", + "action": "pointerize", + "class": "pii_low_entropy", + "capability": "gatos-key://v1/aes-256-gcm/ops-key-01", + "location": "gatos-node://ed25519:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + } + ] + } +} + diff --git a/examples/v1/privacy/opaque_pointer_min.json b/examples/v1/privacy/opaque_pointer_min.json new file mode 100644 index 00000000..eb3cf057 --- /dev/null +++ b/examples/v1/privacy/opaque_pointer_min.json @@ -0,0 +1,9 @@ +{ + "kind": "opaque_pointer", + "algo": "blake3", + "digest": "blake3:0000000000000000000000000000000000000000000000000000000000000000", + "ciphertext_digest": "blake3:1111111111111111111111111111111111111111111111111111111111111111", + "size": 0, + "location": "gatos-node://ed25519:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "capability": "gatos-key://v1/aes-256-gcm/test-key-01" +} diff --git a/examples/v1/privacy/pointer_low_entropy_invalid.json b/examples/v1/privacy/pointer_low_entropy_invalid.json new file mode 100644 index 00000000..21733fc6 --- /dev/null +++ b/examples/v1/privacy/pointer_low_entropy_invalid.json @@ -0,0 +1,8 @@ +{ + "kind": "opaque_pointer", + "algo": "blake3", + "digest": "blake3:0000000000000000000000000000000000000000000000000000000000000000", + "location": "gatos://private/blob/abc", + "capability": "gatos://cap/abc", + "extensions": { "class": "low-entropy" } +} diff --git a/examples/v1/privacy/pointer_low_entropy_min.json b/examples/v1/privacy/pointer_low_entropy_min.json new file mode 100644 index 00000000..d1dfd865 --- /dev/null +++ b/examples/v1/privacy/pointer_low_entropy_min.json @@ -0,0 +1,8 @@ +{ + "kind": "opaque_pointer", + "algo": "blake3", + "ciphertext_digest": "blake3:0000000000000000000000000000000000000000000000000000000000000000", + "location": "gatos://private/blob/abc", + "capability": "gatos://cap/abc", + "extensions": { "class": "low-entropy" } +} diff --git a/examples/v1/shiplog/anchor_min.json b/examples/v1/shiplog/anchor_min.json new file mode 100644 index 00000000..2a756c98 --- /dev/null +++ b/examples/v1/shiplog/anchor_min.json @@ -0,0 +1,8 @@ +{ + "ulid": "01HF4Y9Q1SM8Q7K9DK2R3V4AWB", + "topic": "prod", + "head": "8b1c1e4f3c9a0b5d7e2c1a4f6b8d0c3e5f7a9b1c", + "label": "q4-rollup", + "created_at": "2025-11-10T10:05:00Z" +} + diff --git a/examples/v1/shiplog/checkpoint_commit_only_invalid.json b/examples/v1/shiplog/checkpoint_commit_only_invalid.json new file mode 100644 index 00000000..fce69d9b --- /dev/null +++ b/examples/v1/shiplog/checkpoint_commit_only_invalid.json @@ -0,0 +1,3 @@ +{ + "commit_oid": "8b1c1e4f3c9a0b5d7e2c1a4f6b8d0c3e5f7a9b1c" +} diff --git a/examples/v1/shiplog/checkpoint_min.json b/examples/v1/shiplog/checkpoint_min.json new file mode 100644 index 00000000..52d790f2 --- /dev/null +++ b/examples/v1/shiplog/checkpoint_min.json @@ -0,0 +1,5 @@ +{ + "ulid": "01HF4Y9Q1SM8Q7K9DK2R3V4AWB", + "commit_oid": "8b1c1e4f3c9a0b5d7e2c1a4f6b8d0c3e5f7a9b1c" +} + diff --git a/examples/v1/shiplog/checkpoint_ulid_only_invalid.json b/examples/v1/shiplog/checkpoint_ulid_only_invalid.json new file mode 100644 index 00000000..fad17359 --- /dev/null +++ b/examples/v1/shiplog/checkpoint_ulid_only_invalid.json @@ -0,0 +1,3 @@ +{ + "ulid": "01HF4Y9Q1SM8Q7K9DK2R3V4AWB" +} diff --git a/examples/v1/shiplog/event_min.json b/examples/v1/shiplog/event_min.json new file mode 100644 index 00000000..142bc19d --- /dev/null +++ b/examples/v1/shiplog/event_min.json @@ -0,0 +1,12 @@ +{ + "ulid": "01HF4Y9Q1SM8Q7K9DK2R3V4AWB", + "type": "proposal.created", + "payload": { + "title": "Q4 Budget", + "amount": 100000 + }, + "refs": { + "state": "blake3:0000000000000000000000000000000000000000000000000000000000000000" + }, + "ns": "governance" +} diff --git a/examples/v1/shiplog/event_mismatch_ns.json b/examples/v1/shiplog/event_mismatch_ns.json new file mode 100644 index 00000000..e742dc35 --- /dev/null +++ b/examples/v1/shiplog/event_mismatch_ns.json @@ -0,0 +1,7 @@ +{ + "ulid": "01HF4Y9Q1SM8Q7K9DK2R3V4AWB", + "ns": "wrongspace", + "type": "proposal.created", + "payload": { "title": "Q4 Budget", "amount": 100000 } +} + diff --git a/examples/v1/shiplog/trailer_min.json b/examples/v1/shiplog/trailer_min.json new file mode 100644 index 00000000..30055d15 --- /dev/null +++ b/examples/v1/shiplog/trailer_min.json @@ -0,0 +1,9 @@ +{ + "version": 1, + "env": "prod", + "who": { "name": "Jane Dev", "email": "jane@example.com" }, + "what": { "service": "web", "artifact": "ghcr.io/acme/web:1.2.3" }, + "status": "success", + "when": { "start_ts": "2025-11-10T10:00:00Z", "end_ts": "2025-11-10T10:01:10Z", "dur_s": 70 } +} + diff --git a/schemas/v1/policy/governance_policy.schema.json b/schemas/v1/policy/governance_policy.schema.json index 5c738b7d..42ff6baa 100644 --- a/schemas/v1/policy/governance_policy.schema.json +++ b/schemas/v1/policy/governance_policy.schema.json @@ -46,5 +46,39 @@ } } } + , + "privacy": { + "type": "object", + "additionalProperties": false, + "properties": { + "classes": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "properties": { + "min_entropy_bits": { "type": "integer", "minimum": 0 }, + "publish_plaintext_digest": { "type": "boolean" }, + "require_ciphertext_digest": { "type": "boolean" } + } + } + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "select": { "type": "string" }, + "action": { "type": "string", "enum": ["redact", "pointerize"] }, + "class": { "type": "string" }, + "capability": { "type": "string", "format": "uri" }, + "location": { "type": "string", "format": "uri" } + }, + "required": ["select", "action"] + } + } + } + } } } diff --git a/schemas/v1/privacy/opaque_pointer.schema.json b/schemas/v1/privacy/opaque_pointer.schema.json new file mode 100644 index 00000000..40dd81c1 --- /dev/null +++ b/schemas/v1/privacy/opaque_pointer.schema.json @@ -0,0 +1,108 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "GATOS Opaque Pointer", + "description": "Canonical pointer to a private blob used in public projections.", + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "opaque_pointer" + }, + "algo": { + "type": "string", + "const": "blake3" + }, + "digest": { + "type": "string", + "pattern": "^blake3:[a-f0-9]{64}$" + }, + "ciphertext_digest": { + "type": "string", + "pattern": "^blake3:[a-f0-9]{64}$" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "location": { + "type": "string", + "format": "uri" + }, + "capability": { + "type": "string", + "format": "uri" + }, + "extensions": { + "type": "object", + "properties": { + "class": { + "type": "string" + } + } + } + }, + "required": [ + "kind", + "algo", + "location", + "capability" + ], + "anyOf": [ + { + "required": [ + "digest" + ], + "properties": { + "digest": { + "type": "string", + "pattern": "^blake3:[a-f0-9]{64}$" + } + } + }, + { + "required": [ + "ciphertext_digest" + ], + "properties": { + "ciphertext_digest": { + "type": "string", + "pattern": "^blake3:[a-f0-9]{64}$" + } + } + } + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "extensions": { + "properties": { + "class": { + "const": "low-entropy" + } + }, + "required": [ + "class" + ], + "type": "object" + } + }, + "required": [ + "extensions" + ], + "type": "object" + }, + "then": { + "required": [ + "ciphertext_digest" + ], + "not": { + "required": [ + "digest" + ] + } + } + } + ] +} diff --git a/schemas/v1/shiplog/anchor.schema.json b/schemas/v1/shiplog/anchor.schema.json new file mode 100644 index 00000000..2407b5fa --- /dev/null +++ b/schemas/v1/shiplog/anchor.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://gatos.dev/schemas/v1/shiplog/anchor.schema.json", + "title": "GATOS Shiplog Anchor (v1)", + "type": "object", + "additionalProperties": false, + "required": ["ulid", "topic", "head"], + "properties": { + "ulid": { "type": "string", "pattern": "^[0-9A-HJKMNP-TV-Z]{26}$" }, + "topic": { "type": "string", "pattern": "^[a-z][a-z0-9._-]{0,63}$" }, + "head": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "label": { "type": "string" }, + "created_at": { "type": "string", "format": "date-time" }, + "notes": { "type": "string" } + } +} + diff --git a/schemas/v1/shiplog/consumer_checkpoint.schema.json b/schemas/v1/shiplog/consumer_checkpoint.schema.json new file mode 100644 index 00000000..1d6847e0 --- /dev/null +++ b/schemas/v1/shiplog/consumer_checkpoint.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://gatos.dev/schemas/v1/shiplog/consumer_checkpoint.schema.json", + "title": "GATOS Shiplog Consumer Checkpoint (v1 Canonical JSON)", + "type": "object", + "additionalProperties": false, + "properties": { + "ulid": { + "type": "string", + "pattern": "^[0-9A-HJKMNP-TV-Z]{26}$" + }, + "commit_oid": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + } + }, + "required": ["ulid", "commit_oid"] +} diff --git a/schemas/v1/shiplog/deployment_trailer.schema.json b/schemas/v1/shiplog/deployment_trailer.schema.json new file mode 100644 index 00000000..46928db8 --- /dev/null +++ b/schemas/v1/shiplog/deployment_trailer.schema.json @@ -0,0 +1,137 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://gatos.dev/schemas/v1/shiplog/deployment_trailer.schema.json", + "title": "GATOS Shiplog Deployment Trailer (v1)", + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "status", + "when" + ], + "properties": { + "version": { + "type": "integer", + "const": 1 + }, + "env": { + "type": "string", + "minLength": 1 + }, + "who": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "what": { + "type": "object", + "additionalProperties": true, + "properties": { + "service": { + "type": "string" + }, + "artifact": { + "type": "string" + } + } + }, + "where": { + "type": "object", + "additionalProperties": true, + "properties": { + "region": { + "type": "string" + }, + "cluster": { + "type": "string" + }, + "namespace": { + "type": "string" + } + } + }, + "why": { + "type": "object", + "additionalProperties": true, + "properties": { + "reason": { + "type": "string" + }, + "ticket": { + "type": "string" + } + } + }, + "how": { + "type": "object", + "additionalProperties": true, + "properties": { + "pipeline": { + "type": "string" + }, + "run_url": { + "type": "string", + "format": "uri" + } + } + }, + "status": { + "type": "string", + "enum": [ + "success", + "failed", + "in_progress", + "skipped", + "override", + "revert", + "finalize" + ] + }, + "when": { + "type": "object", + "additionalProperties": false, + "required": [ + "start_ts", + "end_ts", + "dur_s" + ], + "properties": { + "start_ts": { + "type": "string", + "format": "date-time" + }, + "end_ts": { + "type": "string", + "format": "date-time" + }, + "dur_s": { + "type": "number", + "minimum": 0 + } + } + }, + "seq": { + "type": "integer", + "minimum": 0 + }, + "journal_parent": { + "type": "string" + }, + "trust_oid": { + "type": "string" + }, + "previous_anchor": { + "type": "string" + }, + "repo_head": { + "type": "string" + } + } +} diff --git a/schemas/v1/shiplog/event_envelope.schema.json b/schemas/v1/shiplog/event_envelope.schema.json new file mode 100644 index 00000000..fbd93d76 --- /dev/null +++ b/schemas/v1/shiplog/event_envelope.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://gatos.dev/schemas/v1/shiplog/event_envelope.schema.json", + "title": "GATOS Shiplog Event Envelope (v1 Canonical JSON)", + "type": "object", + "additionalProperties": false, + "required": [ + "ulid", + "ns", + "type", + "payload" + ], + "properties": { + "ulid": { + "type": "string", + "pattern": "^[0-9A-HJKMNP-TV-Z]{26}$", + "description": "ULID (26 chars, Crockford base32, uppercase, no I/L/O/U)" + }, + "type": { + "type": "string", + "minLength": 1 + }, + "payload": { + "type": "object" + }, + "refs": { + "type": "object", + "propertyNames": { + "pattern": "^[a-z][a-z0-9._-]{0,63}$" + }, + "additionalProperties": { + "$ref": "../common/ids.schema.json#/$defs/blake3Digest" + }, + "description": "Optional cross-references to related state roots or objects (blake3:<64-hex>)." + }, + "ns": { + "type": "string", + "pattern": "^[a-z][a-z0-9._-]{0,63}$", + "description": "Topic namespace (ASCII, lowercase start)." + } + }, + "description": "Canonical Shiplog event envelope. Content-Id = blake3(JCS(envelope)). Precision‑sensitive values (e.g., money/time) MUST be encoded as integers or strings. Payload MUST NOT embed private data; replace with Opaque Pointer per privacy schema." +} diff --git a/scripts/killcheck/error_casing.sh b/scripts/killcheck/error_casing.sh new file mode 100755 index 00000000..d0b497a3 --- /dev/null +++ b/scripts/killcheck/error_casing.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Disallow common lowercase/underscore variants of error codes anywhere in docs +bad=$(rg -n "append_rejected|not_fast_forward|temporalorder|siginvalid|policydenied|notfound" docs || true) +if [[ -n "$bad" ]]; then + echo "Found non-canonical error code casing or names:" >&2 + echo "$bad" >&2 + exit 1 +fi +echo "ok: error code casing canonical" diff --git a/scripts/killcheck/schema_headers.sh b/scripts/killcheck/schema_headers.sh new file mode 100755 index 00000000..6e7640c4 --- /dev/null +++ b/scripts/killcheck/schema_headers.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Fail if any line contains 'Schema:' that is not 'Envelope-Schema:' or 'Trailer-Schema:' +if rg -n "\\bSchema:\\b" docs | rg -v "Envelope-Schema|Trailer-Schema" -n | sed -n '1,200p'; then + echo "Found legacy 'Schema:' header(s). Use Envelope-Schema and Trailer-Schema only." >&2 + exit 1 +fi +echo "ok: no legacy 'Schema:' headers" + diff --git a/scripts/killcheck/ulid_reference.sh b/scripts/killcheck/ulid_reference.sh new file mode 100755 index 00000000..3bc1e762 --- /dev/null +++ b/scripts/killcheck/ulid_reference.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +file="docs/decisions/ADR-0005/DECISION.md" +if ! rg -n "ULID Spec §4\.1 Monotonic Lexicographic Ordering" "$file" >/dev/null; then + echo "Missing external reference to 'ULID Spec §4.1 Monotonic Lexicographic Ordering' in ADR-0005." >&2 + exit 1 +fi +echo "ok: ULID external reference present" + diff --git a/scripts/mermaid/generate_all.sh b/scripts/mermaid/generate_all.sh old mode 100644 new mode 100755 diff --git a/scripts/shiplog/check_topic_ns.js b/scripts/shiplog/check_topic_ns.js new file mode 100755 index 00000000..6523d210 --- /dev/null +++ b/scripts/shiplog/check_topic_ns.js @@ -0,0 +1,30 @@ +#!/usr/bin/env node +// Simple invariant check: envelope.ns MUST equal the CLI/topic segment +// Usage: check_topic_ns.js +const fs = require('fs'); + +function usage() { + console.error('usage: check_topic_ns.js '); + process.exit(2); +} + +const [, , file, topic] = process.argv; +if (!file || !topic) usage(); + +let ns; +try { + const text = fs.readFileSync(file, 'utf8'); + const obj = JSON.parse(text); + ns = obj.ns; +} catch (e) { + console.error('failed to read/parse envelope:', e.message); + process.exit(2); +} + +if (ns !== topic) { + console.error(`topic/ns mismatch: topic=${topic} ns=${ns}`); + process.exit(1); +} +// match → success (no output) +process.exit(0); + diff --git a/scripts/validate_schemas.sh b/scripts/validate_schemas.sh index 6f0233d1..64c4366b 100755 --- a/scripts/validate_schemas.sh +++ b/scripts/validate_schemas.sh @@ -1,8 +1,16 @@ #!/usr/bin/env bash set -euo pipefail -echo "[schemas] Installing ajv-cli@5.0.0 and ajv-formats@3.0.1…" -npm i -g ajv-cli@5.0.0 ajv-formats@3.0.1 +# Resolve AJV CLI (prefer local npx; fallback to dockerized node) +AJV_RUNNER=() +if command -v node >/dev/null 2>&1; then + AJV_RUNNER=(npx -y ajv-cli@5) +elif command -v docker >/dev/null 2>&1; then + AJV_RUNNER=(docker run --rm -v "$PWD:/work" -w /work node:20 npx -y ajv-cli@5 ajv) +else + echo "Need Node.js or Docker to run AJV validation" >&2 + exit 1 +fi AJV_COMMON_REF="schemas/v1/common/ids.schema.json" AJV_BASE_ARGS=(--spec=draft2020 --strict=true -c ajv-formats) @@ -18,14 +26,15 @@ SCHEMAS=( "schemas/v1/governance/revocation.schema.json" "schemas/v1/governance/proof_of_consensus_envelope.schema.json" "schemas/v1/policy/governance_policy.schema.json" + "schemas/v1/privacy/opaque_pointer.schema.json" ) for schema in "${SCHEMAS[@]}"; do echo " - ajv compile: $schema" if [[ "$schema" == "$AJV_COMMON_REF" || "$schema" == "schemas/v1/policy/governance_policy.schema.json" ]]; then - ajv compile "${AJV_BASE_ARGS[@]}" -s "$schema" + "${AJV_RUNNER[@]}" compile "${AJV_BASE_ARGS[@]}" -s "$schema" else - ajv compile "${AJV_BASE_ARGS[@]}" -s "$schema" -r "$AJV_COMMON_REF" + "${AJV_RUNNER[@]}" compile "${AJV_BASE_ARGS[@]}" -s "$schema" -r "$AJV_COMMON_REF" fi done @@ -38,6 +47,7 @@ declare -A EXAMPLES=( ["schemas/v1/governance/grant.schema.json"]="examples/v1/governance/grant_min.json" ["schemas/v1/governance/revocation.schema.json"]="examples/v1/governance/revocation_min.json" ["schemas/v1/governance/proof_of_consensus_envelope.schema.json"]="examples/v1/governance/poc_envelope_min.json" + ["schemas/v1/privacy/opaque_pointer.schema.json"]="examples/v1/privacy/opaque_pointer_min.json" ) for schema in "${!EXAMPLES[@]}"; do @@ -47,11 +57,13 @@ for schema in "${!EXAMPLES[@]}"; do continue fi echo " - ajv validate: $data against $schema" - ajv validate "${AJV_BASE_ARGS[@]}" -s "$schema" -d "$data" -r "$AJV_COMMON_REF" + "${AJV_RUNNER[@]}" validate "${AJV_BASE_ARGS[@]}" -s "$schema" -d "$data" -r "$AJV_COMMON_REF" done echo " - ajv validate: examples/v1/policy/governance_min.json against schemas/v1/policy/governance_policy.schema.json" -ajv validate "${AJV_BASE_ARGS[@]}" -s schemas/v1/policy/governance_policy.schema.json -d examples/v1/policy/governance_min.json +"${AJV_RUNNER[@]}" validate "${AJV_BASE_ARGS[@]}" -s schemas/v1/policy/governance_policy.schema.json -d examples/v1/policy/governance_min.json +echo " - ajv validate: examples/v1/policy/privacy_min.json against schemas/v1/policy/governance_policy.schema.json" +"${AJV_RUNNER[@]}" validate "${AJV_BASE_ARGS[@]}" -s schemas/v1/policy/governance_policy.schema.json -d examples/v1/policy/privacy_min.json echo "[schemas] Additional encoding tests (ed25519 base64url forms)…" # Root schemas that reference defs using the canonical $id for proper resolution @@ -64,35 +76,35 @@ SIG_B64URL=$(node -e 'process.stdout.write(Buffer.alloc(64).toString("base64url" echo " - positive: base64url unpadded key ($(echo -n "$KEY_B64URL" | wc -c) chars)" printf '"ed25519:%s"' "$KEY_B64URL" > /tmp/key_b64url_unpadded.json -ajv validate "${AJV_BASE_ARGS[@]}" -s /tmp/ed25519Key.schema.json -d /tmp/key_b64url_unpadded.json -r "$AJV_COMMON_REF" +"${AJV_RUNNER[@]}" validate "${AJV_BASE_ARGS[@]}" -s /tmp/ed25519Key.schema.json -d /tmp/key_b64url_unpadded.json -r "$AJV_COMMON_REF" echo " - positive: base64url unpadded sig ($(echo -n "$SIG_B64URL" | wc -c) chars)" printf '"ed25519:%s"' "$SIG_B64URL" > /tmp/sig_b64url_unpadded.json -ajv validate "${AJV_BASE_ARGS[@]}" -s /tmp/ed25519Sig.schema.json -d /tmp/sig_b64url_unpadded.json -r "$AJV_COMMON_REF" +"${AJV_RUNNER[@]}" validate "${AJV_BASE_ARGS[@]}" -s /tmp/ed25519Sig.schema.json -d /tmp/sig_b64url_unpadded.json -r "$AJV_COMMON_REF" echo " - negative: 44-char base64url key without '=' should be rejected" KEY_BADLEN="${KEY_B64URL}A" # 43 -> 44 (no '=') printf '"ed25519:%s"' "$KEY_BADLEN" > /tmp/key_b64url_badlen.json -if ajv validate "${AJV_BASE_ARGS[@]}" -s /tmp/ed25519Key.schema.json -d /tmp/key_b64url_badlen.json -r "$AJV_COMMON_REF"; then +if "${AJV_RUNNER[@]}" validate "${AJV_BASE_ARGS[@]}" -s /tmp/ed25519Key.schema.json -d /tmp/key_b64url_badlen.json -r "$AJV_COMMON_REF"; then echo "[FAIL] Unexpected acceptance of bad key length (44 without '=')" >&2; exit 1 fi echo " - negative: 88-char base64url sig without '==' should be rejected" SIG_BADLEN="${SIG_B64URL}AA" # 86 -> 88 (no '==') printf '"ed25519:%s"' "$SIG_BADLEN" > /tmp/sig_b64url_badlen.json -if ajv validate "${AJV_BASE_ARGS[@]}" -s /tmp/ed25519Sig.schema.json -d /tmp/sig_b64url_badlen.json -r "$AJV_COMMON_REF"; then +if "${AJV_RUNNER[@]}" validate "${AJV_BASE_ARGS[@]}" -s /tmp/ed25519Sig.schema.json -d /tmp/sig_b64url_badlen.json -r "$AJV_COMMON_REF"; then echo "[FAIL] Unexpected acceptance of bad sig length (88 without '==')" >&2; exit 1 fi echo "[schemas] Negative tests (invalid ISO8601 durations)…" echo '{"governance":{"x":{"ttl":"P"}}}' > /tmp/bad1.json echo '{"governance":{"x":{"ttl":"PT"}}}' > /tmp/bad2.json -if ajv validate "${AJV_BASE_ARGS[@]}" -s schemas/v1/policy/governance_policy.schema.json -d /tmp/bad1.json; then +if "${AJV_RUNNER[@]}" validate "${AJV_BASE_ARGS[@]}" -s schemas/v1/policy/governance_policy.schema.json -d /tmp/bad1.json; then echo "[FAIL] Unexpected success: ttl=P should be rejected" >&2; exit 1 else echo " - rejected ttl=P as expected" fi -if ajv validate "${AJV_BASE_ARGS[@]}" -s schemas/v1/policy/governance_policy.schema.json -d /tmp/bad2.json; then +if "${AJV_RUNNER[@]}" validate "${AJV_BASE_ARGS[@]}" -s schemas/v1/policy/governance_policy.schema.json -d /tmp/bad2.json; then echo "[FAIL] Unexpected success: ttl=PT should be rejected" >&2; exit 1 else echo " - rejected ttl=PT as expected"