diff --git a/.golangci.yml b/.golangci.yml index 1dd64d1..2783633 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -47,6 +47,7 @@ linters: - G104 - G304 - G306 + - G703 # CLI reads/writes user-supplied paths from argv by design exclusions: rules: diff --git a/README.md b/README.md index 1288959..f0a93e5 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,20 @@ # okf -A Go CLI toolkit for the [Open Knowledge Format (OKF)](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md) — a vendor-neutral format for representing data catalog knowledge as plain markdown files with YAML frontmatter. +A Go CLI toolkit for the [Open Knowledge Format (OKF)](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md), a vendor-neutral format for representing data catalog knowledge as plain markdown files with YAML frontmatter. `okf` creates, validates, lints, indexes, searches, and inspects OKF knowledge bundles. One static binary, no runtime dependencies, fast enough to validate millions of concepts. ## Why okf? -Google's reference OKF implementation is Python + Gemini + BigQuery — vendor-locked to Google's cloud. `okf` is the vendor-neutral alternative: a single Go binary that works anywhere, speaks JSON natively, and is designed to be driven by any AI agent on any provider. +Google's reference OKF implementation is Python + Gemini + BigQuery, vendor-locked to Google's cloud. `okf` is the vendor-neutral alternative: a single Go binary that works anywhere, speaks JSON natively, and is designed to be driven by any AI agent on any provider. **Agentic-first** means an AI agent can discover, understand, and drive the entire CLI without reading documentation or scraping text output. Three mechanisms make this work: -1. **`okf schema`** — emits a complete machine-readable description of every command: name, description, flags, arguments, output format, exit codes. One call and the agent knows the full CLI surface. +1. **`okf schema`** emits a complete machine-readable description of every command: name, description, flags, arguments, output format, exit codes. One call and the agent knows the full CLI surface. -2. **JSON by default** — every command outputs structured JSON on stdout. No `--json` flag, no screen-scraping. Diagnostics go to stderr. +2. **JSON by default**: every command outputs structured JSON on stdout. No `--json` flag, no screen-scraping. Diagnostics go to stderr. -3. **Structured error envelopes** — all errors emit `{"error": {"kind":..., "code":..., "reason":..., "message":...}}` on stdout with a stable exit code. An agent can branch on the `kind` field to decide what to do next. +3. **Structured error envelopes**: all errors emit `{"error": {"kind":..., "code":..., "reason":..., "message":...}}` on stdout with a stable exit code. An agent can branch on the `kind` field to decide what to do next. ## Quick start @@ -37,7 +37,7 @@ okf graph ./my-bundle ### 1. AI-driven documentation pipeline -An AI agent creates a bundle, writes concept documents from a database schema or API spec, validates them, and generates navigation — all autonomously. +An AI agent creates a bundle, writes concept documents from a database schema or API spec, validates them, and generates navigation, all autonomously. ```bash okf init ./bundles/mydb # start from scratch @@ -117,6 +117,10 @@ Progressive disclosure (index.md) lets the agent navigate level by level instead | `okf search [--tag] [--type] [--text]` | Search concepts by tag, type, or text | | `okf backlinks ` | List concepts that link to a given concept | | `okf graph ` | Print cross-link graph with nodes, edges, and stats | +| `okf export [-o file]` | Export entire bundle as a deterministic .okf tar.gz archive | +| `okf sign keygen` | Generate an ML-DSA-65 post-quantum key pair | +| `okf sign sign --priv ` | Sign the archive with ML-DSA-65 (FIPS 204) | +| `okf sign verify --pub --sig ` | Verify the archive signature with the signer's public key | | `okf version` | Print version | ## Exit codes @@ -131,7 +135,7 @@ Progressive disclosure (index.md) lets the agent navigate level by level instead ## What is OKF? -OKF is an open format from Google for representing knowledge — the metadata, context, and curated insight that surrounds data and systems. A bundle is a directory of markdown files with YAML frontmatter: +OKF is an open format from Google for representing knowledge: the metadata, context, and curated insight that surrounds data and systems. A bundle is a directory of markdown files with YAML frontmatter: ``` my-bundle/ @@ -172,16 +176,58 @@ The format is intentionally minimal: no schema registry, no central authority, n ## Project status -Early development. The CLI surface is functional with 35 tests: +Early development. The CLI surface is functional with 73 tests: -- `schema`, `init`, `validate`, `lint`, `index`, `list`, `show`, `search`, `backlinks`, `graph`, `version` +- `schema`, `init`, `validate`, `lint`, `index`, `list`, `show`, `search`, `backlinks`, `graph`, `export`, `sign`, `version` Planned: -- `okf serve` — local HTTP server to browse a bundle interactively -- `okf render` — export a bundle as a self-contained HTML file -- `okf-go` — Go library package for embedding in applications +- `okf serve`: local HTTP server to browse a bundle interactively +- `okf render`: export a bundle as a self-contained HTML file +- `okf-go`: Go library package for embedding in applications + +## Export & Post-Quantum Signing + +### Export + +Export your entire bundle into a single deterministic `.okf` archive (tar.gz). +The archive is byte-reproducible: the same bundle contents always produce the +same archive bytes, which is essential for signing and verification. + +```bash +okf export ./my-bundle -o bundle.okf +``` + +The output includes a manifest with per-file SHA-256 hashes, total file count, +and the archive hash. + +### Post-Quantum Signing + +Sign archives with **ML-DSA-65** (FIPS 204, formerly CRYSTALS-Dilithium), a +NIST post-quantum digital signature standard, via +[cloudflare/circl](https://github.com/cloudflare/circl). + +The signer signs the archive's SHA-256 hash with the private key. Anyone with +the signer's public key can verify that the archive was signed by the key +holder and has not been modified since. The private key is stored as a 32-byte +seed; keep it secret and distribute only the public key. + +```bash +# 1. Generate an ML-DSA-65 key pair +okf sign bundle.okf keygen + +# 2. Sign the archive with your private key +okf sign bundle.okf sign --priv -o sig.json + +# 3. Anyone verifies with your public key +okf sign bundle.okf verify --pub --sig sig.json +``` + +The signature output includes the algorithm (`ML-DSA-65`), the hex-encoded +signature, the archive SHA-256, and the signer's public key. Verification +always uses the public key you pass on the command line, never the one +embedded in the signature file. ## License -Apache 2.0 — matching the upstream [Google knowledge-catalog](https://github.com/GoogleCloudPlatform/knowledge-catalog) repository. \ No newline at end of file +Apache 2.0, matching the upstream [Google knowledge-catalog](https://github.com/GoogleCloudPlatform/knowledge-catalog) repository. \ No newline at end of file diff --git a/cmd/okf/main.go b/cmd/okf/main.go index dd199ca..61a340a 100644 --- a/cmd/okf/main.go +++ b/cmd/okf/main.go @@ -17,11 +17,13 @@ import ( "github.com/okfcli/okf/internal/backlinks" "github.com/okfcli/okf/internal/bundle" "github.com/okfcli/okf/internal/cerr" + "github.com/okfcli/okf/internal/export" "github.com/okfcli/okf/internal/graph" "github.com/okfcli/okf/internal/index" "github.com/okfcli/okf/internal/initbundle" "github.com/okfcli/okf/internal/search" "github.com/okfcli/okf/internal/show" + "github.com/okfcli/okf/internal/sign" "github.com/okfcli/okf/internal/validate" ) @@ -66,12 +68,16 @@ func main() { runInit(rest) case "backlinks": runBacklinks(rest) + case "export": + runExport(rest) + case "sign": + runSign(rest) default: exitErr(cerr.Usage("unknown command: %s", cmd)) } } -const usage = `okf — Open Knowledge Format toolkit (v%s) +const usage = `okf - Open Knowledge Format toolkit (v%s) Usage: okf @@ -89,6 +95,8 @@ Commands: search [filters] Search concepts by tag, type, or text backlinks List concepts that link to a given concept graph Print cross-link graph statistics + export [-o file] Export entire bundle as a .okf tar.gz archive + sign Post-quantum sign/verify with ML-DSA-65 (FIPS 204) version Print version Exit codes: @@ -355,11 +363,11 @@ func runSearch(args []string) { } outputJSON(map[string]any{ - "command": "search", - "bundle": b.Root, - "filters": f, - "results": concepts, - "count": len(results), + "command": "search", + "bundle": b.Root, + "filters": f, + "results": concepts, + "count": len(results), }) } @@ -411,6 +419,185 @@ func runBacklinks(args []string) { }) } +// --- export --- + +func runExport(args []string) { + if len(args) == 0 { + exitErr(cerr.Usage("usage: okf export [-o ]")) + } + bundlePath := args[0] + outPath := bundlePath + ".okf" + for i := 1; i < len(args); i++ { + switch args[i] { + case "-o", "--output": + if i+1 >= len(args) { + exitErr(cerr.Usage("-o requires a value")) + } + outPath = args[i+1] + i++ + default: + exitErr(cerr.Usage("unknown export flag: %s", args[i])) + } + } + + manifest, err := export.Archive(bundlePath, outPath) + if err != nil { + exitErr(cerr.IO(err, "export bundle %s", bundlePath)) + } + + manifestJSON, err := export.ManifestToJSON(manifest) + if err != nil { + exitErr(cerr.Internal(err, "marshal manifest")) + } + + outputJSON(map[string]any{ + "command": "export", + "bundle": manifest.Bundle, + "archive": manifest.Archive, + "manifest": json.RawMessage(manifestJSON), + }) +} + +// --- sign --- + +func runSign(args []string) { + if len(args) == 0 { + exitErr(cerr.Usage("usage: okf sign [options]")) + } + archivePath := args[0] + if len(args) < 2 { + exitErr(cerr.Usage("usage: okf sign [options]")) + } + action := args[1] + rest := args[2:] + + switch action { + case "keygen": + runSignKeygen() + case "sign": + runSignSign(archivePath, rest) + case "verify": + runSignVerify(archivePath, rest) + default: + exitErr(cerr.Usage("unknown sign action: %s (use keygen, sign, or verify)", action)) + } +} + +func runSignKeygen() { + kp, err := sign.GenerateKeyPair() + if err != nil { + exitErr(cerr.Internal(err, "generate key pair")) + } + + kpJSON, err := sign.KeyPairToJSON(kp) + if err != nil { + exitErr(cerr.Internal(err, "marshal key pair")) + } + + outputJSON(map[string]any{ + "command": "sign", + "action": "keygen", + "algorithm": sign.Algorithm, + "keypair": json.RawMessage(kpJSON), + }) +} + +func runSignSign(archivePath string, rest []string) { + var privKeyHex, sigOutPath string + for i := 0; i < len(rest); i++ { + switch rest[i] { + case "--priv", "--private-key": + if i+1 >= len(rest) { + exitErr(cerr.Usage("--priv requires a value")) + } + privKeyHex = rest[i+1] + i++ + case "-o", "--output": + if i+1 >= len(rest) { + exitErr(cerr.Usage("-o requires a value")) + } + sigOutPath = rest[i+1] + i++ + default: + exitErr(cerr.Usage("unknown sign flag: %s", rest[i])) + } + } + if privKeyHex == "" { + exitErr(cerr.Usage("usage: okf sign sign --priv [-o ]")) + } + + sig, err := sign.Sign(archivePath, privKeyHex) + if err != nil { + exitErr(cerr.Internal(err, "sign archive %s", archivePath)) + } + + sigJSON, err := sign.SignatureToJSON(sig) + if err != nil { + exitErr(cerr.Internal(err, "marshal signature")) + } + + // If -o is given, write the signature to a file for later verification. + if sigOutPath != "" { + if err := os.WriteFile(sigOutPath, sigJSON, 0o644); err != nil { + exitErr(cerr.IO(err, "write signature %s", sigOutPath)) + } + } + + outputJSON(map[string]any{ + "command": "sign", + "action": "sign", + "archive": archivePath, + "signature": json.RawMessage(sigJSON), + }) +} + +func runSignVerify(archivePath string, rest []string) { + var pubKeyHex, sigPath string + for i := 0; i < len(rest); i++ { + switch rest[i] { + case "--pub", "--public-key": + if i+1 >= len(rest) { + exitErr(cerr.Usage("--pub requires a value")) + } + pubKeyHex = rest[i+1] + i++ + case "--sig", "--signature": + if i+1 >= len(rest) { + exitErr(cerr.Usage("--sig requires a value")) + } + sigPath = rest[i+1] + i++ + default: + exitErr(cerr.Usage("unknown verify flag: %s", rest[i])) + } + } + if pubKeyHex == "" || sigPath == "" { + exitErr(cerr.Usage("usage: okf sign verify --pub --sig ")) + } + + sigData, err := os.ReadFile(sigPath) + if err != nil { + exitErr(cerr.IO(err, "read signature %s", sigPath)) + } + + var sig sign.Signature + if err := json.Unmarshal(sigData, &sig); err != nil { + exitErr(cerr.Validation("parse signature: %s", err)) + } + + if err := sign.Verify(archivePath, &sig, pubKeyHex); err != nil { + exitErr(cerr.Validation("%s", err)) + } + + outputJSON(map[string]any{ + "command": "sign", + "action": "verify", + "archive": archivePath, + "verified": true, + "algorithm": sig.Algorithm, + }) +} + // --- schema --- // schemaFlag describes a single flag for machine consumption. @@ -441,10 +628,10 @@ type schemaCommand struct { // schemaRoot is the top-level schema output. type schemaRoot struct { - Name string `json:"name"` - Version string `json:"version"` - Description string `json:"description"` - Commands []schemaCommand `json:"commands"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Commands []schemaCommand `json:"commands"` ExitCodes []cerr.ExitCodeDoc `json:"exit_codes"` } @@ -474,68 +661,68 @@ func buildSchemaRoot() schemaRoot { func allSchemaCommands() []schemaCommand { return []schemaCommand{ { - Name: "schema", - Short: "Print machine-readable CLI metadata as JSON", - Long: "Outputs a JSON document describing every command, its flags, arguments, output format, and exit codes. Pass a command name to describe just that command.", - Args: []schemaArg{{Name: "command", Required: false}}, - Stdout: "json", + Name: "schema", + Short: "Print machine-readable CLI metadata as JSON", + Long: "Outputs a JSON document describing every command, its flags, arguments, output format, and exit codes. Pass a command name to describe just that command.", + Args: []schemaArg{{Name: "command", Required: false}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeUsage}, }, { - Name: "init", - Short: "Create a new empty OKF bundle", - Long: "Creates a bundle directory with standard subdirectories (tables, datasets, playbooks), a root index.md, and a .gitignore. Fails if the directory already exists.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "init", + Short: "Create a new empty OKF bundle", + Long: "Creates a bundle directory with standard subdirectories (tables, datasets, playbooks), a root index.md, and a .gitignore. Fails if the directory already exists.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "validate", - Short: "Validate a bundle against the OKF spec", - Long: "Checks every concept for required frontmatter (type), recommended fields (title, description, tags), non-empty body, and valid cross-links. Exits 1 if any errors are found.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "validate", + Short: "Validate a bundle against the OKF spec", + Long: "Checks every concept for required frontmatter (type), recommended fields (title, description, tags), non-empty body, and valid cross-links. Exits 1 if any errors are found.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeValidation, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "lint", - Short: "Check recommended fields and style (warnings only)", - Long: "Same checks as validate but only emits warnings — errors are suppressed. Exits 0 even with warnings.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "lint", + Short: "Check recommended fields and style (warnings only)", + Long: "Same checks as validate but only emits warnings; errors are suppressed. Exits 0 even with warnings.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "index", - Short: "Generate index.md files (progressive disclosure)", - Long: "Writes index.md into every directory containing concept documents, providing progressive disclosure per OKF spec §6.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "index", + Short: "Generate index.md files (progressive disclosure)", + Long: "Writes index.md into every directory containing concept documents, providing progressive disclosure per OKF spec §6.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "list", - Short: "List all concepts in the bundle", - Long: "Lists every concept document with its ID, type, and title.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "list", + Short: "List all concepts in the bundle", + Long: "Lists every concept document with its ID, type, and title.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "show", - Short: "Show a single concept's full content", - Long: "Returns the concept's ID, file path, frontmatter (type, title, description, resource, tags), and markdown body as JSON.", + Name: "show", + Short: "Show a single concept's full content", + Long: "Returns the concept's ID, file path, frontmatter (type, title, description, resource, tags), and markdown body as JSON.", Args: []schemaArg{ {Name: "bundle", Required: true}, {Name: "concept-id", Required: true}, }, - Stdout: "json", + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeValidation, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "search", - Short: "Search concepts by tag, type, or text", - Long: "Filters concepts in a bundle by tag (--tag), frontmatter type (--type), or full-text search in title, description, and body (--text). Multiple filters are AND-combined. With no filters, returns all concepts.", + Name: "search", + Short: "Search concepts by tag, type, or text", + Long: "Filters concepts in a bundle by tag (--tag), frontmatter type (--type), or full-text search in title, description, and body (--text). Multiple filters are AND-combined. With no filters, returns all concepts.", Flags: []schemaFlag{ {Name: "tag", Type: "string", Default: "", Description: "filter by tag (case-insensitive)"}, {Name: "type", Type: "string", Default: "", Description: "filter by frontmatter type (case-insensitive)"}, @@ -544,32 +731,59 @@ func allSchemaCommands() []schemaCommand { Args: []schemaArg{ {Name: "bundle", Required: true}, }, - Stdout: "json", + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "backlinks", - Short: "List concepts that link to a given concept", - Long: "Returns the IDs of all concepts in the bundle that contain a markdown link to the specified concept. Deduplicates multiple links from the same source.", + Name: "backlinks", + Short: "List concepts that link to a given concept", + Long: "Returns the IDs of all concepts in the bundle that contain a markdown link to the specified concept. Deduplicates multiple links from the same source.", Args: []schemaArg{ {Name: "bundle", Required: true}, {Name: "concept-id", Required: true}, }, - Stdout: "json", + Stdout: "json", + ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, + }, + { + Name: "graph", + Short: "Print cross-link graph statistics", + Long: "Builds the directed cross-link graph from concept markdown links and prints nodes, edges, and summary statistics.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "graph", - Short: "Print cross-link graph statistics", - Long: "Builds the directed cross-link graph from concept markdown links and prints nodes, edges, and summary statistics.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "export", + Short: "Export entire bundle as a .okf tar.gz archive", + Long: "Creates a deterministic tar.gz archive of every file in the bundle (sorted by path for reproducibility). Outputs a manifest with per-file SHA-256 hashes and total archive hash.", + Flags: []schemaFlag{ + {Name: "output", Short: "o", Type: "string", Default: ".okf", Description: "output archive path"}, + }, + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "version", - Short: "Print version", - Stdout: "json", + Name: "sign", + Short: "Post-quantum sign/verify archives with ML-DSA-65 (FIPS 204)", + Long: "Subcommands: 'keygen' generates an ML-DSA-65 key pair. 'sign' signs the archive's SHA-256 hash with the private key. 'verify' checks the signature with the signer's public key and confirms the archive is untampered. Uses ML-DSA-65 (FIPS 204), a NIST post-quantum digital signature standard.", + Args: []schemaArg{ + {Name: "archive", Required: true}, + {Name: "action", Required: true}, + }, + Flags: []schemaFlag{ + {Name: "priv", Type: "string", Default: "", Description: "hex-encoded ML-DSA-65 private key seed (for sign)"}, + {Name: "pub", Type: "string", Default: "", Description: "hex-encoded ML-DSA-65 public key (for verify)"}, + {Name: "sig", Type: "string", Default: "", Description: "path to signature JSON file (for verify)"}, + }, + Stdout: "json", + ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeValidation, cerr.ExitCodeInternal, cerr.ExitCodeIO, cerr.ExitCodeUsage}, + }, + { + Name: "version", + Short: "Print version", + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK}, }, } diff --git a/go.mod b/go.mod index 4416cae..044e14d 100644 --- a/go.mod +++ b/go.mod @@ -3,3 +3,8 @@ module github.com/okfcli/okf go 1.26.4 require gopkg.in/yaml.v3 v3.0.1 + +require ( + github.com/cloudflare/circl v1.6.4 + golang.org/x/sys v0.38.0 // indirect +) diff --git a/go.sum b/go.sum index a62c313..b53db0b 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U= +github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/export/export.go b/internal/export/export.go new file mode 100644 index 0000000..4255d57 --- /dev/null +++ b/internal/export/export.go @@ -0,0 +1,161 @@ +// Package export bundles an entire OKF knowledge bundle into a single +// self-contained archive file. The archive is a deterministic tar of every +// file inside the bundle root (sorted by path), so the same bundle always +// produces the same byte-identical archive, which is essential for signing. +package export + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// Manifest describes the contents of an exported archive. +type Manifest struct { + Bundle string `json:"bundle"` // absolute path to the bundle root + Files []ManifestFile `json:"files"` // every file in the archive + FileCount int `json:"file_count"` // total files + TotalBytes int64 `json:"total_bytes"` // sum of file sizes + SHA256 string `json:"sha256"` // hex digest of the archive bytes + Archive string `json:"archive"` // path to the .okf file + Created string `json:"created"` // RFC 3339 timestamp +} + +// ManifestFile is a single entry in the manifest. +type ManifestFile struct { + Path string `json:"path"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} + +// Archive creates a deterministic .okf (tar.gz) archive of the bundle at root. +// The output is written to outPath. Returns a Manifest describing the archive. +func Archive(root, outPath string) (*Manifest, error) { + absRoot, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("resolve %s: %w", root, err) + } + info, err := os.Stat(absRoot) + if err != nil { + return nil, fmt.Errorf("stat bundle root: %w", err) + } + if !info.IsDir() { + return nil, fmt.Errorf("bundle root %s is not a directory", absRoot) + } + + // Collect all files (sorted for determinism). + files, err := collectFiles(absRoot) + if err != nil { + return nil, err + } + + // Write the archive to a buffer first so we can compute the SHA-256. + var buf bytes.Buffer + h := sha256.New() + mw := io.MultiWriter(&buf, h) + + gw := gzip.NewWriter(mw) + tw := tar.NewWriter(gw) + + var manifestFiles []ManifestFile + var totalBytes int64 + + for _, relPath := range files { + absPath := filepath.Join(absRoot, relPath) + data, err := os.ReadFile(absPath) + if err != nil { + return nil, fmt.Errorf("read %s: %w", relPath, err) + } + size := int64(len(data)) + + fileHash := sha256.Sum256(data) + manifestFiles = append(manifestFiles, ManifestFile{ + Path: filepath.ToSlash(relPath), + Size: size, + SHA256: hex.EncodeToString(fileHash[:]), + }) + totalBytes += size + + hdr := &tar.Header{ + Name: filepath.ToSlash(relPath), + Size: size, + Mode: 0o644, // fixed mode for cross-machine reproducibility + ModTime: time.Time{}, // zero time for determinism + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, fmt.Errorf("tar header %s: %w", relPath, err) + } + if _, err := tw.Write(data); err != nil { + return nil, fmt.Errorf("tar write %s: %w", relPath, err) + } + } + + if err := tw.Close(); err != nil { + return nil, fmt.Errorf("close tar writer: %w", err) + } + if err := gw.Close(); err != nil { + return nil, fmt.Errorf("close gzip writer: %w", err) + } + + archiveHash := h.Sum(nil) + + // Write to output file. + if err := os.WriteFile(outPath, buf.Bytes(), 0o644); err != nil { + return nil, fmt.Errorf("write archive %s: %w", outPath, err) + } + + return &Manifest{ + Bundle: absRoot, + Files: manifestFiles, + FileCount: len(manifestFiles), + TotalBytes: totalBytes, + SHA256: hex.EncodeToString(archiveHash), + Archive: outPath, + Created: time.Now().UTC().Format(time.RFC3339), + }, nil +} + +// collectFiles returns a sorted list of relative file paths inside root. +// Hidden directories (starting with .) are skipped. +func collectFiles(root string) ([]string, error) { + var paths []string + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if path != root && strings.HasPrefix(d.Name(), ".") { + return filepath.SkipDir + } + return nil + } + relPath, err := filepath.Rel(root, path) + if err != nil { + return fmt.Errorf("rel path %s: %w", path, err) + } + paths = append(paths, relPath) + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(paths, func(i, j int) bool { + return paths[i] < paths[j] + }) + return paths, nil +} + +// ManifestToJSON returns the manifest as pretty-printed JSON. +func ManifestToJSON(m *Manifest) ([]byte, error) { + return json.MarshalIndent(m, "", " ") +} diff --git a/internal/export/export_test.go b/internal/export/export_test.go new file mode 100644 index 0000000..0004b84 --- /dev/null +++ b/internal/export/export_test.go @@ -0,0 +1,111 @@ +package export + +import ( + "os" + "path/filepath" + "testing" +) + +func TestArchive_CreatesArchiveAndManifest(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "tables", "users.md"), "---\ntype: Table\ntitle: Users\n---\n\n# Users\n") + writeFile(t, filepath.Join(root, "tables", "orders.md"), "---\ntype: Table\ntitle: Orders\n---\n\n# Orders\n") + writeFile(t, filepath.Join(root, "index.md"), "# Index\n") + + outPath := filepath.Join(t.TempDir(), "bundle.okf") + manifest, err := Archive(root, outPath) + if err != nil { + t.Fatalf("Archive: %v", err) + } + + if manifest.FileCount != 3 { + t.Errorf("file_count = %d, want 3", manifest.FileCount) + } + if manifest.SHA256 == "" { + t.Error("sha256 is empty") + } + + info, err := os.Stat(outPath) + if err != nil { + t.Fatalf("archive not written: %v", err) + } + if info.Size() == 0 { + t.Error("archive is empty") + } +} + +func TestArchive_Deterministic(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "a.md"), "---\ntype: T\ntitle: A\n---\n\nbody A\n") + writeFile(t, filepath.Join(root, "b.md"), "---\ntype: T\ntitle: B\n---\n\nbody B\n") + + out1 := filepath.Join(t.TempDir(), "a1.okf") + out2 := filepath.Join(t.TempDir(), "a2.okf") + + m1, err := Archive(root, out1) + if err != nil { + t.Fatalf("first Archive: %v", err) + } + m2, err := Archive(root, out2) + if err != nil { + t.Fatalf("second Archive: %v", err) + } + + if m1.SHA256 != m2.SHA256 { + t.Errorf("non-deterministic: %s != %s", m1.SHA256, m2.SHA256) + } +} + +func TestArchive_SkipsHiddenDirs(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "visible.md"), "---\ntype: T\ntitle: V\n---\n\nbody\n") + writeFile(t, filepath.Join(root, ".git", "config"), "hidden\n") + + outPath := filepath.Join(t.TempDir(), "out.okf") + manifest, err := Archive(root, outPath) + if err != nil { + t.Fatalf("Archive: %v", err) + } + if manifest.FileCount != 1 { + t.Errorf("file_count = %d, want 1 (hidden dir should be skipped)", manifest.FileCount) + } +} + +func TestArchive_NotADirectory(t *testing.T) { + f := filepath.Join(t.TempDir(), "notdir.txt") + writeFile(t, f, "data") + _, err := Archive(f, f+".okf") + if err == nil { + t.Fatal("expected error for non-directory") + } +} + +func TestArchive_ManifestFileHashes(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "a.md"), "---\ntype: T\ntitle: A\n---\n\nbody\n") + + outPath := filepath.Join(t.TempDir(), "out.okf") + manifest, err := Archive(root, outPath) + if err != nil { + t.Fatalf("Archive: %v", err) + } + if len(manifest.Files) != 1 { + t.Fatalf("files = %d, want 1", len(manifest.Files)) + } + if manifest.Files[0].SHA256 == "" { + t.Error("file sha256 is empty") + } + if manifest.Files[0].Path != "a.md" { + t.Errorf("path = %q, want a.md", manifest.Files[0].Path) + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} diff --git a/internal/sign/sign.go b/internal/sign/sign.go new file mode 100644 index 0000000..bd0fa85 --- /dev/null +++ b/internal/sign/sign.go @@ -0,0 +1,146 @@ +// Package sign provides post-quantum digital signatures for OKF bundle +// exports using ML-DSA-65 (FIPS 204, formerly CRYSTALS-Dilithium). The signer +// holds the private key and signs the archive's SHA-256 hash; anyone with the +// public key can verify that the archive was signed by the key holder and has +// not been modified since. +package sign + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/cloudflare/circl/sign/mldsa/mldsa65" +) + +// Algorithm identifies the signature scheme used by this package. +const Algorithm = "ML-DSA-65" + +// signContext domain-separates OKF archive signatures from any other use of +// the same key (FIPS 204 context string). +var signContext = []byte("okf-pq-sign") + +// KeyPair is a generated ML-DSA-65 key pair. The private key is stored as the +// 32-byte FIPS 204 seed, from which the full key is deterministically derived. +type KeyPair struct { + PublicKey string `json:"public_key"` // hex-encoded ML-DSA-65 public key + PrivateKey string `json:"private_key"` // hex-encoded 32-byte seed +} + +// Signature is the result of signing an archive. +type Signature struct { + Algorithm string `json:"algorithm"` // "ML-DSA-65" + Signature string `json:"signature"` // hex-encoded ML-DSA-65 signature + ArchiveSHA256 string `json:"archive_sha256"` // hex SHA-256 of the archive + PublicKey string `json:"public_key"` // hex public key of the signer (informational) +} + +// GenerateKeyPair creates a new ML-DSA-65 key pair from a random seed. +func GenerateKeyPair() (*KeyPair, error) { + var seed [mldsa65.SeedSize]byte + if _, err := rand.Read(seed[:]); err != nil { + return nil, fmt.Errorf("generate ML-DSA-65 seed: %w", err) + } + pk, _ := mldsa65.NewKeyFromSeed(&seed) + return &KeyPair{ + PublicKey: hex.EncodeToString(pk.Bytes()), + PrivateKey: hex.EncodeToString(seed[:]), + }, nil +} + +// Sign signs the SHA-256 hash of the archive with ML-DSA-65. +// privKeyHex is the hex-encoded 32-byte seed from GenerateKeyPair. +func Sign(archivePath, privKeyHex string) (*Signature, error) { + hash, err := hashFile(archivePath) + if err != nil { + return nil, err + } + + seedBytes, err := hex.DecodeString(privKeyHex) + if err != nil { + return nil, fmt.Errorf("decode private key: %w", err) + } + if len(seedBytes) != mldsa65.SeedSize { + return nil, fmt.Errorf("private key must be a %d-byte seed, got %d bytes", mldsa65.SeedSize, len(seedBytes)) + } + var seed [mldsa65.SeedSize]byte + copy(seed[:], seedBytes) + pk, sk := mldsa65.NewKeyFromSeed(&seed) + + sig := make([]byte, mldsa65.SignatureSize) + if err := mldsa65.SignTo(sk, hash, signContext, false, sig); err != nil { + return nil, fmt.Errorf("sign archive hash: %w", err) + } + + return &Signature{ + Algorithm: Algorithm, + Signature: hex.EncodeToString(sig), + ArchiveSHA256: hex.EncodeToString(hash), + PublicKey: hex.EncodeToString(pk.Bytes()), + }, nil +} + +// Verify checks the ML-DSA-65 signature over the archive's SHA-256 hash. +// pubKeyHex is the trusted hex-encoded public key of the expected signer; the +// public key embedded in the signature file is never used for verification. +func Verify(archivePath string, sig *Signature, pubKeyHex string) error { + if sig.Algorithm != Algorithm { + return fmt.Errorf("unsupported algorithm %q (expected %q)", sig.Algorithm, Algorithm) + } + + hash, err := hashFile(archivePath) + if err != nil { + return err + } + + pubBytes, err := hex.DecodeString(pubKeyHex) + if err != nil { + return fmt.Errorf("decode public key: %w", err) + } + var pk mldsa65.PublicKey + if err := pk.UnmarshalBinary(pubBytes); err != nil { + return fmt.Errorf("parse public key: %w", err) + } + + sigBytes, err := hex.DecodeString(sig.Signature) + if err != nil { + return fmt.Errorf("decode signature: %w", err) + } + + if !mldsa65.Verify(&pk, hash, signContext, sigBytes) { + return fmt.Errorf("signature verification failed: archive was tampered with or signed by a different key") + } + + return nil +} + +// hashFile streams the file through SHA-256. +func hashFile(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("read archive %s: %w", path, err) + } + defer func() { _ = f.Close() }() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, fmt.Errorf("hash archive %s: %w", path, err) + } + return h.Sum(nil), nil +} + +// KeyPairToJSON returns the key pair as pretty-printed JSON. Emitting the +// private key is the purpose of keygen; the caller is responsible for +// storing it securely. +func KeyPairToJSON(kp *KeyPair) ([]byte, error) { + return json.MarshalIndent(kp, "", " ") // #nosec G117 -- keygen intentionally outputs the private key +} + +// SignatureToJSON returns the signature as pretty-printed JSON. +func SignatureToJSON(sig *Signature) ([]byte, error) { + return json.MarshalIndent(sig, "", " ") +} diff --git a/internal/sign/sign_test.go b/internal/sign/sign_test.go new file mode 100644 index 0000000..3816ad5 --- /dev/null +++ b/internal/sign/sign_test.go @@ -0,0 +1,166 @@ +package sign + +import ( + "os" + "path/filepath" + "testing" +) + +func writeArchive(t *testing.T, content string) string { + t.Helper() + archivePath := filepath.Join(t.TempDir(), "bundle.okf") + if err := os.WriteFile(archivePath, []byte(content), 0o644); err != nil { + t.Fatalf("write archive: %v", err) + } + return archivePath +} + +func TestGenerateKeyPair(t *testing.T) { + kp, err := GenerateKeyPair() + if err != nil { + t.Fatalf("GenerateKeyPair: %v", err) + } + if kp.PublicKey == "" { + t.Error("public key is empty") + } + if kp.PrivateKey == "" { + t.Error("private key is empty") + } + if kp.PublicKey == kp.PrivateKey { + t.Error("public and private keys are identical") + } +} + +func TestSignAndVerify(t *testing.T) { + kp, err := GenerateKeyPair() + if err != nil { + t.Fatalf("GenerateKeyPair: %v", err) + } + + archivePath := writeArchive(t, "fake archive content") + + sig, err := Sign(archivePath, kp.PrivateKey) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + if sig.Algorithm != Algorithm { + t.Errorf("algorithm = %q", sig.Algorithm) + } + if sig.Signature == "" { + t.Error("signature is empty") + } + if sig.ArchiveSHA256 == "" { + t.Error("archive_sha256 is empty") + } + if sig.PublicKey != kp.PublicKey { + t.Error("signature public key does not match key pair") + } + + if err := Verify(archivePath, sig, kp.PublicKey); err != nil { + t.Fatalf("Verify: %v", err) + } +} + +func TestVerify_TamperedArchive(t *testing.T) { + kp, err := GenerateKeyPair() + if err != nil { + t.Fatalf("GenerateKeyPair: %v", err) + } + + archivePath := writeArchive(t, "original content") + + sig, err := Sign(archivePath, kp.PrivateKey) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + // Tamper with the archive. + if err := os.WriteFile(archivePath, []byte("TAMPERED content"), 0o644); err != nil { + t.Fatalf("write tampered archive: %v", err) + } + + if err := Verify(archivePath, sig, kp.PublicKey); err == nil { + t.Fatal("expected error for tampered archive, got nil") + } +} + +func TestVerify_WrongKey(t *testing.T) { + kp1, err := GenerateKeyPair() + if err != nil { + t.Fatalf("GenerateKeyPair 1: %v", err) + } + kp2, err := GenerateKeyPair() + if err != nil { + t.Fatalf("GenerateKeyPair 2: %v", err) + } + + archivePath := writeArchive(t, "archive data") + + sig, err := Sign(archivePath, kp1.PrivateKey) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + // Verify against a different signer's public key. + if err := Verify(archivePath, sig, kp2.PublicKey); err == nil { + t.Fatal("expected error for wrong key, got nil") + } +} + +// TestVerify_ForgedSignature is the attack the previous ML-KEM/HPKE design +// could not resist: an attacker who tampers with the archive and knows only +// the signer's PUBLIC key tries to mint a fresh signature over the modified +// archive. With a real signature scheme, signing requires the private key, so +// the best the attacker can do is tamper with the signature bytes. +func TestVerify_ForgedSignature(t *testing.T) { + kp, err := GenerateKeyPair() + if err != nil { + t.Fatalf("GenerateKeyPair: %v", err) + } + + archivePath := writeArchive(t, "original content") + + sig, err := Sign(archivePath, kp.PrivateKey) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + // Attacker cannot call Sign with the public key. + if _, err := Sign(archivePath, kp.PublicKey); err == nil { + t.Fatal("Sign accepted a public key as the signing key") + } + + // Attacker flips a byte in the signature instead. + forged := *sig + raw := []byte(forged.Signature) + if raw[0] == 'a' { + raw[0] = 'b' + } else { + raw[0] = 'a' + } + forged.Signature = string(raw) + + if err := Verify(archivePath, &forged, kp.PublicKey); err == nil { + t.Fatal("expected error for forged signature, got nil") + } +} + +func TestVerify_WrongAlgorithm(t *testing.T) { + kp, err := GenerateKeyPair() + if err != nil { + t.Fatalf("GenerateKeyPair: %v", err) + } + + archivePath := writeArchive(t, "archive data") + + sig, err := Sign(archivePath, kp.PrivateKey) + if err != nil { + t.Fatalf("Sign: %v", err) + } + + sig.Algorithm = "ML-KEM-768/HPKE-SHA256" + if err := Verify(archivePath, sig, kp.PublicKey); err == nil { + t.Fatal("expected error for wrong algorithm, got nil") + } +}