From f1a34f92636303681990f5150e225c04e26c5f62 Mon Sep 17 00:00:00 2001 From: Larri Date: Mon, 13 Jul 2026 11:06:57 -0600 Subject: [PATCH 1/3] feat: add provider-neutral secret values --- Cargo.lock | 1 + Cargo.toml | 1 + docs/STDLIB_REFERENCE.md | 82 ++++- src/interpreter.rs | 110 ++++++ src/lib.rs | 3 + src/main.rs | 1 + src/secret.rs | 77 ++++ src/std_secrets_tests.rs | 425 ++++++++++++++++++++++ src/stdlib/collections.rs | 5 + src/stdlib/csv.rs | 10 + src/stdlib/http.rs | 140 +++++-- src/stdlib/http_server.rs | 5 +- src/stdlib/http_server_async.rs | 25 +- src/stdlib/jobs.rs | 30 ++ src/stdlib/json.rs | 94 ++++- src/stdlib/kv.rs | 49 ++- src/stdlib/log.rs | 12 + src/stdlib/mod.rs | 2 + src/stdlib/secrets.rs | 601 +++++++++++++++++++++++++++++++ src/stdlib/string.rs | 74 ++-- src/stdlib/url.rs | 35 +- src/typechecker.rs | 37 ++ src/types.rs | 5 + tests/language_features_tests.rs | 146 ++++++++ tests/type_checker_tests.rs | 58 +++ 25 files changed, 1909 insertions(+), 119 deletions(-) create mode 100644 src/secret.rs create mode 100644 src/std_secrets_tests.rs create mode 100644 src/stdlib/secrets.rs diff --git a/Cargo.lock b/Cargo.lock index d2efa7cd..38073e1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1950,6 +1950,7 @@ dependencies = [ "uuid", "webpki-root-certs", "x509-parser", + "zeroize", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 14496aa9..d2886eb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,6 +99,7 @@ jsonwebtoken = "9" pulldown-cmark = "0.13.0" hickory-resolver = "0.24.4" socket2 = "0.6" +zeroize = "1.8.2" [build-dependencies] # Doc comment scanner (build.rs extracts /// @ntnt blocks) diff --git a/docs/STDLIB_REFERENCE.md b/docs/STDLIB_REFERENCE.md index afbde377..ca9a29e9 100644 --- a/docs/STDLIB_REFERENCE.md +++ b/docs/STDLIB_REFERENCE.md @@ -26,6 +26,7 @@ - [std/net](#stdnet) - [std/path](#stdpath) - [std/postgres](#stdpostgres) +- [std/secrets](#stdsecrets) - [std/sqlite](#stdsqlite) - [std/string](#stdstring) - [std/time](#stdtime) @@ -6732,7 +6733,7 @@ fetch(url_or_options: String | Map, options?: Map) -> Result Make an HTTP request to a URL. -Accepts one or two arguments: - One argument: a URL string for a simple GET request, or an options map with full control over method, headers, body, authentication, cookies, and timeout. - Two arguments: a URL string and an options map. The URL is merged into the options map automatically. Options map keys: url (set automatically in 2-arg form), method, headers, body, json, form, auth, cookies, timeout. +Accepts one or two arguments: - One argument: a URL string for a simple GET request, or an options map with full control over method, headers, body, authentication, cookies, and timeout. - Two arguments: a URL string and an options map. The URL is merged into the options map automatically. Options map keys: url (set automatically in 2-arg form), method, headers, body, json, form, auth, cookies, timeout. Opaque Secret values are accepted only in header values, cookie values, basic-auth fields, raw bodies, JSON leaves, and form values. Requests containing a Secret do not follow redirects, preventing credentials or 307/308 bodies from crossing origins. **Parameters:** @@ -10880,6 +10881,85 @@ rollback(db) // => true // Roll back an active transaction --- +## std/secrets + +Provider-neutral secret lookup with opaque, redacted values + +```ntnt +import { get_secret, require_secret } from "std/secrets" +``` + +### Functions + +| Function | Description | +|----------|-------------| +| [`get_secret`](#getsecret) | Looks up a secret by its provider-neutral logical name. | +| [`require_secret`](#requiresecret) | Looks up a required secret and fails closed when it is not configured. | + +#### `get_secret` + +```ntnt +get_secret(name: String) -> Option +``` + +Looks up a secret by its provider-neutral logical name. + +The v0.5.1 development-only environment provider reads the exact environment variable name and is disabled when `NTNT_ENV` is `production` or `prod`. Projects with an `ntnt.toml` must declare accessible names under `[secrets.]`; undeclared lookups fail before contacting the provider. Declaration metadata may contain `label`, `description`, `required`, and `environments`; secret values are never accepted in the manifest. Secret values remain opaque and redact themselves in output and diagnostics. + +**Parameters:** + +- `name` — A validated logical secret name + +**Returns:** Some(Secret) when configured, otherwise None + +**Examples:** + +```ntnt +get_secret("STRIPE_SECRET_KEY") // => Some([REDACTED]) // Optional lookup +``` + +**Errors:** + +- **RuntimeError**: Unsupported secrets provider — *Fix: Set NTNT_SECRETS_PROVIDER=env for v0.5.1* + +**See also:** `require_secret` + +*Since v0.5.1* + +--- + +#### `require_secret` + +```ntnt +require_secret(name: String) -> Secret +``` + +Looks up a required secret and fails closed when it is not configured. + +Use this at startup or immediately before an approved secret-consuming sink. In v0.5.1, `std/http.fetch` accepts Secret values as header, cookie, basic-auth, raw body, JSON-leaf, and form values. Templates, public JSON, URL/CSV/string conversion, KV storage, and job payloads reject secrets. There is intentionally no general Secret-to-String reveal function. The error identifies only the logical name and never includes the value. + +**Parameters:** + +- `name` — A validated logical secret name + +**Returns:** The opaque Secret value + +**Examples:** + +```ntnt +require_secret("STRIPE_SECRET_KEY") // => [REDACTED] // Required lookup +``` + +**Errors:** + +- **RuntimeError**: Required secret is not configured — *Fix: Configure the named secret in the selected provider* + +**See also:** `get_secret` + +*Since v0.5.1* + +--- + ## std/sqlite SQLite database operations diff --git a/src/interpreter.rs b/src/interpreter.rs index 5d8b43c7..9759400b 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -14,6 +14,7 @@ use crate::ast::*; use crate::config::{get_type_mode, type_warn_dedup, TypeMode}; use crate::contracts::{ContractChecker, OldValues, StoredValue}; use crate::error::{IntentError, Result, TypeContext}; +pub use crate::secret::SecretValue; use std::cell::RefCell; use std::collections::HashMap; use std::fmt; @@ -57,6 +58,9 @@ pub enum Value { /// String value String(String), + /// Opaque secret value. Ordinary formatting is always redacted. + Secret(SecretValue), + /// Array value Array(Vec), @@ -205,6 +209,7 @@ impl Value { Value::Float(_) => true, // Empty collections are falsy Value::String(s) => !s.is_empty(), + Value::Secret(_) => true, Value::Array(a) => !a.is_empty(), Value::Map(m) => !m.is_empty(), // None is falsy, Some(x) is truthy @@ -223,6 +228,7 @@ impl Value { Value::Float(_) => "Float", Value::Bool(_) => "Bool", Value::String(_) => "String", + Value::Secret(_) => "Secret", Value::Array(_) => "Array", Value::Map(_) => "Map", Value::Range { .. } => "Range", @@ -240,6 +246,19 @@ impl Value { Value::Continue => "Continue", } } + + /// Return true when this value directly or recursively contains a secret. + pub fn contains_secret(&self) -> bool { + match self { + Value::Secret(_) => true, + Value::Array(values) => values.iter().any(Value::contains_secret), + Value::Map(values) => values.values().any(Value::contains_secret), + Value::Struct { fields, .. } => fields.values().any(Value::contains_secret), + Value::EnumValue { values, .. } => values.iter().any(Value::contains_secret), + Value::Return(value) => value.contains_secret(), + _ => false, + } + } } impl fmt::Display for Value { @@ -250,6 +269,7 @@ impl fmt::Display for Value { Value::Float(n) => write!(f, "{}", n), Value::Bool(b) => write!(f, "{}", b), Value::String(s) => write!(f, "{}", s), + Value::Secret(secret) => write!(f, "{}", secret), Value::Array(arr) => { let items: Vec = arr.iter().map(|v| v.to_string()).collect(); write!(f, "[{}]", items.join(", ")) @@ -1749,6 +1769,8 @@ impl Interpreter { /// Set the main source file for hot-reload tracking pub fn set_main_source_file(&mut self, path: &str) { self.main_source_file = Some(path.to_string()); + #[cfg(not(test))] + crate::stdlib::secrets::configure_for_source(path); // Store the current mtime self.main_source_mtime = std::fs::metadata(path).ok().and_then(|m| m.modified().ok()); } @@ -8435,6 +8457,16 @@ impl Interpreter { result: &mut String, ) -> Result { match self.eval_expression(condition) { + Ok(v) if v.contains_secret() => { + Self::handle_template_error( + IntentError::type_error( + "Secret values cannot be used in template conditions".to_string(), + ), + context, + result, + )?; + Ok(false) + } Ok(v) => Ok(v.is_truthy()), Err(IntentError::UndefinedVariable { .. }) => Ok(false), Err(e) => { @@ -8458,6 +8490,16 @@ impl Interpreter { // forgiving → render empty string silently match self.eval_expression(expr) { Ok(v) => { + if v.contains_secret() { + Self::handle_template_error( + IntentError::type_error( + "Secret values cannot be rendered in templates".to_string(), + ), + "expression", + &mut result, + )?; + continue 'parts; + } let s = v.to_string(); result.push_str(&html_escape_string(&s)); } @@ -8470,6 +8512,16 @@ impl Interpreter { // Error boundary: behaviour depends on NTNT_TYPE_MODE. match self.eval_expression(expr) { Ok(v) => { + if v.contains_secret() { + Self::handle_template_error( + IntentError::type_error( + "Secret values cannot be rendered in templates".to_string(), + ), + "raw expression", + &mut result, + )?; + continue 'parts; + } result.push_str(&v.to_string()); } // Undefined template variables render as empty string @@ -8499,6 +8551,17 @@ impl Interpreter { } } }; + if value.contains_secret() { + Self::handle_template_error( + IntentError::type_error( + "Secret values cannot be passed through template filters" + .to_string(), + ), + "filtered expression", + &mut result, + )?; + continue 'parts; + } let mut skip_escape = false; for filter in filters { if filter.name == "safe" || filter.name == "raw" { @@ -8512,6 +8575,16 @@ impl Interpreter { } } } + if value.contains_secret() { + Self::handle_template_error( + IntentError::type_error( + "Secret values cannot be rendered in templates".to_string(), + ), + "filtered expression result", + &mut result, + )?; + continue 'parts; + } let s = value.to_string(); if skip_escape { result.push_str(&s); @@ -8544,6 +8617,17 @@ impl Interpreter { } } }; + if value.contains_secret() { + Self::handle_template_error( + IntentError::type_error( + "Secret values cannot be passed through template filters" + .to_string(), + ), + "raw filtered expression", + &mut result, + )?; + continue 'parts; + } for filter in filters { match self.apply_template_filter(&value, filter) { Ok(v) => value = v, @@ -8553,6 +8637,16 @@ impl Interpreter { } } } + if value.contains_secret() { + Self::handle_template_error( + IntentError::type_error( + "Secret values cannot be rendered in templates".to_string(), + ), + "raw filtered expression result", + &mut result, + )?; + continue 'parts; + } result.push_str(&value.to_string()); } TemplatePart::ForLoop { @@ -8574,6 +8668,16 @@ impl Interpreter { continue; } }; + if iterable_value.contains_secret() { + Self::handle_template_error( + IntentError::type_error( + "Secret values cannot be iterated in templates".to_string(), + ), + "for-loop iterable", + &mut result, + )?; + continue 'parts; + } match iterable_value { Value::Array(ref items) if items.is_empty() => { @@ -10959,6 +11063,12 @@ impl Interpreter { } fn eval_binary_op(&self, op: BinaryOp, lhs: Value, rhs: Value) -> Result { + if lhs.contains_secret() || rhs.contains_secret() { + return Err(IntentError::type_error( + "Secret values cannot be used with binary operators".to_string(), + )); + } + // Handle EnumValue and handle type equality if matches!(op, BinaryOp::Eq | BinaryOp::Ne) { let lhs_is_enum = matches!(&lhs, Value::EnumValue { .. }); diff --git a/src/lib.rs b/src/lib.rs index 9d5e758c..5f3588b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,9 @@ pub mod intent_studio_server; pub mod interpreter; pub mod lexer; pub mod parser; +mod secret; +#[cfg(test)] +mod std_secrets_tests; pub mod stdlib; pub mod typechecker; pub mod types; diff --git a/src/main.rs b/src/main.rs index 69a49ebe..0ac4cf52 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1196,6 +1196,7 @@ fn print_repl_help() { " {} get_env, set_env, load_env, args, cwd", "std/env".cyan() ); + println!(" {} get_secret, require_secret", "std/secrets".cyan()); println!( " {} join, dirname, basename, extname", "std/path".cyan() diff --git a/src/secret.rs b/src/secret.rs new file mode 100644 index 00000000..3c7db5ff --- /dev/null +++ b/src/secret.rs @@ -0,0 +1,77 @@ +//! Opaque runtime storage for secret values. + +use crate::error::{IntentError, Result}; +use std::fmt; +use std::sync::Arc; +use zeroize::Zeroizing; + +/// Redacted marker used by every ordinary formatting path. +pub const REDACTED_SECRET: &str = "[REDACTED]"; + +/// A secret value with a validated, non-sensitive logical name. +/// +/// Clones share one zeroizing allocation. Approved outbound sinks may borrow the +/// plaintext, but ordinary formatting and debugging never expose it. +#[derive(Clone)] +pub struct SecretValue { + name: Arc, + value: Arc>, +} + +impl SecretValue { + /// Construct a secret after validating its logical name. + pub(crate) fn new(name: impl Into, value: impl Into) -> Result { + let name = name.into(); + validate_secret_name(&name)?; + Ok(Self { + name: Arc::from(name), + value: Arc::new(Zeroizing::new(value.into())), + }) + } + + /// The validated, non-sensitive logical name. + pub fn name(&self) -> &str { + &self.name + } + + /// Borrow plaintext for an audited sink. Keep visibility crate-local. + pub(crate) fn expose(&self) -> &str { + self.value.as_str() + } +} + +impl fmt::Debug for SecretValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SecretValue") + .field("name", &self.name) + .field("value", &REDACTED_SECRET) + .finish() + } +} + +impl fmt::Display for SecretValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(REDACTED_SECRET) + } +} + +/// Validate provider-neutral secret identifiers before they reach diagnostics or providers. +pub fn validate_secret_name(name: &str) -> Result<()> { + if name.is_empty() || name.len() > 128 { + return Err(IntentError::type_error( + "Secret names must contain between 1 and 128 characters".to_string(), + )); + } + + let mut chars = name.chars(); + let first = chars.next().expect("non-empty checked above"); + if !(first.is_ascii_alphabetic() || first == '_') + || !chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-')) + { + return Err(IntentError::type_error( + "Secret names must match [A-Za-z_][A-Za-z0-9_.-]*".to_string(), + )); + } + + Ok(()) +} diff --git a/src/std_secrets_tests.rs b/src/std_secrets_tests.rs new file mode 100644 index 00000000..94541a62 --- /dev/null +++ b/src/std_secrets_tests.rs @@ -0,0 +1,425 @@ +use crate::interpreter::{SecretValue, Value}; +use crate::stdlib::json::{intent_value_to_json, intent_value_to_json_reject}; +use crate::stdlib::secrets; +use base64::Engine; +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::{Mutex, OnceLock}; +use std::thread; +use std::time::{Duration, Instant}; + +const SECRET_CANARY: &str = "s3cr3t-canary-value"; + +#[test] +fn secret_value_is_redacted_in_display_and_debug_output() { + let secret = SecretValue::new("TEST_SECRET", SECRET_CANARY).expect("valid secret"); + let value = Value::Secret(secret); + + let display = value.to_string(); + let debug = format!("{value:?}"); + + assert_eq!(display, "[REDACTED]"); + assert!(debug.contains("[REDACTED]")); + assert!(!debug.contains(SECRET_CANARY)); +} + +#[test] +fn secret_value_has_distinct_runtime_type_and_is_truthy() { + let value = + Value::Secret(SecretValue::new("TEST_SECRET", SECRET_CANARY).expect("valid secret")); + + assert_eq!(value.type_name(), "Secret"); + assert!(value.is_truthy()); +} + +#[test] +fn contains_secret_finds_nested_secret_values() { + let secret = + Value::Secret(SecretValue::new("TEST_SECRET", SECRET_CANARY).expect("valid secret")); + let mut nested = HashMap::new(); + nested.insert( + "payload".to_string(), + Value::Array(vec![Value::some(secret)]), + ); + + assert!(Value::Map(nested).contains_secret()); + assert!(!Value::Array(vec![Value::String("safe".to_string())]).contains_secret()); +} + +fn env_lock() -> std::sync::MutexGuard<'static, ()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + .lock() + .expect("environment test lock") +} + +fn native_fn(module: &HashMap, name: &str) -> fn(&[Value]) -> crate::Result { + match module.get(name) { + Some(Value::NativeFunction { func, .. }) => *func, + other => panic!("expected native function {name}, got {other:?}"), + } +} + +#[test] +fn secret_names_reject_unsafe_diagnostic_content() { + for invalid in ["", "1STARTS_WITH_DIGIT", "HAS SPACE", "HAS\nNEWLINE"] { + let err = SecretValue::new(invalid, SECRET_CANARY).expect_err("name must be rejected"); + assert!(!err.to_string().contains(SECRET_CANARY)); + } + + let too_long = "A".repeat(129); + SecretValue::new(too_long, SECRET_CANARY).expect_err("overlong name must be rejected"); +} + +#[test] +fn std_secrets_gets_an_optional_redacted_secret_from_env() { + let _guard = env_lock(); + let name = "NTNT_TEST_STD_SECRET_OPTIONAL"; + std::env::remove_var("NTNT_SECRETS_PROVIDER"); + std::env::set_var(name, SECRET_CANARY); + + let module = secrets::init(); + let value = native_fn(&module, "get_secret")(&[Value::String(name.to_string())]) + .expect("get_secret succeeds"); + + assert!(value.contains_secret()); + assert!(!format!("{value:?}").contains(SECRET_CANARY)); + + std::env::remove_var(name); +} + +#[test] +fn std_secrets_distinguishes_missing_and_required_values() { + let _guard = env_lock(); + let name = "NTNT_TEST_STD_SECRET_MISSING"; + std::env::remove_var("NTNT_SECRETS_PROVIDER"); + std::env::remove_var(name); + + let module = secrets::init(); + let optional = native_fn(&module, "get_secret")(&[Value::String(name.to_string())]) + .expect("optional lookup succeeds"); + assert!(matches!( + optional, + Value::EnumValue { + ref enum_name, + ref variant, + .. + } if enum_name == "Option" && variant == "None" + )); + + let err = native_fn(&module, "require_secret")(&[Value::String(name.to_string())]) + .expect_err("required lookup must fail"); + assert!(err.to_string().contains(name)); + assert!(!err.to_string().contains(SECRET_CANARY)); +} + +#[test] +fn unsupported_provider_fails_without_falling_back_to_env() { + let _guard = env_lock(); + let name = "NTNT_TEST_STD_SECRET_NO_FALLBACK"; + std::env::set_var(name, SECRET_CANARY); + std::env::set_var("NTNT_SECRETS_PROVIDER", "unsupported"); + + let module = secrets::init(); + let err = native_fn(&module, "get_secret")(&[Value::String(name.to_string())]) + .expect_err("unsupported provider must fail closed"); + let rendered = err.to_string(); + assert!(rendered.to_lowercase().contains("unsupported")); + assert!(!rendered.contains(SECRET_CANARY)); + + std::env::remove_var("NTNT_SECRETS_PROVIDER"); + std::env::remove_var(name); +} + +#[test] +fn defensive_json_conversion_recursively_redacts_secrets() { + let mut payload = HashMap::new(); + payload.insert( + "token".to_string(), + Value::some(Value::Secret( + SecretValue::new("API_KEY", SECRET_CANARY).expect("valid secret"), + )), + ); + + let json = intent_value_to_json(&Value::Map(payload)); + let rendered = json.to_string(); + assert!(rendered.contains("[REDACTED]")); + assert!(!rendered.contains(SECRET_CANARY)); +} + +#[test] +fn public_json_conversion_rejects_nested_secrets() { + let value = Value::Array(vec![Value::Map(HashMap::from([( + "token".to_string(), + Value::Secret(SecretValue::new("API_KEY", SECRET_CANARY).expect("valid secret")), + )]))]); + + let err = intent_value_to_json_reject(&value).expect_err("secret must not serialize"); + let rendered = err.to_string(); + assert!(rendered.contains("Secret")); + assert!(!rendered.contains(SECRET_CANARY)); +} + +#[test] +fn stringify_and_server_json_reject_secret_values() { + let secret = Value::Secret(SecretValue::new("API_KEY", SECRET_CANARY).expect("valid secret")); + + let json_module = crate::stdlib::json::init(); + let stringify_err = native_fn(&json_module, "stringify")(&[secret.clone()]) + .expect_err("stringify must reject secrets"); + assert!(!stringify_err.to_string().contains(SECRET_CANARY)); + + let server_module = crate::stdlib::http_server::init(); + let response_err = native_fn(&server_module, "json")(&[Value::Map(HashMap::from([( + "token".to_string(), + secret, + )]))]) + .expect_err("server JSON must reject secrets"); + assert!(!response_err.to_string().contains(SECRET_CANARY)); +} + +fn fetch_and_capture(mut options: HashMap) -> (Value, String) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind local listener"); + let address = listener.local_addr().expect("listener address"); + options.insert( + "url".to_string(), + Value::String(format!("http://{address}/capture")), + ); + + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + let capture = thread::spawn(move || { + let started = Instant::now(); + let (mut stream, _) = loop { + match listener.accept() { + Ok(connection) => break connection, + Err(error) + if error.kind() == std::io::ErrorKind::WouldBlock + && started.elapsed() < Duration::from_secs(5) => + { + thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("accept request: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("set timeout"); + let mut bytes = Vec::new(); + let mut buffer = [0_u8; 4096]; + let mut expected_len = None; + + loop { + let count = stream.read(&mut buffer).expect("read request"); + if count == 0 { + break; + } + bytes.extend_from_slice(&buffer[..count]); + + if expected_len.is_none() { + if let Some(header_end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&bytes[..header_end]); + let content_len = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0); + expected_len = Some(header_end + 4 + content_len); + } + } + + if expected_len.is_some_and(|length| bytes.len() >= length) { + break; + } + } + + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .expect("write response"); + String::from_utf8(bytes).expect("request is utf-8") + }); + + let module = crate::stdlib::http::init(); + let result = native_fn(&module, "fetch")(&[Value::Map(options)]).expect("fetch executes"); + (result, capture.join().expect("capture thread")) +} + +fn test_secret() -> Value { + Value::Secret(SecretValue::new("API_KEY", SECRET_CANARY).expect("valid secret")) +} + +#[test] +fn http_fetch_exposes_secrets_only_in_approved_header_auth_cookie_and_json_fields() { + let mut options = HashMap::new(); + options.insert("method".to_string(), Value::String("POST".to_string())); + options.insert( + "headers".to_string(), + Value::Map(HashMap::from([("x-api-key".to_string(), test_secret())])), + ); + options.insert( + "cookies".to_string(), + Value::Map(HashMap::from([("session".to_string(), test_secret())])), + ); + options.insert( + "auth".to_string(), + Value::Map(HashMap::from([ + ("user".to_string(), Value::String("client".to_string())), + ("pass".to_string(), test_secret()), + ])), + ); + options.insert( + "json".to_string(), + Value::Map(HashMap::from([("token".to_string(), test_secret())])), + ); + + let (_result, request) = fetch_and_capture(options); + let lower = request.to_lowercase(); + assert!(lower.contains(&format!("x-api-key: {}", SECRET_CANARY))); + assert!(lower.contains(&format!("cookie: session={}", SECRET_CANARY))); + let basic = base64::engine::general_purpose::STANDARD + .encode(format!("client:{SECRET_CANARY}").as_bytes()) + .to_lowercase(); + assert!(lower.contains(&format!("authorization: basic {basic}"))); + assert!(request.contains(&format!(r#"{{"token":"{}"}}"#, SECRET_CANARY))); + assert!(!request.contains("[REDACTED]")); +} + +#[test] +fn http_fetch_exposes_secret_as_raw_body() { + let options = HashMap::from([ + ("method".to_string(), Value::String("POST".to_string())), + ("body".to_string(), test_secret()), + ]); + + let (_result, request) = fetch_and_capture(options); + assert!(request.ends_with(SECRET_CANARY)); + assert!(!request.contains("[REDACTED]")); +} + +#[test] +fn http_fetch_exposes_secret_as_form_value() { + let options = HashMap::from([ + ("method".to_string(), Value::String("POST".to_string())), + ( + "form".to_string(), + Value::Map(HashMap::from([("token".to_string(), test_secret())])), + ), + ]); + + let (_result, request) = fetch_and_capture(options); + assert!(request.ends_with(&format!("token={SECRET_CANARY}"))); + assert!(!request.contains("[REDACTED]")); +} + +#[test] +fn http_fetch_does_not_follow_redirects_when_request_contains_secrets() { + let target = TcpListener::bind("127.0.0.1:0").expect("bind redirect target"); + let target_addr = target.local_addr().expect("target address"); + target.set_nonblocking(true).expect("nonblocking target"); + let target_request = thread::spawn(move || { + for _ in 0..100 { + match target.accept() { + Ok((mut stream, _)) => { + let mut request = Vec::new(); + stream + .set_read_timeout(Some(Duration::from_secs(1))) + .expect("target read timeout"); + stream.read_to_end(&mut request).ok(); + stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", + ) + .ok(); + return Some(String::from_utf8_lossy(&request).to_string()); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("target accept failed: {error}"), + } + } + None + }); + + let redirect = TcpListener::bind("127.0.0.1:0").expect("bind redirect server"); + let redirect_addr = redirect.local_addr().expect("redirect address"); + let redirect_server = thread::spawn(move || { + let (mut stream, _) = redirect.accept().expect("accept redirect request"); + let response = format!( + "HTTP/1.1 307 Temporary Redirect\r\nLocation: http://{target_addr}/target\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + stream + .write_all(response.as_bytes()) + .expect("write redirect"); + }); + + let mut options = HashMap::new(); + options.insert( + "url".to_string(), + Value::String(format!("http://{redirect_addr}/start")), + ); + options.insert( + "headers".to_string(), + Value::Map(HashMap::from([("x-api-key".to_string(), test_secret())])), + ); + let module = crate::stdlib::http::init(); + native_fn(&module, "fetch")(&[Value::Map(options)]).expect("fetch redirect response"); + + redirect_server.join().expect("redirect server"); + let forwarded = target_request.join().expect("target thread"); + assert!( + forwarded.is_none(), + "secret-bearing request followed redirect: {forwarded:?}" + ); +} + +#[test] +fn generic_stringification_sinks_reject_secrets_without_leaking_them() { + let cases = [ + ( + "csv.stringify", + native_fn(&crate::stdlib::csv::init(), "stringify")(&[Value::Array(vec![ + Value::Array(vec![test_secret()]), + ])]), + ), + ( + "url.build_query", + native_fn(&crate::stdlib::url::init(), "build_query")(&[Value::Map(HashMap::from([ + ("token".to_string(), test_secret()), + ]))]), + ), + ( + "string.join", + native_fn(&crate::stdlib::string::init(), "join")(&[ + Value::Array(vec![test_secret()]), + Value::String(",".to_string()), + ]), + ), + ( + "string.concat", + native_fn(&crate::stdlib::string::init(), "concat")(&[ + Value::Array(vec![test_secret()]), + Value::String(String::new()), + ]), + ), + ( + "collections.sort", + native_fn(&crate::stdlib::collections::init(), "sort")(&[Value::Array(vec![ + test_secret(), + ])]), + ), + ]; + + for (name, result) in cases { + let error = result.expect_err(&format!("{name} must reject secrets")); + let rendered = error.to_string(); + assert!(rendered.contains("Secret"), "{name}: {rendered}"); + assert!(!rendered.contains(SECRET_CANARY), "{name}: {rendered}"); + } +} diff --git a/src/stdlib/collections.rs b/src/stdlib/collections.rs index fa1ccc86..8cd55eb7 100644 --- a/src/stdlib/collections.rs +++ b/src/stdlib/collections.rs @@ -271,6 +271,11 @@ pub fn init() -> HashMap { "sort() requires 1 or 2 arguments (arr, key_or_fn?)".to_string(), )); } + if args[0].contains_secret() { + return Err(IntentError::type_error( + "sort() cannot compare Secret values".to_string(), + )); + } match &args[0] { Value::Array(arr) => { diff --git a/src/stdlib/csv.rs b/src/stdlib/csv.rs index 45373dbb..9d99c728 100644 --- a/src/stdlib/csv.rs +++ b/src/stdlib/csv.rs @@ -149,6 +149,11 @@ pub fn init() -> HashMap { max_arity: 1, requires: None, func: |args| { + if args[0].contains_secret() { + return Err(IntentError::type_error( + "csv.stringify() cannot serialize Secret values".to_string(), + )); + } let rows = match &args[0] { Value::Array(arr) => arr, _ => { @@ -220,6 +225,11 @@ pub fn init() -> HashMap { max_arity: 2, requires: None, func: |args| { + if args[0].contains_secret() || args[1].contains_secret() { + return Err(IntentError::type_error( + "csv.stringify_with_headers() cannot serialize Secret values".to_string(), + )); + } let rows = match &args[0] { Value::Array(arr) => arr, _ => { diff --git a/src/stdlib/http.rs b/src/stdlib/http.rs index c01464f7..17568eb1 100644 --- a/src/stdlib/http.rs +++ b/src/stdlib/http.rs @@ -21,7 +21,7 @@ use crate::error::IntentError; use crate::interpreter::Value; -use crate::stdlib::json::intent_value_to_json; +use crate::stdlib::json::intent_value_to_json_expose; use base64::Engine; use reqwest::header::{AUTHORIZATION, COOKIE, SET_COOKIE}; use std::collections::HashMap; @@ -624,6 +624,31 @@ fn http_get(url: &str) -> Result { } } +fn secret_or_string<'a>(value: &'a Value, field: &str) -> Result<&'a str> { + match value { + Value::String(value) => Ok(value), + Value::Secret(secret) => Ok(secret.expose()), + other => Err(IntentError::type_error(format!( + "fetch() {field} must be a String or Secret, got {}", + other.type_name() + ))), + } +} + +fn form_scalar(value: &Value) -> Result { + match value { + Value::String(value) => Ok(value.clone()), + Value::Secret(secret) => Ok(secret.expose().to_string()), + Value::Int(value) => Ok(value.to_string()), + Value::Float(value) => Ok(value.to_string()), + Value::Bool(value) => Ok(value.to_string()), + other => Err(IntentError::type_error(format!( + "fetch() form values must be scalar or Secret, got {}", + other.type_name() + ))), + } +} + /// Full HTTP request with all options fn http_fetch(opts: &HashMap) -> Result { // Cancellation yield point (rule 19): check before making the network request @@ -649,13 +674,23 @@ fn http_fetch(opts: &HashMap) -> Result { } let method = match opts.get("method") { - Some(Value::String(m)) => m.to_uppercase(), - _ => "GET".to_string(), + Some(Value::String(method)) => method.to_uppercase(), + None => "GET".to_string(), + Some(other) => { + return Err(IntentError::type_error(format!( + "fetch() method must be a String, got {}", + other.type_name() + ))) + } }; - // Build client with cookie store - let client = reqwest::blocking::Client::builder() - .cookie_store(true) + // Secret-bearing requests never follow redirects: custom credentials and + // 307/308 bodies could otherwise be forwarded to a different origin. + let mut client_builder = reqwest::blocking::Client::builder().cookie_store(true); + if opts.values().any(Value::contains_secret) { + client_builder = client_builder.redirect(reqwest::redirect::Policy::none()); + } + let client = client_builder .build() .map_err(|e| IntentError::runtime_error(format!("Failed to create HTTP client: {}", e)))?; @@ -674,57 +709,67 @@ fn http_fetch(opts: &HashMap) -> Result { } }; - // Add headers + // Add headers. Secrets are approved only as values, never names. if let Some(Value::Map(headers)) = opts.get("headers") { for (key, value) in headers { - if let Value::String(v) = value { - request = request.header(key.as_str(), v.as_str()); - } + request = request.header(key.as_str(), secret_or_string(value, "header value")?); } + } else if let Some(other) = opts.get("headers") { + return Err(IntentError::type_error(format!( + "fetch() headers must be a Map, got {}", + other.type_name() + ))); } // Add cookies if let Some(Value::Map(cookies)) = opts.get("cookies") { - let cookie_str: Vec = cookies + let cookie_str: Result> = cookies .iter() - .filter_map(|(k, v)| { - if let Value::String(val) = v { - Some(format!("{}={}", k, val)) - } else { - None - } + .map(|(key, value)| { + secret_or_string(value, "cookie value").map(|value| format!("{key}={value}")) }) .collect(); + let cookie_str = cookie_str?; if !cookie_str.is_empty() { request = request.header(COOKIE, cookie_str.join("; ")); } + } else if let Some(other) = opts.get("cookies") { + return Err(IntentError::type_error(format!( + "fetch() cookies must be a Map, got {}", + other.type_name() + ))); } // Add basic auth if let Some(Value::Map(auth)) = opts.get("auth") { let username = match auth.get("user") { - Some(Value::String(u)) => u.clone(), - _ => String::new(), + Some(value) => secret_or_string(value, "basic-auth user")?, + None => "", }; let password = match auth.get("pass") { - Some(Value::String(p)) => p.clone(), - _ => String::new(), + Some(value) => secret_or_string(value, "basic-auth password")?, + None => "", }; if !username.is_empty() { - let credentials = format!("{}:{}", username, password); + let credentials = format!("{username}:{password}"); let encoded = base64::engine::general_purpose::STANDARD.encode(credentials.as_bytes()); - request = request.header(AUTHORIZATION, format!("Basic {}", encoded)); + request = request.header(AUTHORIZATION, format!("Basic {encoded}")); } + } else if let Some(other) = opts.get("auth") { + return Err(IntentError::type_error(format!( + "fetch() auth must be a Map, got {}", + other.type_name() + ))); } // Add raw body - if let Some(Value::String(body)) = opts.get("body") { - request = request.body(body.clone()); + if let Some(body) = opts.get("body") { + request = request.body(secret_or_string(body, "body")?.to_string()); } // Add JSON body if let Some(data) = opts.get("json") { - let json_body = intent_value_to_json(data); + let json_body = intent_value_to_json_expose(data)?; request = request .header("Content-Type", "application/json") .body(json_body.to_string()); @@ -732,23 +777,35 @@ fn http_fetch(opts: &HashMap) -> Result { // Add form data if let Some(Value::Map(form_data)) = opts.get("form") { - let mut form: Vec<(String, String)> = Vec::new(); - for (key, value) in form_data { - let string_value = match value { - Value::String(s) => s.clone(), - Value::Int(i) => i.to_string(), - Value::Float(f) => f.to_string(), - Value::Bool(b) => b.to_string(), - _ => format!("{:?}", value), - }; - form.push((key.clone(), string_value)); - } - request = request.form(&form); + let form: Result> = form_data + .iter() + .map(|(key, value)| form_scalar(value).map(|value| (key.clone(), value))) + .collect(); + request = request.form(&form?); + } else if let Some(other) = opts.get("form") { + return Err(IntentError::type_error(format!( + "fetch() form must be a Map, got {}", + other.type_name() + ))); } // Add timeout - if let Some(Value::Int(timeout)) = opts.get("timeout") { - request = request.timeout(Duration::from_secs(*timeout as u64)); + match opts.get("timeout") { + Some(Value::Int(timeout)) if *timeout >= 0 => { + request = request.timeout(Duration::from_secs(*timeout as u64)); + } + Some(Value::Int(_)) => { + return Err(IntentError::type_error( + "fetch() timeout must be non-negative".to_string(), + )) + } + Some(other) => { + return Err(IntentError::type_error(format!( + "fetch() timeout must be an Int, got {}", + other.type_name() + ))) + } + None => {} } // Execute request @@ -887,6 +944,9 @@ pub fn init() -> HashMap { // - Two arguments: a URL string and an options map. The URL is merged into // the options map automatically. // Options map keys: url (set automatically in 2-arg form), method, headers, body, json, form, auth, cookies, timeout. + // Opaque Secret values are accepted only in header values, cookie values, basic-auth + // fields, raw bodies, JSON leaves, and form values. Requests containing a Secret + // do not follow redirects, preventing credentials or 307/308 bodies from crossing origins. // @param url_or_options A URL string for GET, or a Map with request options // @param options (optional) A Map with request options when first argument is a URL string // @returns Result where Response is a Map with status, status_text, headers, body, ok, url, redirected, and cookies fields diff --git a/src/stdlib/http_server.rs b/src/stdlib/http_server.rs index 12dc1983..d0d58d92 100644 --- a/src/stdlib/http_server.rs +++ b/src/stdlib/http_server.rs @@ -1238,7 +1238,8 @@ fn headers_get_string(req_map: &HashMap, key: &str) -> Option serde_json::Value { crate::stdlib::json::intent_value_to_json(value) } @@ -1472,7 +1473,7 @@ pub fn init() -> HashMap { } } - let json_value = intent_value_to_json(&args[0]); + let json_value = crate::stdlib::json::intent_value_to_json_reject(&args[0])?; let body = json_value.to_string(); let mut headers = HashMap::new(); headers.insert( diff --git a/src/stdlib/http_server_async.rs b/src/stdlib/http_server_async.rs index 2bcd1d4c..d9177884 100644 --- a/src/stdlib/http_server_async.rs +++ b/src/stdlib/http_server_async.rs @@ -1069,9 +1069,9 @@ async fn shutdown_signal() { // === Helper functions for creating NTNT response Values === -/// Create a JSON response Value -pub fn create_json_response(data: &Value, status: i64) -> Value { - let json_value = crate::stdlib::json::intent_value_to_json(data); +/// Create a JSON response Value, rejecting opaque secrets. +pub fn create_json_response(data: &Value, status: i64) -> Result { + let json_value = crate::stdlib::json::intent_value_to_json_reject(data)?; let json_string = json_value.to_string(); let mut headers = HashMap::new(); // Default security headers @@ -1100,7 +1100,7 @@ pub fn create_json_response(data: &Value, status: i64) -> Value { response.insert("status".to_string(), Value::Int(status)); response.insert("headers".to_string(), Value::Map(headers)); response.insert("body".to_string(), Value::String(json_string)); - Value::Map(response) + Ok(Value::Map(response)) } /// Create an error response Value @@ -1201,6 +1201,23 @@ mod tests { assert_eq!(guess_mime_type("unknown"), "application/octet-stream"); } + #[test] + fn test_create_json_response_rejects_nested_secret_values() { + let data = Value::Map(HashMap::from([( + "token".to_string(), + Value::Secret( + crate::interpreter::SecretValue::new( + "ASYNC_RESPONSE_SECRET", + "async-response-canary", + ) + .unwrap(), + ), + )])); + let error = create_json_response(&data, 200).expect_err("secret response must fail closed"); + assert!(error.to_string().contains("Secret")); + assert!(!error.to_string().contains("async-response-canary")); + } + #[test] fn test_create_error_response() { let resp = create_error_response(404, "Not Found"); diff --git a/src/stdlib/jobs.rs b/src/stdlib/jobs.rs index fa75f3fb..a2c13531 100644 --- a/src/stdlib/jobs.rs +++ b/src/stdlib/jobs.rs @@ -721,6 +721,8 @@ fn try_fire_batch_callback( /// Returns `Ok(job_id)` on success, or an error if the batch doesn't exist, /// is complete, or is closed. fn enqueue_to_sealed_batch(batch_id: &str, job_name: &str, payload: Value) -> Result { + reject_secret_job_payload(&payload)?; + // Step 1: Validate job type is registered (before any KV access). let job_def = JOB_RUNTIME.get_job(job_name)?.ok_or_else(|| { IntentError::runtime_error(format!( @@ -1428,6 +1430,16 @@ fn execute_on_failure_in_worker( } } +fn reject_secret_job_payload(payload: &Value) -> Result<()> { + if payload.contains_secret() { + return Err(IntentError::type_error( + "Secret values cannot be captured in job payloads; call require_secret() inside the job" + .to_string(), + )); + } + Ok(()) +} + /// Convert an `EnqueueResult` to the `Value::ok(Value::String(job_id))` format /// expected by ntnt stdlib callers. fn enqueue_result_to_value(result: EnqueueResult) -> Value { @@ -1483,6 +1495,8 @@ fn enqueue_internal_with_def( batch_id: Option<&str>, override_job_id: Option<&str>, ) -> Result { + reject_secret_job_payload(&payload)?; + // Resolve numeric priority from job options // Named: critical=5, high=25, normal=50 (default), low=85 // Numeric: 0-99 inclusive @@ -3847,6 +3861,7 @@ pub fn init() -> HashMap { )) } }; + reject_secret_job_payload(&payload)?; let payload_json = serde_json::to_string( &kv::value_to_json_public(&payload), ) @@ -5701,6 +5716,21 @@ pub(crate) mod tests { f(); } + #[test] + fn secret_values_are_rejected_from_job_payloads() { + let payload = Value::Map(HashMap::from([( + "token".to_string(), + Value::Secret( + crate::interpreter::SecretValue::new("JOB_SECRET", "job-secret-canary") + .expect("valid secret"), + ), + )])); + + let error = reject_secret_job_payload(&payload).expect_err("jobs must reject secrets"); + assert!(error.to_string().contains("require_secret")); + assert!(!error.to_string().contains("job-secret-canary")); + } + /// Set up an isolated in-memory SQLite KV store for the duration of a test. /// Uses `sqlite::memory:` for a clean, isolated DB per test. /// The `db_name` parameter is kept for API compatibility but is ignored. diff --git a/src/stdlib/json.rs b/src/stdlib/json.rs index d0aa3182..0747a0b9 100644 --- a/src/stdlib/json.rs +++ b/src/stdlib/json.rs @@ -32,9 +32,46 @@ pub fn json_to_intent_value(json: &serde_json::Value) -> Value { } } -/// Convert Intent Value to JSON value +/// Secret handling policy for JSON conversion. +#[derive(Clone, Copy, PartialEq, Eq)] +enum SecretSerialization { + Redact, + Reject, + Expose, +} + +/// Convert an ntnt value to JSON while redacting any direct or nested secrets. +/// +/// This compatibility helper is safe for diagnostics and internal metadata, but +/// public serialization boundaries should use [`intent_value_to_json_reject`]. pub fn intent_value_to_json(value: &Value) -> serde_json::Value { - match value { + intent_value_to_json_with_policy(value, SecretSerialization::Redact) + .unwrap_or_else(|_| serde_json::Value::String(crate::secret::REDACTED_SECRET.to_string())) +} + +/// Convert an ntnt value to public JSON, rejecting direct or nested secrets. +pub fn intent_value_to_json_reject(value: &Value) -> crate::error::Result { + intent_value_to_json_with_policy(value, SecretSerialization::Reject) +} + +/// Convert outbound HTTP request JSON, exposing secret leaves only at this audited sink. +pub(crate) fn intent_value_to_json_expose( + value: &Value, +) -> crate::error::Result { + intent_value_to_json_with_policy(value, SecretSerialization::Expose) +} + +fn intent_value_to_json_with_policy( + value: &Value, + policy: SecretSerialization, +) -> crate::error::Result { + if policy == SecretSerialization::Reject && value.contains_secret() { + return Err(IntentError::type_error( + "Secret values cannot be serialized to public JSON".to_string(), + )); + } + + Ok(match value { Value::Unit => serde_json::Value::Null, Value::Bool(b) => serde_json::Value::Bool(*b), Value::Int(i) => serde_json::Value::Number(serde_json::Number::from(*i)), @@ -42,21 +79,38 @@ pub fn intent_value_to_json(value: &Value) -> serde_json::Value { .map(serde_json::Value::Number) .unwrap_or(serde_json::Value::Null), Value::String(s) => serde_json::Value::String(s.clone()), - Value::Array(arr) => { - serde_json::Value::Array(arr.iter().map(intent_value_to_json).collect()) - } + Value::Secret(secret) => match policy { + SecretSerialization::Redact => { + serde_json::Value::String(crate::secret::REDACTED_SECRET.to_string()) + } + SecretSerialization::Expose => { + serde_json::Value::String(secret.expose().to_string()) + } + SecretSerialization::Reject => unreachable!("rejected before conversion"), + }, + Value::Array(arr) => serde_json::Value::Array( + arr.iter() + .map(|item| intent_value_to_json_with_policy(item, policy)) + .collect::>>()?, + ), Value::Map(map) => { let obj: serde_json::Map = map .iter() - .map(|(k, v)| (k.clone(), intent_value_to_json(v))) - .collect(); + .map(|(key, item)| { + intent_value_to_json_with_policy(item, policy) + .map(|converted| (key.clone(), converted)) + }) + .collect::>()?; serde_json::Value::Object(obj) } Value::Struct { fields, .. } => { let obj: serde_json::Map = fields .iter() - .map(|(k, v)| (k.clone(), intent_value_to_json(v))) - .collect(); + .map(|(key, item)| { + intent_value_to_json_with_policy(item, policy) + .map(|converted| (key.clone(), converted)) + }) + .collect::>()?; serde_json::Value::Object(obj) } Value::EnumValue { @@ -65,15 +119,21 @@ pub fn intent_value_to_json(value: &Value) -> serde_json::Value { values, } if enum_name == "Option" => match variant.as_str() { "None" => serde_json::Value::Null, - "Some" => values - .first() - .map(intent_value_to_json) - .unwrap_or(serde_json::Value::Null), + "Some" => match values.first() { + Some(inner) => intent_value_to_json_with_policy(inner, policy)?, + None => serde_json::Value::Null, + }, _ => serde_json::Value::String(value.to_string()), }, - // For other types, convert to string representation + other if policy == SecretSerialization::Expose && other.contains_secret() => { + return Err(IntentError::type_error( + "Secret values in outbound JSON must be leaves inside maps, arrays, structs, or Option" + .to_string(), + )) + } + // For other types, preserve the existing string representation. _ => serde_json::Value::String(value.to_string()), - } + }) } /// Initialize the std/json module @@ -161,7 +221,7 @@ pub fn init() -> HashMap { max_arity: 1, requires: None, func: |args| { - let json_val = intent_value_to_json(&args[0]); + let json_val = intent_value_to_json_reject(&args[0])?; Ok(Value::String(json_val.to_string())) }, }, @@ -188,7 +248,7 @@ pub fn init() -> HashMap { max_arity: 1, requires: None, func: |args| { - let json_val = intent_value_to_json(&args[0]); + let json_val = intent_value_to_json_reject(&args[0])?; match serde_json::to_string_pretty(&json_val) { Ok(s) => Ok(Value::String(s)), Err(e) => Ok(Value::String(format!("{{\"error\": \"{}\"}}", e))), diff --git a/src/stdlib/kv.rs b/src/stdlib/kv.rs index 9f007db3..cf9be001 100644 --- a/src/stdlib/kv.rs +++ b/src/stdlib/kv.rs @@ -55,9 +55,15 @@ fn now_unix() -> i64 { .unwrap_or(0) } -/// Serialize a Value to string for storage -fn serialize_value(value: &Value) -> (String, String) { - match value { +/// Serialize a Value to string for storage. +fn serialize_value(value: &Value) -> Result<(String, String)> { + if value.contains_secret() { + return Err(IntentError::type_error( + "Secret values cannot be persisted in std/kv".to_string(), + )); + } + + Ok(match value { Value::String(s) => (s.clone(), "string".to_string()), Value::Int(i) => (i.to_string(), "int".to_string()), Value::Float(f) => (f.to_string(), "float".to_string()), @@ -71,14 +77,20 @@ fn serialize_value(value: &Value) -> (String, String) { (json, "map".to_string()) } _ => (format!("{:?}", value), "unknown".to_string()), - } + }) } /// Serialize a Value to a JSON envelope for Redis storage. /// Non-string types get wrapped: {"__ntnt_t":"","v":} /// Plain strings are stored as-is for backward compatibility and efficiency. -fn serialize_value_envelope(value: &Value) -> String { - match value { +fn serialize_value_envelope(value: &Value) -> Result { + if value.contains_secret() { + return Err(IntentError::type_error( + "Secret values cannot be persisted in std/kv".to_string(), + )); + } + + Ok(match value { Value::String(s) => s.clone(), Value::Int(i) => serde_json::json!({"__ntnt_t": "int", "v": i}).to_string(), Value::Float(f) => serde_json::json!({"__ntnt_t": "float", "v": f}).to_string(), @@ -93,7 +105,7 @@ fn serialize_value_envelope(value: &Value) -> String { serde_json::json!({"__ntnt_t": type_name, "v": json_val}).to_string() } _ => format!("{:?}", value), - } + }) } /// Deserialize a Redis value using envelope format, with backward compatibility. @@ -257,7 +269,7 @@ impl SQLiteKV { return self.del(key).map(|_| ()); } - let (serialized, type_hint) = serialize_value(value); + let (serialized, type_hint) = serialize_value(value)?; let expires_at = ttl_seconds.map(|ttl| now_unix() + ttl); self.conn @@ -287,7 +299,7 @@ impl SQLiteKV { IntentError::runtime_error(format!("KV set_nx delete expired error: {}", e)) })?; - let (serialized, type_hint) = serialize_value(value); + let (serialized, type_hint) = serialize_value(value)?; let expires_at = ttl_seconds.map(|ttl| now + ttl); let changes = self @@ -628,7 +640,7 @@ impl RedisKV { } // Use envelope format — single key, no __type sibling - let serialized = serialize_value_envelope(value); + let serialized = serialize_value_envelope(value)?; match ttl_seconds { Some(ttl) => { @@ -655,7 +667,7 @@ impl RedisKV { /// Uses `SET key value NX [EX ttl]` — atomic in Redis. /// Returns `Ok(true)` if set, `Ok(false)` if the key already existed. pub fn set_nx(&mut self, key: &str, value: &Value, ttl_seconds: Option) -> Result { - let serialized = serialize_value_envelope(value); + let serialized = serialize_value_envelope(value)?; let result: Option = match ttl_seconds { Some(ttl) => redis::cmd("SET") @@ -3018,4 +3030,19 @@ mod tests { "i64::MIN decr_by should not panic or wrap" ); } + + #[test] + fn secret_values_cannot_be_serialized_for_kv_storage() { + let secret = Value::Secret( + crate::interpreter::SecretValue::new("KV_SECRET", "kv-secret-canary") + .expect("valid secret"), + ); + let nested = Value::Map(HashMap::from([("secret".to_string(), secret)])); + + let direct_error = serialize_value(&nested).expect_err("KV must reject nested secrets"); + assert!(!direct_error.to_string().contains("kv-secret-canary")); + let envelope_error = + serialize_value_envelope(&nested).expect_err("Redis envelope must reject secrets"); + assert!(!envelope_error.to_string().contains("kv-secret-canary")); + } } diff --git a/src/stdlib/log.rs b/src/stdlib/log.rs index 09b31485..6467f48f 100644 --- a/src/stdlib/log.rs +++ b/src/stdlib/log.rs @@ -45,6 +45,7 @@ fn value_to_json(value: &Value) -> serde_json::Value { .map(serde_json::Value::Number) .unwrap_or(serde_json::Value::Null), Value::String(s) => serde_json::Value::String(s.clone()), + Value::Secret(_) => serde_json::Value::String(crate::secret::REDACTED_SECRET.to_string()), Value::Array(arr) => serde_json::Value::Array(arr.iter().map(value_to_json).collect()), Value::Map(map) => { let obj: serde_json::Map = map @@ -439,6 +440,17 @@ mod tests { value_to_json(&Value::String("test".to_string())), serde_json::Value::String("test".to_string()) ); + + let secret = Value::Map(HashMap::from([( + "token".to_string(), + Value::Secret( + crate::interpreter::SecretValue::new("LOG_SECRET", "log-secret-canary") + .expect("valid secret"), + ), + )])); + let logged = value_to_json(&secret).to_string(); + assert!(logged.contains(crate::secret::REDACTED_SECRET)); + assert!(!logged.contains("log-secret-canary")); } #[test] diff --git a/src/stdlib/mod.rs b/src/stdlib/mod.rs index 18b07aba..2b2e958c 100644 --- a/src/stdlib/mod.rs +++ b/src/stdlib/mod.rs @@ -27,6 +27,7 @@ pub mod math; pub mod net; pub mod path; pub mod postgres; +pub mod secrets; pub mod sqlite; pub mod string; pub mod template; @@ -48,6 +49,7 @@ pub fn init_all_modules() -> HashMap { modules.insert("std/math".to_string(), math::init()); modules.insert("std/collections".to_string(), collections::init()); modules.insert("std/env".to_string(), env::init()); + modules.insert("std/secrets".to_string(), secrets::init()); modules.insert("std/fs".to_string(), fs::init()); modules.insert("std/path".to_string(), path::init()); modules.insert("std/json".to_string(), json::init()); diff --git a/src/stdlib/secrets.rs b/src/stdlib/secrets.rs new file mode 100644 index 00000000..430e42bc --- /dev/null +++ b/src/stdlib/secrets.rs @@ -0,0 +1,601 @@ +//! Provider-neutral secret lookup for ntnt applications. + +use crate::error::{IntentError, Result}; +use crate::interpreter::{SecretValue, Value}; +use crate::secret::validate_secret_name; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock, RwLock}; + +const PROVIDER_ENV: &str = "NTNT_SECRETS_PROVIDER"; +const DEFAULT_ATTEMPTS_PER_ENDPOINT: usize = 2; + +#[cfg_attr(test, allow(dead_code))] +#[derive(Default)] +enum SecretDeclarations { + #[default] + Unconfigured, + NoManifest, + Loaded(HashSet), + Invalid, + ConflictingApplication, +} + +#[cfg_attr(test, allow(dead_code))] +#[derive(Default)] +struct DeclarationState { + identity: Option, + declarations: SecretDeclarations, +} + +static SECRET_DECLARATIONS: OnceLock> = OnceLock::new(); + +fn declaration_state() -> &'static RwLock { + SECRET_DECLARATIONS.get_or_init(|| RwLock::new(DeclarationState::default())) +} + +/// Configure manifest-backed secret declarations for an application's main source file. +/// +/// The closest ancestor `ntnt.toml` wins. A project with a manifest must declare +/// every accessible secret under `[secrets.]`; manifest-free development +/// remains compatible with simple environment-provider experiments. +/// ntnt runs one application per process. Reconfiguring a process for a different +/// application fails closed instead of letting the last interpreter replace policy. +#[cfg_attr(test, allow(dead_code))] +pub(crate) fn configure_for_source(source_path: &str) { + let (identity, declarations) = load_declarations(Path::new(source_path)); + if let Ok(mut state) = declaration_state().write() { + match &state.identity { + None => { + state.identity = Some(identity); + state.declarations = declarations; + } + Some(current) if current == &identity => state.declarations = declarations, + Some(_) => state.declarations = SecretDeclarations::ConflictingApplication, + } + } +} + +#[cfg_attr(test, allow(dead_code))] +fn load_declarations(source_path: &Path) -> (PathBuf, SecretDeclarations) { + let start = if source_path.is_file() { + source_path.parent().unwrap_or(source_path) + } else { + source_path + }; + + let source_identity = start.canonicalize().unwrap_or_else(|_| start.to_path_buf()); + let manifest = start + .ancestors() + .map(|directory| directory.join("ntnt.toml")) + .find(|candidate| candidate.is_file()); + let Some(manifest) = manifest else { + return (source_identity, SecretDeclarations::NoManifest); + }; + let identity = manifest.canonicalize().unwrap_or_else(|_| manifest.clone()); + + let Ok(content) = std::fs::read_to_string(manifest) else { + return (identity, SecretDeclarations::Invalid); + }; + let Ok(document) = content.parse::() else { + return (identity, SecretDeclarations::Invalid); + }; + let Some(secrets) = document.get("secrets") else { + return (identity, SecretDeclarations::Loaded(HashSet::new())); + }; + let Some(table) = secrets.as_table() else { + return (identity, SecretDeclarations::Invalid); + }; + + let mut declared = HashSet::with_capacity(table.len()); + for (name, metadata) in table { + let Some(metadata) = metadata.as_table() else { + return (identity, SecretDeclarations::Invalid); + }; + if validate_secret_name(name).is_err() + || metadata.keys().any(|key| { + !matches!( + key.as_str(), + "label" | "description" | "required" | "environments" + ) + }) + || metadata.get("label").is_some_and(|value| !value.is_str()) + || metadata + .get("description") + .is_some_and(|value| !value.is_str()) + || metadata + .get("required") + .is_some_and(|value| !value.is_bool()) + || metadata.get("environments").is_some_and(|value| { + value + .as_array() + .is_none_or(|items| items.iter().any(|item| !item.is_str())) + }) + { + return (identity, SecretDeclarations::Invalid); + } + declared.insert(name.clone()); + } + (identity, SecretDeclarations::Loaded(declared)) +} + +fn enforce_declared(name: &str) -> Result<()> { + let declarations = declaration_state().read().map_err(|_| { + IntentError::runtime_error("Secret declaration state is unavailable".to_string()) + })?; + match &declarations.declarations { + SecretDeclarations::Unconfigured | SecretDeclarations::NoManifest => Ok(()), + SecretDeclarations::Loaded(names) if names.contains(name) => Ok(()), + SecretDeclarations::Loaded(_) => Err(IntentError::runtime_error(format!( + "Secret '{name}' is not declared in ntnt.toml" + ))), + SecretDeclarations::Invalid => Err(IntentError::runtime_error( + "Secret declarations in ntnt.toml are invalid".to_string(), + )), + SecretDeclarations::ConflictingApplication => Err(IntentError::runtime_error( + "Secret declarations cannot be shared across multiple applications in one process" + .to_string(), + )), + } +} + +/// A provider result that is deliberately not `Debug`: the found payload is plaintext. +enum ProviderLookup { + Found(String), + Missing, +} + +/// Stable failure classes for HA failover. The v0.5.1 environment provider +/// can only emit invalid configuration; socket providers use the remaining classes. +#[cfg_attr(not(test), allow(dead_code))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ProviderErrorKind { + Unavailable, + AccessDenied, + InvalidConfiguration, +} + +#[derive(Debug, Clone)] +struct ProviderError { + kind: ProviderErrorKind, + endpoint: String, +} + +impl ProviderError { + fn new(kind: ProviderErrorKind, endpoint: impl Into) -> Self { + Self { + kind, + endpoint: sanitize_endpoint(&endpoint.into()), + } + } +} + +trait SecretProvider: Send + Sync { + fn endpoint(&self) -> &str; + fn authorization_scope(&self) -> &str; + + /// Perform one endpoint lookup. Implementations must apply a bounded timeout + /// and classify timeouts/transient transport failures as `Unavailable`. + fn lookup(&self, name: &str) -> std::result::Result; +} + +/// Ordered equivalent provider endpoints with classified failover. +/// +/// Only `Unavailable` advances to the next endpoint. Missing, denied, and invalid +/// configuration results are terminal so failover cannot change authorization scope. +struct ProviderGroup { + providers: Vec>, + attempts_per_endpoint: usize, +} + +impl ProviderGroup { + fn new(providers: Vec>) -> Result { + if providers.is_empty() { + return Err(IntentError::runtime_error( + "Secrets provider configuration has no endpoints".to_string(), + )); + } + let scope = providers[0].authorization_scope(); + if scope.is_empty() + || providers + .iter() + .any(|provider| provider.authorization_scope() != scope) + { + return Err(IntentError::runtime_error( + "Secrets provider endpoints must share one authorization scope".to_string(), + )); + } + Ok(Self { + providers, + attempts_per_endpoint: DEFAULT_ATTEMPTS_PER_ENDPOINT, + }) + } + + fn lookup(&self, name: &str) -> Result> { + validate_secret_name(name)?; + let mut unavailable = Vec::new(); + let mut attempts = 0; + + for provider in &self.providers { + for attempt in 0..self.attempts_per_endpoint { + attempts += 1; + match provider.lookup(name) { + Ok(ProviderLookup::Found(value)) => { + return SecretValue::new(name, value).map(Some) + } + Ok(ProviderLookup::Missing) => return Ok(None), + Err(error) => match error.kind { + ProviderErrorKind::Unavailable + if attempt + 1 < self.attempts_per_endpoint => + { + continue; + } + ProviderErrorKind::Unavailable => { + unavailable.push(error.endpoint); + break; + } + ProviderErrorKind::AccessDenied => { + return Err(IntentError::runtime_error(format!( + "Secret provider access denied for '{name}' at endpoint '{}'", + error.endpoint + ))) + } + ProviderErrorKind::InvalidConfiguration => { + return Err(IntentError::runtime_error(format!( + "Secret provider configuration is invalid for endpoint '{}'", + error.endpoint + ))) + } + }, + } + } + } + + Err(IntentError::runtime_error(format!( + "Secret provider unavailable after {attempts} bounded attempt(s) across {} endpoint(s): {}", + unavailable.len(), + unavailable.join(", ") + ))) + } +} + +struct EnvSecretProvider; + +impl SecretProvider for EnvSecretProvider { + fn endpoint(&self) -> &str { + "env" + } + + fn authorization_scope(&self) -> &str { + "process-env" + } + + fn lookup(&self, name: &str) -> std::result::Result { + match std::env::var(name) { + Ok(value) if value.is_empty() => Ok(ProviderLookup::Missing), + Ok(value) => Ok(ProviderLookup::Found(value)), + Err(std::env::VarError::NotPresent) => Ok(ProviderLookup::Missing), + Err(std::env::VarError::NotUnicode(_)) => Err(ProviderError::new( + ProviderErrorKind::InvalidConfiguration, + self.endpoint(), + )), + } + } +} + +fn sanitize_endpoint(endpoint: &str) -> String { + if endpoint.is_empty() + || endpoint.len() > 64 + || !endpoint + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | ':' | '/' | '-')) + { + return "unknown".to_string(); + } + endpoint.to_string() +} + +fn is_production_mode() -> bool { + std::env::var("NTNT_ENV") + .map(|value| matches!(value.to_ascii_lowercase().as_str(), "production" | "prod")) + .unwrap_or(false) +} + +fn configured_provider_group() -> Result { + let provider = std::env::var(PROVIDER_ENV).unwrap_or_else(|_| "env".to_string()); + match provider.as_str() { + "env" if is_production_mode() => Err(IntentError::runtime_error( + "The environment secrets provider is development-only and is disabled in production" + .to_string(), + )), + "env" => ProviderGroup::new(vec![Arc::new(EnvSecretProvider)]), + _ => Err(IntentError::runtime_error( + "Unsupported secrets provider; ntnt v0.5.1 supports 'env' in development".to_string(), + )), + } +} + +fn lookup_secret(name: &str) -> Result> { + validate_secret_name(name)?; + enforce_declared(name)?; + configured_provider_group()?.lookup(name) +} + +fn secret_name_arg(args: &[Value], function: &str) -> Result { + match args.first() { + Some(Value::String(name)) => { + validate_secret_name(name)?; + Ok(name.clone()) + } + _ => Err(IntentError::type_error(format!( + "{function}() requires a secret name string" + ))), + } +} + +/// Initialize the `std/secrets` module. +pub fn init() -> HashMap { + let mut module = HashMap::new(); + + // @ntnt get_secret + // @module std/secrets + // @module_description Provider-neutral secret lookup with opaque, redacted values + // @signature get_secret(name: String) -> Option + // Looks up a secret by its provider-neutral logical name. + // + // The v0.5.1 development-only environment provider reads the exact environment + // variable name and is disabled when `NTNT_ENV` is `production` or `prod`. + // Projects with an `ntnt.toml` must declare accessible names under + // `[secrets.]`; undeclared lookups fail before contacting the provider. + // Declaration metadata may contain `label`, `description`, `required`, and + // `environments`; secret values are never accepted in the manifest. + // Secret values remain opaque and redact themselves in output and diagnostics. + // @param name A validated logical secret name + // @returns Some(Secret) when configured, otherwise None + // @see_also require_secret + // @since v0.5.1 + // @tags #security, #secrets + // @example get_secret("STRIPE_SECRET_KEY") => Some([REDACTED]) ~ "Optional lookup" + // @error RuntimeError ~ "Unsupported secrets provider" fix: "Set NTNT_SECRETS_PROVIDER=env for v0.5.1" + module.insert( + "get_secret".to_string(), + Value::NativeFunction { + name: "get_secret".to_string(), + arity: 1, + max_arity: 1, + requires: None, + func: |args| { + let name = secret_name_arg(args, "get_secret")?; + Ok(match lookup_secret(&name)? { + Some(secret) => Value::some(Value::Secret(secret)), + None => Value::none(), + }) + }, + }, + ); + + // @ntnt require_secret + // @module std/secrets + // @module_description Provider-neutral secret lookup with opaque, redacted values + // @signature require_secret(name: String) -> Secret + // Looks up a required secret and fails closed when it is not configured. + // + // Use this at startup or immediately before an approved secret-consuming sink. + // In v0.5.1, `std/http.fetch` accepts Secret values as header, cookie, + // basic-auth, raw body, JSON-leaf, and form values. Templates, public JSON, + // URL/CSV/string conversion, KV storage, and job payloads reject secrets. + // There is intentionally no general Secret-to-String reveal function. + // The error identifies only the logical name and never includes the value. + // @param name A validated logical secret name + // @returns The opaque Secret value + // @see_also get_secret + // @since v0.5.1 + // @tags #security, #secrets + // @example require_secret("STRIPE_SECRET_KEY") => [REDACTED] ~ "Required lookup" + // @error RuntimeError ~ "Required secret is not configured" fix: "Configure the named secret in the selected provider" + module.insert( + "require_secret".to_string(), + Value::NativeFunction { + name: "require_secret".to_string(), + arity: 1, + max_arity: 1, + requires: None, + func: |args| { + let name = secret_name_arg(args, "require_secret")?; + match lookup_secret(&name)? { + Some(secret) => Ok(Value::Secret(secret)), + None => Err(IntentError::runtime_error(format!( + "Required secret '{name}' is not configured" + ))), + } + }, + }, + ); + + module +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::VecDeque; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Mutex; + + struct MockProvider { + endpoint: String, + authorization_scope: String, + calls: AtomicUsize, + results: Mutex>>, + } + + impl MockProvider { + fn new( + endpoint: &str, + results: Vec>, + ) -> Self { + Self { + endpoint: endpoint.to_string(), + authorization_scope: "deployment-a".to_string(), + calls: AtomicUsize::new(0), + results: Mutex::new(results.into()), + } + } + + fn calls(&self) -> usize { + self.calls.load(Ordering::SeqCst) + } + + fn with_scope(mut self, scope: &str) -> Self { + self.authorization_scope = scope.to_string(); + self + } + } + + impl SecretProvider for MockProvider { + fn endpoint(&self) -> &str { + &self.endpoint + } + + fn authorization_scope(&self) -> &str { + &self.authorization_scope + } + + fn lookup(&self, _name: &str) -> std::result::Result { + self.calls.fetch_add(1, Ordering::SeqCst); + self.results + .lock() + .expect("mock results lock") + .pop_front() + .expect("mock result") + } + } + + #[test] + fn provider_group_rejects_mixed_authorization_scopes() { + let first = Arc::new( + MockProvider::new("socket-a", vec![Ok(ProviderLookup::Missing)]) + .with_scope("deployment-a"), + ); + let second = Arc::new( + MockProvider::new("socket-b", vec![Ok(ProviderLookup::Missing)]) + .with_scope("deployment-b"), + ); + + let result = ProviderGroup::new(vec![first, second]); + let Err(error) = result else { + panic!("mixed authorization scopes must fail closed"); + }; + assert!(error.to_string().contains("authorization scope")); + } + + #[test] + fn provider_group_fails_over_only_after_unavailable() { + let first = Arc::new(MockProvider::new( + "socket-a", + vec![ + Err(ProviderError::new( + ProviderErrorKind::Unavailable, + "socket-a", + )), + Err(ProviderError::new( + ProviderErrorKind::Unavailable, + "socket-a", + )), + ], + )); + let second = Arc::new(MockProvider::new( + "socket-b", + vec![Ok(ProviderLookup::Found("secret-value".to_string()))], + )); + let group = ProviderGroup::new(vec![first.clone(), second.clone()]).expect("group"); + + let value = group.lookup("API_KEY").expect("lookup").expect("found"); + assert_eq!(value.expose(), "secret-value"); + assert_eq!(first.calls(), 2); + assert_eq!(second.calls(), 1); + } + + #[test] + fn provider_group_retries_transient_failure_before_failover() { + let first = Arc::new(MockProvider::new( + "socket-a", + vec![ + Err(ProviderError::new( + ProviderErrorKind::Unavailable, + "socket-a", + )), + Ok(ProviderLookup::Found("secret-value".to_string())), + ], + )); + let second = Arc::new(MockProvider::new( + "socket-b", + vec![Ok(ProviderLookup::Found("must-not-read".to_string()))], + )); + let group = ProviderGroup::new(vec![first.clone(), second.clone()]).expect("group"); + + let value = group.lookup("API_KEY").expect("lookup").expect("found"); + assert_eq!(value.expose(), "secret-value"); + assert_eq!(first.calls(), 2); + assert_eq!(second.calls(), 0); + } + + #[test] + fn provider_group_stops_on_access_denied() { + let first = Arc::new(MockProvider::new( + "socket-a", + vec![Err(ProviderError::new( + ProviderErrorKind::AccessDenied, + "socket-a", + ))], + )); + let second = Arc::new(MockProvider::new( + "socket-b", + vec![Ok(ProviderLookup::Found("must-not-read".to_string()))], + )); + let group = ProviderGroup::new(vec![first, second.clone()]).expect("group"); + + let err = group.lookup("API_KEY").expect_err("access denied"); + assert!(err.to_string().contains("access denied")); + assert_eq!(second.calls(), 0); + } + + #[test] + fn provider_group_stops_on_missing() { + let first = Arc::new(MockProvider::new( + "socket-a", + vec![Ok(ProviderLookup::Missing)], + )); + let second = Arc::new(MockProvider::new( + "socket-b", + vec![Ok(ProviderLookup::Found("must-not-read".to_string()))], + )); + let group = ProviderGroup::new(vec![first, second.clone()]).expect("group"); + + assert!(group.lookup("API_KEY").expect("lookup").is_none()); + assert_eq!(second.calls(), 0); + } + + #[test] + fn provider_group_reports_bounded_all_unavailable_attempts() { + let providers: Vec> = ["socket-a", "socket-b"] + .into_iter() + .map(|endpoint| { + Arc::new(MockProvider::new( + endpoint, + vec![ + Err(ProviderError::new(ProviderErrorKind::Unavailable, endpoint)), + Err(ProviderError::new(ProviderErrorKind::Unavailable, endpoint)), + ], + )) as Arc + }) + .collect(); + let group = ProviderGroup::new(providers).expect("group"); + + let err = group.lookup("API_KEY").expect_err("all unavailable"); + let rendered = err.to_string(); + assert!(rendered.contains("4 bounded attempt(s) across 2 endpoint(s)")); + assert!(rendered.contains("socket-a, socket-b")); + assert!(!rendered.contains("secret-value")); + } +} diff --git a/src/stdlib/string.rs b/src/stdlib/string.rs index 92a1594f..fadcc456 100644 --- a/src/stdlib/string.rs +++ b/src/stdlib/string.rs @@ -70,20 +70,27 @@ pub fn init() -> HashMap { arity: 2, max_arity: 2, requires: None, - func: |args| match (&args[0], &args[1]) { - (Value::Array(arr), Value::String(delim)) => { - let parts: Vec = arr - .iter() - .map(|v| match v { - Value::String(s) => s.clone(), - other => other.to_string(), - }) - .collect(); - Ok(Value::String(parts.join(delim))) + func: |args| { + if args[0].contains_secret() || args[1].contains_secret() { + return Err(IntentError::type_error( + "join() cannot stringify Secret values".to_string(), + )); + } + match (&args[0], &args[1]) { + (Value::Array(arr), Value::String(delim)) => { + let parts: Vec = arr + .iter() + .map(|v| match v { + Value::String(s) => s.clone(), + other => other.to_string(), + }) + .collect(); + Ok(Value::String(parts.join(delim))) + } + _ => Err(IntentError::type_error( + "join() requires array and string".to_string(), + )), } - _ => Err(IntentError::type_error( - "join() requires array and string".to_string(), - )), }, }, ); @@ -108,26 +115,33 @@ pub fn init() -> HashMap { arity: 2, // At least 2, but handles variadic via array max_arity: 2, requires: None, - func: |args| match &args[0] { - Value::Array(arr) => { - let result: String = arr - .iter() - .map(|v| match v { - Value::String(s) => s.clone(), - other => other.to_string(), - }) - .collect(); - Ok(Value::String(result)) + func: |args| { + if args[0].contains_secret() || args[1].contains_secret() { + return Err(IntentError::type_error( + "concat() cannot stringify Secret values".to_string(), + )); } - Value::String(s1) => match &args[1] { - Value::String(s2) => Ok(Value::String(format!("{}{}", s1, s2))), + match &args[0] { + Value::Array(arr) => { + let result: String = arr + .iter() + .map(|v| match v { + Value::String(s) => s.clone(), + other => other.to_string(), + }) + .collect(); + Ok(Value::String(result)) + } + Value::String(s1) => match &args[1] { + Value::String(s2) => Ok(Value::String(format!("{}{}", s1, s2))), + _ => Err(IntentError::type_error( + "concat() requires strings".to_string(), + )), + }, _ => Err(IntentError::type_error( - "concat() requires strings".to_string(), + "concat() requires strings or array of strings".to_string(), )), - }, - _ => Err(IntentError::type_error( - "concat() requires strings or array of strings".to_string(), - )), + } }, }, ); diff --git a/src/stdlib/url.rs b/src/stdlib/url.rs index 0d6986f0..f1ed8f16 100644 --- a/src/stdlib/url.rs +++ b/src/stdlib/url.rs @@ -355,21 +355,28 @@ pub fn init() -> HashMap { arity: 1, max_arity: 1, requires: None, - func: |args| match &args[0] { - Value::Map(params) => { - let pairs: Vec = params - .iter() - .map(|(k, v)| { - let key = url_encode_component(k); - let value = url_encode_component(&v.to_string()); - format!("{}={}", key, value) - }) - .collect(); - Ok(Value::String(pairs.join("&"))) + func: |args| { + if args[0].contains_secret() { + return Err(IntentError::type_error( + "build_query() cannot serialize Secret values into URLs".to_string(), + )); + } + match &args[0] { + Value::Map(params) => { + let pairs: Vec = params + .iter() + .map(|(k, v)| { + let key = url_encode_component(k); + let value = url_encode_component(&v.to_string()); + format!("{}={}", key, value) + }) + .collect(); + Ok(Value::String(pairs.join("&"))) + } + _ => Err(IntentError::type_error( + "build_query() requires a map".to_string(), + )), } - _ => Err(IntentError::type_error( - "build_query() requires a map".to_string(), - )), }, }, ); diff --git a/src/typechecker.rs b/src/typechecker.rs index 32128e7b..1ac2cb17 100644 --- a/src/typechecker.rs +++ b/src/typechecker.rs @@ -1026,6 +1026,7 @@ impl TypeContext { "Int" => Type::Int, "Float" => Type::Float, "String" => Type::String, + "Secret" => Type::Secret, "Bool" => Type::Bool, "Unit" | "()" => Type::Unit, "Any" => Type::Any, @@ -1405,6 +1406,15 @@ impl TypeContext { type_params, .. } => { + if name == "Secret" { + let line = self.find_line_near("struct Secret"); + self.error( + "'Secret' is reserved for opaque std/secrets values".to_string(), + line, + Some("Choose a different struct name".to_string()), + ); + return; + } // Store type parameter names for generic structs (e.g., struct Pair) let tp_names: Vec = type_params.iter().map(|t| t.name.clone()).collect(); if !tp_names.is_empty() { @@ -1422,6 +1432,15 @@ impl TypeContext { type_params: _, .. } => { + if name == "Secret" { + let line = self.find_line_near("enum Secret"); + self.error( + "'Secret' is reserved for opaque std/secrets values".to_string(), + line, + Some("Choose a different enum name".to_string()), + ); + return; + } let variant_types: Vec<(String, Option>)> = variants .iter() .map(|v| { @@ -1439,6 +1458,15 @@ impl TypeContext { target, type_params: _, } => { + if name == "Secret" { + let line = self.find_line_near("type Secret"); + self.error( + "'Secret' is reserved for opaque std/secrets values".to_string(), + line, + Some("Choose a different type alias name".to_string()), + ); + return; + } // Insert a placeholder first so self-references resolve to Type::Named(name) // rather than Type::Any during resolution (supports recursive type aliases). self.type_aliases @@ -4317,6 +4345,15 @@ fn get_module_signatures(module: &str) -> HashMap { sig!("args", [], Type::Array(Box::new(Type::String))); sig!("cwd", [], Type::String); } + "std/secrets" => { + let secret = Type::Secret; + sig!( + "get_secret", + ["name" => Type::String], + Type::Optional(Box::new(secret.clone())) + ); + sig!("require_secret", ["name" => Type::String], secret); + } "std/http" => { sig!("fetch", ["url_or_options" => Type::Union(vec![Type::String, Type::Map { key_type: Box::new(Type::String), value_type: Box::new(Type::Any) }])], Type::Generic { name: "Result".to_string(), diff --git a/src/types.rs b/src/types.rs index 1ff42601..3e49b1bc 100644 --- a/src/types.rs +++ b/src/types.rs @@ -23,6 +23,9 @@ pub enum Type { /// String type String, + /// Opaque provider-backed secret type + Secret, + /// Array type Array(Box), @@ -114,6 +117,7 @@ impl Type { (Type::Int, Type::Float) | (Type::Float, Type::Int) => true, // Allow numeric coercion (Type::Bool, Type::Bool) => true, (Type::String, Type::String) => true, + (Type::Secret, Type::Secret) => true, (Type::Array(a), Type::Array(b)) => a.is_compatible(b), (Type::Optional(a), Type::Optional(b)) => a.is_compatible(b), ( @@ -160,6 +164,7 @@ impl Type { Type::Float => "Float".to_string(), Type::Bool => "Bool".to_string(), Type::String => "String".to_string(), + Type::Secret => "Secret".to_string(), Type::Array(inner) => format!("[{}]", inner.name()), Type::Tuple(types) => { let names: Vec<_> = types.iter().map(|t| t.name()).collect(); diff --git a/tests/language_features_tests.rs b/tests/language_features_tests.rs index 38ec8741..3d7f81d9 100644 --- a/tests/language_features_tests.rs +++ b/tests/language_features_tests.rs @@ -650,6 +650,152 @@ print(greeting) ); } +#[test] +fn test_secret_prints_redacted_without_exposing_plaintext() { + let code = r#" +import { require_secret } from "std/secrets" +let token = require_secret("NTNT_TEMPLATE_SECRET") +print(token) +"#; + let (stdout, stderr, exit_code) = + run_ntnt_code_with_env(code, &[("NTNT_TEMPLATE_SECRET", "template-secret-canary")]); + assert_eq!(exit_code, 0, "stderr={stderr}"); + assert!(stdout.contains("[REDACTED]"), "stdout={stdout}"); + assert!(!stdout.contains("template-secret-canary")); + assert!(!stderr.contains("template-secret-canary")); +} + +#[test] +fn test_environment_secret_provider_is_disabled_in_production() { + let code = r#" +import { require_secret } from "std/secrets" +let token = require_secret("NTNT_PRODUCTION_SECRET") +print(token) +"#; + let (stdout, stderr, exit_code) = run_ntnt_code_with_env( + code, + &[ + ("NTNT_ENV", "production"), + ("NTNT_SECRETS_PROVIDER", "env"), + ("NTNT_PRODUCTION_SECRET", "production-secret-canary"), + ], + ); + assert_ne!(exit_code, 0, "stdout={stdout}\nstderr={stderr}"); + assert!(stderr.contains("development-only"), "stderr={stderr}"); + assert!(!stdout.contains("production-secret-canary")); + assert!(!stderr.contains("production-secret-canary")); +} + +#[test] +fn test_secret_manifest_allows_declared_names_and_rejects_undeclared_names() { + let app_dir = unique_test_dir("secret_manifest"); + fs::create_dir_all(&app_dir).expect("create app dir"); + let app_path = app_dir.join("main.tnt"); + let manifest_path = app_dir.join("ntnt.toml"); + write_test_file( + &app_path, + r#" +import { require_secret } from "std/secrets" +let token = require_secret("DECLARED_SECRET") +print(token) +"#, + ); + write_test_file( + &manifest_path, + r#" +[secrets.DECLARED_SECRET] +label = "Declared test secret" +required = true +"#, + ); + + let (stdout, stderr, exit_code) = + run_ntnt_file(&app_path, &[("DECLARED_SECRET", "manifest-secret-canary")]); + assert_eq!(exit_code, 0, "stdout={stdout}\nstderr={stderr}"); + assert!(stdout.contains("[REDACTED]")); + assert!(!stdout.contains("manifest-secret-canary")); + assert!(!stderr.contains("manifest-secret-canary")); + + write_test_file( + &app_path, + r#" +import { require_secret } from "std/secrets" +let token = require_secret("UNDECLARED_SECRET") +print(token) +"#, + ); + let (stdout, stderr, exit_code) = run_ntnt_file( + &app_path, + &[("UNDECLARED_SECRET", "undeclared-secret-canary")], + ); + assert_ne!(exit_code, 0, "stdout={stdout}\nstderr={stderr}"); + assert!(stderr.contains("not declared"), "stderr={stderr}"); + assert!(!stdout.contains("undeclared-secret-canary")); + assert!(!stderr.contains("undeclared-secret-canary")); + + write_test_file( + &app_path, + r#" +import { require_secret } from "std/secrets" +let token = require_secret("DECLARED_SECRET") +print(token) +"#, + ); + write_test_file( + &manifest_path, + r#" +[secrets.DECLARED_SECRET] +value = "manifest-value-canary" +"#, + ); + let (stdout, stderr, exit_code) = + run_ntnt_file(&app_path, &[("DECLARED_SECRET", "provider-value-canary")]); + assert_ne!(exit_code, 0, "stdout={stdout}\nstderr={stderr}"); + assert!(stderr.contains("declarations") && stderr.contains("invalid")); + assert!(!stdout.contains("manifest-value-canary")); + assert!(!stderr.contains("manifest-value-canary")); + assert!(!stdout.contains("provider-value-canary")); + assert!(!stderr.contains("provider-value-canary")); + + fs::remove_dir_all(&app_dir).ok(); +} + +#[test] +fn test_template_rejects_secret_interpolation_in_strict_mode() { + let code = r#" +import { require_secret } from "std/secrets" +let token = require_secret("NTNT_TEMPLATE_SECRET") +let body = """token={{token}}""" +print(body) +"#; + let (stdout, stderr, exit_code) = run_ntnt_code_with_env( + code, + &[ + ("NTNT_TEMPLATE_SECRET", "template-secret-canary"), + ("NTNT_TYPE_MODE", "strict"), + ], + ); + assert_ne!(exit_code, 0, "stdout={stdout}\nstderr={stderr}"); + assert!(!stdout.contains("template-secret-canary")); + assert!(!stderr.contains("template-secret-canary")); + assert!(stderr.contains("Secret"), "stderr={stderr}"); +} + +#[test] +fn test_nested_secret_equality_is_rejected() { + let code = r#" +import { require_secret } from "std/secrets" +let token = require_secret("NTNT_TEMPLATE_SECRET") +print(Some(token) == Some(token)) +"#; + let (stdout, stderr, exit_code) = + run_ntnt_code_with_env(code, &[("NTNT_TEMPLATE_SECRET", "template-secret-canary")]); + assert_ne!(exit_code, 0, "stdout={stdout}\nstderr={stderr}"); + assert!(!stdout.contains("template-secret-canary")); + assert!(!stderr.contains("template-secret-canary")); + assert!(stderr.contains("Secret"), "stderr={stderr}"); +} + #[test] fn test_template_string_css_passthrough() { let code = r#" diff --git a/tests/type_checker_tests.rs b/tests/type_checker_tests.rs index feb5a5ea..a98bf660 100644 --- a/tests/type_checker_tests.rs +++ b/tests/type_checker_tests.rs @@ -1503,3 +1503,61 @@ print(a + b) ); assert!(stdout.contains("amount > 0 is false"), "{stdout}"); } + +#[test] +fn test_std_secrets_require_secret_has_opaque_secret_type() { + let source = r#"import { require_secret } from "std/secrets" + +fn keep_secret(value: Secret) -> Secret { + return value +} + +let token = require_secret("API_KEY") +keep_secret(token) +"#; + let (stdout, stderr, exit) = lint_strict_code(source); + assert_eq!(exit, 0, "stderr={stderr}\nstdout={stdout}"); + assert!(!stdout.contains("type_error"), "{stdout}"); +} + +#[test] +fn test_std_secrets_secret_is_not_compatible_with_string() { + let source = r#"import { require_secret } from "std/secrets" + +fn takes_string(value: String) -> String { + return value +} + +let token = require_secret("API_KEY") +takes_string(token) +"#; + let (stdout, _stderr, _exit) = lint_strict_code(source); + assert!( + stdout.contains("\"rule\": \"type_check\""), + "expected type error: {stdout}" + ); + assert!(stdout.contains("Secret"), "expected Secret type: {stdout}"); + assert!(stdout.contains("String"), "expected String type: {stdout}"); +} + +#[test] +fn test_std_secrets_type_cannot_collide_with_user_defined_secret() { + let source = r#"import { require_secret } from "std/secrets" + +struct Secret { + value: String +} + +fn read_user_secret(value: Secret) -> String { + return value.value +} + +let token = require_secret("API_KEY") +read_user_secret(token) +"#; + let (stdout, _stderr, _exit) = lint_strict_code(source); + assert!( + stdout.contains("\"rule\": \"type_check\""), + "provider Secret must not unify with a user-defined Secret: {stdout}" + ); +} From c4d35a54e82e6609552f4fffba50359375b96d68 Mon Sep 17 00:00:00 2001 From: Larri Date: Mon, 13 Jul 2026 11:22:30 -0600 Subject: [PATCH 2/3] fix: harden secret serialization invariants --- src/interpreter.rs | 4 ++++ src/stdlib/kv.rs | 5 +++++ src/stdlib/secrets.rs | 16 +++++++++++----- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/interpreter.rs b/src/interpreter.rs index 9759400b..d5c103ad 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -8575,6 +8575,8 @@ impl Interpreter { } } } + // Filter arguments can introduce a Secret (for example, + // `default(secret)`), so re-check the result before rendering. if value.contains_secret() { Self::handle_template_error( IntentError::type_error( @@ -8637,6 +8639,8 @@ impl Interpreter { } } } + // Filter arguments can introduce a Secret (for example, + // `default(secret)`), so re-check the result before rendering. if value.contains_secret() { Self::handle_template_error( IntentError::type_error( diff --git a/src/stdlib/kv.rs b/src/stdlib/kv.rs index cf9be001..ee5bcd2d 100644 --- a/src/stdlib/kv.rs +++ b/src/stdlib/kv.rs @@ -154,6 +154,7 @@ fn value_to_json(value: &Value) -> serde_json::Value { .map(serde_json::Value::Number) .unwrap_or(serde_json::Value::Null), Value::String(s) => serde_json::Value::String(s.clone()), + Value::Secret(_) => serde_json::Value::String(crate::secret::REDACTED_SECRET.to_string()), Value::Array(arr) => serde_json::Value::Array(arr.iter().map(value_to_json).collect()), Value::Map(map) => { let obj: serde_json::Map = map @@ -3044,5 +3045,9 @@ mod tests { let envelope_error = serialize_value_envelope(&nested).expect_err("Redis envelope must reject secrets"); assert!(!envelope_error.to_string().contains("kv-secret-canary")); + + let defensive_json = value_to_json_public(&nested).to_string(); + assert!(defensive_json.contains(crate::secret::REDACTED_SECRET)); + assert!(!defensive_json.contains("kv-secret-canary")); } } diff --git a/src/stdlib/secrets.rs b/src/stdlib/secrets.rs index 430e42bc..e875da0e 100644 --- a/src/stdlib/secrets.rs +++ b/src/stdlib/secrets.rs @@ -45,12 +45,20 @@ fn declaration_state() -> &'static RwLock { pub(crate) fn configure_for_source(source_path: &str) { let (identity, declarations) = load_declarations(Path::new(source_path)); if let Ok(mut state) = declaration_state().write() { + if matches!( + state.declarations, + SecretDeclarations::ConflictingApplication + ) { + return; + } match &state.identity { None => { state.identity = Some(identity); state.declarations = declarations; } Some(current) if current == &identity => state.declarations = declarations, + // A mixed application identity is a process-level security violation. + // Keep the state permanently poisoned; only a process restart may reset it. Some(_) => state.declarations = SecretDeclarations::ConflictingApplication, } } @@ -212,7 +220,6 @@ impl ProviderGroup { } fn lookup(&self, name: &str) -> Result> { - validate_secret_name(name)?; let mut unavailable = Vec::new(); let mut attempts = 0; @@ -316,6 +323,8 @@ fn configured_provider_group() -> Result { } fn lookup_secret(name: &str) -> Result> { + // Validate before the name can reach declaration diagnostics or a provider. + // SecretValue validates again only to preserve its own construction invariant. validate_secret_name(name)?; enforce_declared(name)?; configured_provider_group()?.lookup(name) @@ -323,10 +332,7 @@ fn lookup_secret(name: &str) -> Result> { fn secret_name_arg(args: &[Value], function: &str) -> Result { match args.first() { - Some(Value::String(name)) => { - validate_secret_name(name)?; - Ok(name.clone()) - } + Some(Value::String(name)) => Ok(name.clone()), _ => Err(IntentError::type_error(format!( "{function}() requires a secret name string" ))), From 9a5f5f022846c9ae18f28a297fa9530bf5b5cd99 Mon Sep 17 00:00:00 2001 From: Larri Date: Mon, 13 Jul 2026 11:37:30 -0600 Subject: [PATCH 3/3] fix: stabilize manifest declaration enforcement --- src/stdlib/secrets.rs | 83 +++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/src/stdlib/secrets.rs b/src/stdlib/secrets.rs index e875da0e..01f48e8f 100644 --- a/src/stdlib/secrets.rs +++ b/src/stdlib/secrets.rs @@ -44,23 +44,35 @@ fn declaration_state() -> &'static RwLock { #[cfg_attr(test, allow(dead_code))] pub(crate) fn configure_for_source(source_path: &str) { let (identity, declarations) = load_declarations(Path::new(source_path)); - if let Ok(mut state) = declaration_state().write() { - if matches!( - state.declarations, - SecretDeclarations::ConflictingApplication - ) { + let lock = declaration_state(); + let mut state = match lock.write() { + Ok(state) => state, + Err(poisoned) => { + // Recover the guard only to record a durable fail-closed state. Clearing + // the poison makes later lookups report the explicit manifest error. + let mut state = poisoned.into_inner(); + state.declarations = SecretDeclarations::Invalid; + drop(state); + lock.clear_poison(); return; } - match &state.identity { - None => { - state.identity = Some(identity); - state.declarations = declarations; - } - Some(current) if current == &identity => state.declarations = declarations, - // A mixed application identity is a process-level security violation. - // Keep the state permanently poisoned; only a process restart may reset it. - Some(_) => state.declarations = SecretDeclarations::ConflictingApplication, + }; + + if matches!( + state.declarations, + SecretDeclarations::ConflictingApplication + ) { + return; + } + match &state.identity { + None => { + state.identity = Some(identity); + state.declarations = declarations; } + Some(current) if current == &identity => state.declarations = declarations, + // A mixed application identity is a process-level security violation. + // Keep the state permanently poisoned; only a process restart may reset it. + Some(_) => state.declarations = SecretDeclarations::ConflictingApplication, } } @@ -80,25 +92,24 @@ fn load_declarations(source_path: &Path) -> (PathBuf, SecretDeclarations) { let Some(manifest) = manifest else { return (source_identity, SecretDeclarations::NoManifest); }; - let identity = manifest.canonicalize().unwrap_or_else(|_| manifest.clone()); let Ok(content) = std::fs::read_to_string(manifest) else { - return (identity, SecretDeclarations::Invalid); + return (source_identity, SecretDeclarations::Invalid); }; let Ok(document) = content.parse::() else { - return (identity, SecretDeclarations::Invalid); + return (source_identity, SecretDeclarations::Invalid); }; let Some(secrets) = document.get("secrets") else { - return (identity, SecretDeclarations::Loaded(HashSet::new())); + return (source_identity, SecretDeclarations::Loaded(HashSet::new())); }; let Some(table) = secrets.as_table() else { - return (identity, SecretDeclarations::Invalid); + return (source_identity, SecretDeclarations::Invalid); }; let mut declared = HashSet::with_capacity(table.len()); for (name, metadata) in table { let Some(metadata) = metadata.as_table() else { - return (identity, SecretDeclarations::Invalid); + return (source_identity, SecretDeclarations::Invalid); }; if validate_secret_name(name).is_err() || metadata.keys().any(|key| { @@ -120,11 +131,11 @@ fn load_declarations(source_path: &Path) -> (PathBuf, SecretDeclarations) { .is_none_or(|items| items.iter().any(|item| !item.is_str())) }) { - return (identity, SecretDeclarations::Invalid); + return (source_identity, SecretDeclarations::Invalid); } declared.insert(name.clone()); } - (identity, SecretDeclarations::Loaded(declared)) + (source_identity, SecretDeclarations::Loaded(declared)) } fn enforce_declared(name: &str) -> Result<()> { @@ -427,6 +438,34 @@ mod tests { use std::collections::VecDeque; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Mutex; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn declaration_identity_is_stable_when_manifest_appears() { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "ntnt-secret-identity-{}-{suffix}", + std::process::id() + )); + std::fs::create_dir_all(&root).expect("create project"); + let source = root.join("main.tnt"); + std::fs::write(&source, "print(\"ok\")\n").expect("write source"); + + let (without_manifest, _) = load_declarations(&source); + std::fs::write( + root.join("ntnt.toml"), + "[secrets.API_KEY]\nrequired = true\n", + ) + .expect("write manifest"); + let (with_manifest, declarations) = load_declarations(&source); + + assert_eq!(without_manifest, with_manifest); + assert!(matches!(declarations, SecretDeclarations::Loaded(_))); + std::fs::remove_dir_all(root).ok(); + } struct MockProvider { endpoint: String,