diff --git a/docs/STDLIB_REFERENCE.md b/docs/STDLIB_REFERENCE.md index 5f17951..726f13c 100644 --- a/docs/STDLIB_REFERENCE.md +++ b/docs/STDLIB_REFERENCE.md @@ -6733,7 +6733,7 @@ fetch(url_or_options: String | Map, options?: Map) -> Result Make an HTTP request to a URL. -Accepts one or two arguments: - One argument: a URL string for a simple GET request, or an options map with full control over method, headers, body, authentication, cookies, and timeout. - Two arguments: a URL string and an options map. The URL is merged into the options map automatically. Options map keys: url (set automatically in 2-arg form), method, headers, body, json, form, auth, cookies, timeout. Opaque Secret values are accepted only in header values, cookie values, basic-auth fields, raw bodies, JSON leaves, and form values. Requests containing a Secret do not follow redirects, preventing credentials or 307/308 bodies from crossing origins. +Accepts one or two arguments: - One argument: a URL string for a simple GET request, or an options map with full control over method, headers, body, authentication, cookies, and timeout. - Two arguments: a URL string and an options map. The URL is merged into the options map automatically. Options map keys: url (set automatically in 2-arg form), method, headers, body, json, form, auth, cookies, timeout. Opaque Secret values are accepted only in header values, cookie values, basic-auth fields, raw bodies, JSON leaves, and form values. Secret-bearing requests require HTTPS; APP_ENV=development permits direct HTTP only for localhost and loopback IPs, bypassing system proxies. Requests containing a Secret do not follow redirects, preventing credentials or 307/308 bodies from crossing origins. **Parameters:** @@ -6766,6 +6766,7 @@ fetch("https://api.example.com", map { - **TypeError**: fetch() requires a URL string or options map — *Fix: Pass a String URL or a Map with request options* - **TypeError**: fetch() requires 'url' option — *Fix: Include 'url' key in the options map* +- **TypeError**: Secret-bearing HTTP requests require HTTPS — *Fix: Use HTTPS, or set APP_ENV=development for localhost/loopback HTTP* - **RuntimeError**: Unsupported HTTP method: ... — *Fix: Use GET, POST, PUT, DELETE, PATCH, or HEAD* **See also:** `download`, `cache_fetch` @@ -10936,7 +10937,7 @@ require_secret(name: String) -> Secret Looks up a required secret and fails closed when it is not configured. -Use this at startup or immediately before an approved secret-consuming sink. In v0.5.1, `std/http.fetch` accepts Secret values as header, cookie, basic-auth, raw body, JSON-leaf, and form values. Templates, public JSON, URL/CSV/string conversion, KV storage, and job payloads reject secrets. There is intentionally no general Secret-to-String reveal function. The error identifies only the logical name and never includes the value. +Use this at startup or immediately before an approved secret-consuming sink. In v0.5.1, `std/http.fetch` accepts Secret values as header, cookie, basic-auth, raw body, JSON-leaf, and form values. These requests require HTTPS; APP_ENV=development permits direct HTTP only for localhost and loopback IPs. Templates, public JSON, URL/CSV/string conversion, KV storage, and job payloads reject secrets. There is intentionally no general Secret-to-String reveal function. The error identifies only the logical name and never includes the value. **Parameters:** diff --git a/src/std_secrets_tests.rs b/src/std_secrets_tests.rs index 94541a6..cf37815 100644 --- a/src/std_secrets_tests.rs +++ b/src/std_secrets_tests.rs @@ -245,8 +245,8 @@ fn fetch_and_capture(mut options: HashMap) -> (Value, String) { String::from_utf8(bytes).expect("request is utf-8") }); - let module = crate::stdlib::http::init(); - let result = native_fn(&module, "fetch")(&[Value::Map(options)]).expect("fetch executes"); + let result = crate::stdlib::http::http_fetch_with_app_env(&options, Some("development")) + .expect("fetch executes"); (result, capture.join().expect("capture thread")) } @@ -368,8 +368,8 @@ fn http_fetch_does_not_follow_redirects_when_request_contains_secrets() { "headers".to_string(), Value::Map(HashMap::from([("x-api-key".to_string(), test_secret())])), ); - let module = crate::stdlib::http::init(); - native_fn(&module, "fetch")(&[Value::Map(options)]).expect("fetch redirect response"); + crate::stdlib::http::http_fetch_with_app_env(&options, Some("development")) + .expect("fetch redirect response"); redirect_server.join().expect("redirect server"); let forwarded = target_request.join().expect("target thread"); diff --git a/src/stdlib/http.rs b/src/stdlib/http.rs index 8e763bd..0fd0887 100644 --- a/src/stdlib/http.rs +++ b/src/stdlib/http.rs @@ -7,6 +7,9 @@ //! - `download(url, path)` - Download file to disk //! - `Cache(ttl)` - Create a response cache //! +//! Requests containing `Secret` values require HTTPS. `APP_ENV=development` +//! permits direct plain HTTP only for `localhost` and loopback IP addresses. +//! //! # Options for fetch() //! //! - `url`: Request URL (required when using options map) @@ -27,7 +30,7 @@ use reqwest::header::{AUTHORIZATION, COOKIE, SET_COOKIE}; use std::collections::HashMap; use std::fs::File; use std::io::Write; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, ToSocketAddrs}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, ToSocketAddrs}; use std::path::Path; use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant}; @@ -656,8 +659,69 @@ fn form_scalar(value: &Value) -> Result { } } +fn is_loopback_url(url: &reqwest::Url) -> bool { + let Some(host) = url.host_str() else { + return false; + }; + let host = host + .strip_prefix('[') + .and_then(|value| value.strip_suffix(']')) + .unwrap_or(host); + + host.eq_ignore_ascii_case("localhost") + || host + .parse::() + .is_ok_and(|address| address.is_loopback()) +} + +fn validate_secret_transport( + url: &str, + contains_secret: bool, + app_env: Option<&str>, +) -> Result { + if !contains_secret { + return Ok(false); + } + + let parsed = reqwest::Url::parse(url).map_err(|_| { + IntentError::type_error("Secret-bearing HTTP requests require a valid URL".to_string()) + })?; + + if parsed.scheme() == "https" { + return Ok(false); + } + + let development = app_env.is_some_and(|value| value.eq_ignore_ascii_case("development")); + if development && parsed.scheme() == "http" && is_loopback_url(&parsed) { + return Ok(true); + } + + Err(IntentError::type_error( + "Secret-bearing HTTP requests require HTTPS; APP_ENV=development permits HTTP only for localhost or loopback IPs" + .to_string(), + )) +} + +fn format_ssrf_error(reason: &str, direct_loopback_http: bool) -> String { + if direct_loopback_http { + format!( + "SSRF protection: {reason}. APP_ENV=development permits plaintext loopback transport, but NTNT's SSRF policy remains independent" + ) + } else { + format!("SSRF protection: {reason}") + } +} + /// Full HTTP request with all options fn http_fetch(opts: &HashMap) -> Result { + let app_env = std::env::var("APP_ENV").ok(); + http_fetch_with_app_env(opts, app_env.as_deref()) +} + +pub(crate) fn http_fetch_with_app_env( + opts: &HashMap, + app_env: Option<&str>, +) -> Result { // Cancellation yield point (rule 19): check before making the network request if crate::stdlib::concurrent::is_current_task_cancelled() { return Err(IntentError::runtime_error("Task cancelled".to_string())); @@ -672,11 +736,14 @@ fn http_fetch(opts: &HashMap) -> Result { } }; + let contains_secret = opts.values().any(Value::contains_secret); + let direct_loopback_http = validate_secret_transport(&url, contains_secret, app_env)?; + // SSRF protection: validate URL before making request if let Err(reason) = validate_url_for_ssrf(&url) { - return Ok(Value::err(Value::String(format!( - "SSRF protection: {}", - reason + return Ok(Value::err(Value::String(format_ssrf_error( + &reason, + direct_loopback_http, )))); } @@ -694,9 +761,27 @@ fn http_fetch(opts: &HashMap) -> Result { // Secret-bearing requests never follow redirects: custom credentials and // 307/308 bodies could otherwise be forwarded to a different origin. let mut client_builder = reqwest::blocking::Client::builder().cookie_store(true); - if opts.values().any(Value::contains_secret) { + if contains_secret { client_builder = client_builder.redirect(reqwest::redirect::Policy::none()); } + if direct_loopback_http { + // Plaintext development traffic must remain on loopback even when the process + // has system proxy settings or a nonstandard localhost resolver. + client_builder = client_builder.no_proxy(); + + let parsed = reqwest::Url::parse(&url).expect("secret transport URL was validated"); + if parsed + .host_str() + .is_some_and(|host| host.eq_ignore_ascii_case("localhost")) + { + // reqwest uses the URL's port; zero is only a DNS-override placeholder. + let loopback = [ + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0), + ]; + client_builder = client_builder.resolve_to_addrs("localhost", &loopback); + } + } let client = client_builder .build() .map_err(|e| IntentError::runtime_error(format!("Failed to create HTTP client: {}", e)))?; @@ -952,8 +1037,10 @@ pub fn init() -> HashMap { // the options map automatically. // Options map keys: url (set automatically in 2-arg form), method, headers, body, json, form, auth, cookies, timeout. // Opaque Secret values are accepted only in header values, cookie values, basic-auth - // fields, raw bodies, JSON leaves, and form values. Requests containing a Secret - // do not follow redirects, preventing credentials or 307/308 bodies from crossing origins. + // fields, raw bodies, JSON leaves, and form values. Secret-bearing requests require + // HTTPS; APP_ENV=development permits direct HTTP only for localhost and loopback IPs, + // bypassing system proxies. Requests containing a Secret do not follow redirects, + // preventing credentials or 307/308 bodies from crossing origins. // @param url_or_options A URL string for GET, or a Map with request options // @param options (optional) A Map with request options when first argument is a URL string // @returns Result where Response is a Map with status, status_text, headers, body, ok, url, redirected, and cookies fields @@ -977,6 +1064,7 @@ pub fn init() -> HashMap { // @expected Ok({status: 201, ...}) // @error TypeError ~ "fetch() requires a URL string or options map" fix: "Pass a String URL or a Map with request options" // @error TypeError ~ "fetch() requires 'url' option" fix: "Include 'url' key in the options map" + // @error TypeError ~ "Secret-bearing HTTP requests require HTTPS" fix: "Use HTTPS, or set APP_ENV=development for localhost/loopback HTTP" // @error RuntimeError ~ "Unsupported HTTP method: ..." fix: "Use GET, POST, PUT, DELETE, PATCH, or HEAD" module.insert( "fetch".to_string(), @@ -1246,6 +1334,59 @@ pub fn init() -> HashMap { mod tests { use super::*; + #[test] + fn secret_transport_requires_https_by_default() { + let error = validate_secret_transport("http://127.0.0.1:8080/api", true, None) + .expect_err("secret-bearing HTTP must fail closed without APP_ENV=development"); + assert!(error.to_string().contains("HTTPS")); + + assert!(validate_secret_transport("https://api.example.com/v1", true, None).is_ok()); + assert!(validate_secret_transport("http://api.example.com/v1", false, None).is_ok()); + } + + #[test] + fn development_allows_secret_http_only_to_loopback_hosts() { + for url in [ + "http://localhost:8080/api", + "http://127.0.0.1:8080/api", + "http://127.42.0.9:8080/api", + "http://[::1]:8080/api", + ] { + assert!( + validate_secret_transport(url, true, Some("development")).is_ok(), + "development should allow loopback URL: {url}" + ); + } + + for url in [ + "http://example.com/api", + "http://localhost.example.com/api", + "http://localhost./api", + "http://10.0.0.8/api", + "http://192.168.1.10/api", + "http://0.0.0.0:8080/api", + "http://[::]:8080/api", + ] { + assert!( + validate_secret_transport(url, true, Some("development")).is_err(), + "development must reject non-loopback URL: {url}" + ); + } + + assert!( + validate_secret_transport("http://localhost:8080/api", true, Some("production")) + .is_err() + ); + assert!(validate_secret_transport("http://localhost:8080/api", true, Some("dev")).is_err()); + let invalid = validate_secret_transport("not a URL", true, Some("development")) + .expect_err("malformed secret-bearing URL must fail"); + assert!(invalid.to_string().contains("valid URL")); + + let ssrf_error = format_ssrf_error("Localhost requests blocked", true); + assert!(ssrf_error.contains("APP_ENV=development")); + assert!(ssrf_error.contains("SSRF policy remains independent")); + } + #[test] fn cache_fetch_rejects_secret_bearing_options_before_cache_lookup() { let canary = "cache-secret-canary"; diff --git a/src/stdlib/secrets.rs b/src/stdlib/secrets.rs index 01f48e8..683ca0b 100644 --- a/src/stdlib/secrets.rs +++ b/src/stdlib/secrets.rs @@ -399,8 +399,10 @@ pub fn init() -> HashMap { // // Use this at startup or immediately before an approved secret-consuming sink. // In v0.5.1, `std/http.fetch` accepts Secret values as header, cookie, - // basic-auth, raw body, JSON-leaf, and form values. Templates, public JSON, - // URL/CSV/string conversion, KV storage, and job payloads reject secrets. + // basic-auth, raw body, JSON-leaf, and form values. These requests require HTTPS; + // APP_ENV=development permits direct HTTP only for localhost and loopback IPs. + // Templates, public JSON, URL/CSV/string conversion, KV storage, and job payloads + // reject secrets. // There is intentionally no general Secret-to-String reveal function. // The error identifies only the logical name and never includes the value. // @param name A validated logical secret name diff --git a/tests/language_features_tests.rs b/tests/language_features_tests.rs index 3d7f81d..4dd11f4 100644 --- a/tests/language_features_tests.rs +++ b/tests/language_features_tests.rs @@ -6,12 +6,53 @@ //! - CSV parsing use std::fs; -use std::io::Write; +use std::io::{Read, Write}; +use std::net::TcpListener; use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; static TEST_COUNTER: AtomicU64 = AtomicU64::new(0); +fn capture_http_request(listener: TcpListener) -> std::thread::JoinHandle> { + listener + .set_nonblocking(true) + .expect("set HTTP capture nonblocking"); + std::thread::spawn(move || { + for _ in 0..300 { + match listener.accept() { + Ok((mut stream, _)) => { + let mut request = Vec::new(); + stream + .set_read_timeout(Some(std::time::Duration::from_secs(2))) + .expect("request read timeout"); + let mut chunk = [0_u8; 1024]; + loop { + let read = stream.read(&mut chunk).expect("read local request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", + ) + .expect("write local response"); + return Some(String::from_utf8_lossy(&request).to_string()); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + Err(error) => panic!("HTTP capture accept failed: {error}"), + } + } + None + }) +} + /// Generate a unique test file path fn unique_test_file(prefix: &str) -> String { let counter = TEST_COUNTER.fetch_add(1, Ordering::SeqCst); @@ -686,6 +727,121 @@ print(token) assert!(!stderr.contains("production-secret-canary")); } +#[test] +fn test_secret_bearing_http_requires_https_outside_app_development() { + let code = r#" +import { fetch } from "std/http" +import { require_secret } from "std/secrets" +let token = require_secret("NTNT_HTTP_TRANSPORT_SECRET") +fetch(map { + "url": "http://127.0.0.1:9/private", + "headers": map { "authorization": token } +}) +"#; + let (stdout, stderr, exit_code) = run_ntnt_code_with_env( + code, + &[ + ("APP_ENV", "production"), + ("NTNT_HTTP_TRANSPORT_SECRET", "http-transport-secret-canary"), + ], + ); + + assert_ne!(exit_code, 0, "stdout={stdout}\nstderr={stderr}"); + assert!(stderr.contains("require HTTPS"), "stderr={stderr}"); + assert!(!stdout.contains("http-transport-secret-canary")); + assert!(!stderr.contains("http-transport-secret-canary")); +} + +#[test] +fn test_app_development_allows_secret_bearing_http_to_loopback() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind local HTTP server"); + let port = listener.local_addr().expect("local HTTP address").port(); + let capture = capture_http_request(listener); + + let code = format!( + r#" +import {{ fetch }} from "std/http" +import {{ require_secret }} from "std/secrets" +let token = require_secret("NTNT_HTTP_TRANSPORT_SECRET") +fetch(map {{ + "url": "http://localhost:{port}/private", + "headers": map {{ "x-api-key": token }} +}}) +"# + ); + let (stdout, stderr, exit_code) = run_ntnt_code_with_env( + &code, + &[ + ("APP_ENV", "development"), + ("NTNT_HTTP_TRANSPORT_SECRET", "http-transport-secret-canary"), + ], + ); + + assert_eq!(exit_code, 0, "stdout={stdout}\nstderr={stderr}"); + let request = capture + .join() + .expect("capture local request") + .expect("loopback server received request"); + assert!( + request.contains("x-api-key: http-transport-secret-canary"), + "request={request}" + ); + assert!(!stdout.contains("http-transport-secret-canary")); + assert!(!stderr.contains("http-transport-secret-canary")); +} + +#[test] +fn test_app_development_loopback_secret_http_bypasses_system_proxy() { + let target = TcpListener::bind("127.0.0.1:0").expect("bind loopback target"); + let target_address = target.local_addr().expect("loopback target address"); + let target_capture = capture_http_request(target); + + let proxy = TcpListener::bind("127.0.0.1:0").expect("bind proxy capture"); + let proxy_address = proxy.local_addr().expect("proxy capture address"); + let proxy_capture = capture_http_request(proxy); + + let code = format!( + r#" +import {{ fetch }} from "std/http" +import {{ require_secret }} from "std/secrets" +let token = require_secret("NTNT_HTTP_TRANSPORT_SECRET") +fetch(map {{ + "url": "http://{target_address}/private", + "headers": map {{ "x-api-key": token }} +}}) +"# + ); + let proxy_url = format!("http://{proxy_address}"); + let (stdout, stderr, exit_code) = run_ntnt_code_with_env( + &code, + &[ + ("APP_ENV", "development"), + ("NTNT_HTTP_TRANSPORT_SECRET", "proxy-secret-canary"), + ("HTTP_PROXY", &proxy_url), + ("http_proxy", &proxy_url), + ("NO_PROXY", ""), + ("no_proxy", ""), + ], + ); + + assert_eq!(exit_code, 0, "stdout={stdout}\nstderr={stderr}"); + let target_request = target_capture + .join() + .expect("target capture") + .expect("loopback target received request"); + let proxy_request = proxy_capture.join().expect("proxy capture"); + assert!( + target_request.contains("x-api-key: proxy-secret-canary"), + "target request={target_request}" + ); + assert!( + proxy_request.is_none(), + "secret-bearing loopback request reached proxy: {proxy_request:?}" + ); + assert!(!stdout.contains("proxy-secret-canary")); + assert!(!stderr.contains("proxy-secret-canary")); +} + #[test] fn test_secret_manifest_allows_declared_names_and_rejects_undeclared_names() { let app_dir = unique_test_dir("secret_manifest");