feat(export): persistent on-disk helmTemplate cache

.github/workflows/benchmarks.yaml

@@ -90,6 +90,9 @@ jobs:
           - name: "Validate Manifests"
             config: "rtk-benchmarks/validate-manifests.yaml"
             id: "validate-manifests"
+          - name: "Helm Template"
+            config: "rtk-benchmarks/helm-template.yaml"
+            id: "helm-template"
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.4
@@ -122,6 +125,11 @@ jobs:
           chmod +x /usr/local/bin/tk
           tk --version
 
+      - name: Install Helm
+        run: |
+          curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          helm version
+
       - name: Install uv
         uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
 

cmds/rtk/src/commands/export.rs

@@ -68,6 +68,11 @@ pub struct ExportArgs {
 	#[arg(long)]
 	pub skip_manifest: bool,
 
+	/// Experimental: maintain a `helm-cache/` directory in the output dir to
+	/// cache helmTemplate results across runs and environments.
+	#[arg(long)]
+	pub helm_cache: bool,
+
 	/// Regex filter on '<kind>/<name>'. See https://tanka.dev/output-filtering
 	#[arg(short = 't', long)]
 	pub target: Vec<String>,
@@ -170,6 +175,7 @@ fn build_export_opts(args: ExportArgs) -> Result<(Vec<PathBuf>, ExportOpts)> {
 		merge_strategy,
 		merge_deleted_envs: args.merge_deleted_envs,
 		show_timing: false,
+		helm_cache: args.helm_cache,
 	};
 	Ok((paths, opts))
 }

cmds/rtk/src/environments/export.rs

@@ -25,6 +25,7 @@ use tracing::{debug, trace};
 use super::{compile_target_matchers, keep_target, manifest_kind_name, TargetMatcher};
 use crate::{
 	environments::discover::{Discover, Discovered},
+	environments::helm_cache,
 	jsonnet::evaluator::{DefaultEvaluator, Evaluator, EvaluatorOptions, GlobalEvaluatorOptions},
 	yaml::sort_json_keys,
 };
@@ -128,6 +129,9 @@ pub struct ExportOpts {
 	pub merge_deleted_envs: Vec<String>,
 	/// Show detailed timing breakdown
 	pub show_timing: bool,
+	/// Maintain a `helm-cache/` metadata directory in the output dir to cache
+	/// `helmTemplate` results across runs and environments (experimental).
+	pub helm_cache: bool,
 }
 
 impl Default for ExportOpts {
@@ -147,6 +151,7 @@ impl Default for ExportOpts {
 			merge_strategy: ExportMergeStrategy::default(),
 			merge_deleted_envs: vec![],
 			show_timing: false,
+			helm_cache: false,
 		}
 	}
 }
@@ -421,6 +426,18 @@ pub fn export(paths: &[PathBuf], opts: ExportOpts) -> Result<ExportResult> {
 	fs::create_dir_all(&opts.output_dir)
 		.context(format!("creating output directory {:?}", opts.output_dir))?;
 
+	// Experimental helm-cache: load the single global on-disk cache exactly
+	// once, before the parallel loop. Entries are preloaded into the
+	// process-global in-memory cache (shared across all worker threads) and the
+	// directory is removed so only entries touched this run are written back.
+	// Done before the emptiness check so a stale cache dir does not trip it.
+	let helm_cache_dir = opts
+		.helm_cache
+		.then(|| helm_cache::cache_dir(&opts.output_dir));
+	if let Some(ref dir) = helm_cache_dir {
+		helm_cache::load_and_clear(dir);
+	}
+
 	// Check if directory is empty (if required by merge strategy)
 	if opts.merge_strategy == ExportMergeStrategy::None {
 		trace!("Checking if output directory is empty (merge_strategy=none)");
@@ -551,6 +568,12 @@ pub fn export(paths: &[PathBuf], opts: ExportOpts) -> Result<ExportResult> {
 		parallel_start.elapsed().as_millis()
 	);
 
+	// Experimental helm-cache: persist the touched entries exactly once, now
+	// that the parallel loop is done (single-threaded, so writes never race).
+	if let Some(ref dir) = helm_cache_dir {
+		helm_cache::save(dir);
+	}
+
 	// Summarize results
 	let total_envs = results.len();
 	let successful = results.iter().filter(|r| r.error.is_none()).count();
@@ -777,6 +800,19 @@ fn export_single_env(
 		..Default::default()
 	};
 
+	// Environment identifier: the path to main.jsonnet (relative to the working
+	// directory when possible).
+	let main_jsonnet_path = env.path.join("main.jsonnet");
+	let env_namespace = if let Ok(cwd) = std::env::current_dir() {
+		main_jsonnet_path
+			.strip_prefix(&cwd)
+			.unwrap_or(&main_jsonnet_path)
+			.to_string_lossy()
+			.to_string()
+	} else {
+		main_jsonnet_path.to_string_lossy().to_string()
+	};
+
 	trace!("[{}] Starting Jsonnet evaluation", env_display);
 	let eval_start = Instant::now();
 	let evaluator = DefaultEvaluator::new(opts.eval_opts.clone());
@@ -802,20 +838,6 @@ fn export_single_env(
 		));
 	}
 
-	// Extract environment identifier for manifest.json tracking
-	// This should be the path to main.jsonnet (relative to working directory if possible)
-	let main_jsonnet_path = env.path.join("main.jsonnet");
-	let env_namespace = if let Ok(cwd) = std::env::current_dir() {
-		// Make path relative to current directory if possible
-		main_jsonnet_path
-			.strip_prefix(&cwd)
-			.unwrap_or(&main_jsonnet_path)
-			.to_string_lossy()
-			.to_string()
-	} else {
-		main_jsonnet_path.to_string_lossy().to_string()
-	};
-
 	// Extract Environment objects (matching Tanka's inline.go/static.go pattern)
 	// Move result.value and result.spec to avoid cloning the entire JSON tree
 	trace!("[{}] Extracting Environment objects", env_display);

cmds/rtk/src/environments/helm_cache.rs

@@ -0,0 +1,234 @@
+//! On-disk `helmTemplate` cache for exports (experimental).
+//!
+//! The export driver maintains a single global `helm-cache/` metadata directory
+//! inside the export output directory. It stores the rendered output of every
+//! `helmTemplate` call under:
+//!
+//! ```text
+//! <output_dir>/helm-cache/<sha256>.json
+//! ```
+//!
+//! where `<sha256>` is a hash of the entire `helmTemplate` call (release name,
+//! chart path + `Chart.yaml`, namespace, values, flags, `nameFormat`, ...) and
+//! the JSON file contains the map of all resources that call produced.
+//!
+//! The cache is global and is loaded and written exactly once per export run:
+//!
+//! 1. Before the parallel export loop, [`load_and_clear`] reads every cached
+//!    entry into the process-global in-memory Helm cache (shared across all
+//!    worker threads, so a chart rendered for one environment is reused by
+//!    every other) and then deletes the directory. It also begins recording
+//!    which cache keys get touched.
+//! 2. During evaluation, each `helmTemplate` call that hits the in-memory cache
+//!    is served without invoking helm or parsing YAML, and its key is recorded
+//!    as "touched" (across all threads).
+//! 3. After the parallel loop completes, [`save`] writes only the touched
+//!    entries back. Stale entries that were not referenced this run are pruned,
+//!    because the directory was deleted in step 1 and only touched keys are
+//!    rewritten.
+
+use std::{
+	collections::HashSet,
+	path::{Path, PathBuf},
+};
+
+use anyhow::{Context, Result};
+use tracing::{debug, warn};
+
+use crate::jsonnet::evaluator::jrsonnet::builtins;
+
+/// Name of the metadata directory created inside the export output directory.
+pub const HELM_CACHE_DIR: &str = "helm-cache";
+
+/// Resolve the single global cache directory inside `output_dir`.
+pub fn cache_dir(output_dir: &Path) -> PathBuf {
+	output_dir.join(HELM_CACHE_DIR)
+}
+
+/// Begin recording touched cache keys for the whole run, preload any previously
+/// persisted entries into the global in-memory cache, then delete the directory
+/// so only entries touched this run are written back.
+///
+/// Best-effort: a failure to read the existing cache is logged and ignored.
+/// Must be called once, before the parallel export loop.
+pub fn load_and_clear(dir: &Path) {
+	builtins::helm_disk_cache_begin();
+
+	if !dir.exists() {
+		return;
+	}
+
+	if let Err(err) = preload(dir) {
+		warn!("helm-cache: failed to preload {}: {err:#}", dir.display());
+	}
+
+	if let Err(err) = std::fs::remove_dir_all(dir) {
+		warn!(
+			"helm-cache: failed to clear {} (stale entries may persist): {err:#}",
+			dir.display()
+		);
+	}
+}
+
+fn preload(dir: &Path) -> Result<()> {
+	let mut loaded = 0usize;
+	for entry in std::fs::read_dir(dir).context("reading helm-cache directory")? {
+		let entry = entry.context("reading helm-cache entry")?;
+		let path = entry.path();
+		if path.extension().and_then(|e| e.to_str()) != Some("json") {
+			continue;
+		}
+		let Some(key) = path.file_stem().and_then(|s| s.to_str()) else {
+			continue;
+		};
+		match std::fs::read_to_string(&path) {
+			Ok(json) => {
+				builtins::helm_cache_put_json(key.to_owned(), json);
+				loaded += 1;
+			}
+			Err(err) => warn!("helm-cache: failed to read {}: {err:#}", path.display()),
+		}
+	}
+	debug!(
+		"helm-cache: preloaded {loaded} entries from {}",
+		dir.display()
+	);
+	Ok(())
+}
+
+/// Stop recording and persist the entries touched during this run to `dir`.
+///
+/// Best-effort: write failures are logged and ignored so caching never breaks
+/// an otherwise successful export. Must be called once, after the parallel
+/// export loop completes (single-threaded), so writes never race.
+pub fn save(dir: &Path) {
+	let touched: HashSet<String> = builtins::helm_disk_cache_take();
+	if touched.is_empty() {
+		return;
+	}
+
+	if let Err(err) = std::fs::create_dir_all(dir) {
+		warn!(
+			"helm-cache: failed to create {}: {err:#} (not caching)",
+			dir.display()
+		);
+		return;
+	}
+
+	let mut written = 0usize;
+	for key in &touched {
+		let Some(json) = builtins::helm_cache_get_json(key) else {
+			// The entry was evicted or never stored; nothing to persist.
+			continue;
+		};
+		let path = dir.join(format!("{key}.json"));
+		match std::fs::write(&path, json) {
+			Ok(()) => written += 1,
+			Err(err) => warn!("helm-cache: failed to write {}: {err:#}", path.display()),
+		}
+	}
+	debug!("helm-cache: wrote {written} entries to {}", dir.display());
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::jsonnet::evaluator::jrsonnet::builtins::{
+		helm_disk_cache_begin, helm_disk_cache_take, record_helm_disk_touch, HELM_CACHE_TEST_LOCK,
+	};
+
+	#[test]
+	fn test_cache_dir() {
+		assert_eq!(
+			cache_dir(std::path::Path::new("/out")),
+			std::path::PathBuf::from("/out/helm-cache"),
+		);
+	}
+
+	#[test]
+	fn test_save_writes_only_touched_present_entries() {
+		let _guard = HELM_CACHE_TEST_LOCK
+			.lock()
+			.unwrap_or_else(|e| e.into_inner());
+
+		let tmp = tempfile::tempdir().unwrap();
+		let dir = tmp.path().join("helm-cache");
+
+		// Unique keys so other tests' global-cache entries cannot interfere.
+		let touched_present = "save_present_0001";
+		let touched_absent = "save_absent_0001";
+
+		builtins::helm_cache_put_json(touched_present.to_string(), "{\"k\":1}".to_string());
+
+		helm_disk_cache_begin();
+		record_helm_disk_touch(touched_present);
+		// Touched but never stored in the cache: must be skipped, not error.
+		record_helm_disk_touch(touched_absent);
+		save(&dir);
+
+		let present_file = dir.join(format!("{touched_present}.json"));
+		assert_eq!(std::fs::read_to_string(&present_file).unwrap(), "{\"k\":1}");
+		assert!(!dir.join(format!("{touched_absent}.json")).exists());
+	}
+
+	#[test]
+	fn test_save_empty_touched_creates_nothing() {
+		let _guard = HELM_CACHE_TEST_LOCK
+			.lock()
+			.unwrap_or_else(|e| e.into_inner());
+
+		let tmp = tempfile::tempdir().unwrap();
+		let dir = tmp.path().join("helm-cache");
+
+		helm_disk_cache_begin();
+		save(&dir);
+
+		assert!(!dir.exists());
+	}
+
+	#[test]
+	fn test_load_and_clear_preloads_and_removes() {
+		let _guard = HELM_CACHE_TEST_LOCK
+			.lock()
+			.unwrap_or_else(|e| e.into_inner());
+
+		let tmp = tempfile::tempdir().unwrap();
+		let dir = tmp.path().join("helm-cache");
+		std::fs::create_dir_all(&dir).unwrap();
+
+		// A unique key not present in the global cache, plus a non-json file
+		// that must be ignored.
+		let key = "load_key_0001";
+		std::fs::write(dir.join(format!("{key}.json")), "{\"loaded\":true}").unwrap();
+		std::fs::write(dir.join("README.txt"), "ignore me").unwrap();
+		assert_eq!(builtins::helm_cache_get_json(key), None);
+
+		load_and_clear(&dir);
+
+		// Entry is now in the global in-memory cache, and the directory is gone.
+		assert_eq!(
+			builtins::helm_cache_get_json(key),
+			Some("{\"loaded\":true}".to_string())
+		);
+		assert!(!dir.exists());
+
+		// Recording was enabled by load_and_clear; clean it up for other tests.
+		helm_disk_cache_take();
+	}
+
+	#[test]
+	fn test_load_and_clear_missing_dir_is_ok() {
+		let _guard = HELM_CACHE_TEST_LOCK
+			.lock()
+			.unwrap_or_else(|e| e.into_inner());
+
+		let tmp = tempfile::tempdir().unwrap();
+		let dir = tmp.path().join("does-not-exist");
+
+		// Must not panic, and recording is still enabled.
+		load_and_clear(&dir);
+		assert!(!dir.exists());
+
+		helm_disk_cache_take();
+	}
+}

cmds/rtk/src/environments/mod.rs

@@ -18,6 +18,7 @@ use tabwriter::TabWriter;
 
 pub mod discover;
 pub mod export;
+pub mod helm_cache;
 
 use crate::jsonnet::jpath;
 use crate::spec::{Environment, EnvironmentData, Spec};