diff --git a/boulder/src/build.rs b/boulder/src/build.rs index 8a76c3379..dea292aa0 100644 --- a/boulder/src/build.rs +++ b/boulder/src/build.rs @@ -6,7 +6,9 @@ use std::{ io, os::unix::process::ExitStatusExt, path::{Path, PathBuf}, - process, thread, + process, + sync::Mutex, + thread, time::Duration, }; @@ -17,6 +19,8 @@ use nix::{ sys::signal::Signal, unistd::{Pid, getpgrp, setpgid}, }; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; use stone_recipe::{ Script, script::{self, Breakpoint}, @@ -34,6 +38,25 @@ use crate::{ Env, Macros, Paths, Recipe, Timing, architecture::BuildTarget, container, macros, profile, recipe, timing, util, }; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum LockedUpstream { + Git { + uri: String, + #[serde(skip_serializing_if = "Option::is_none")] + tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branch: Option, + rev: String, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct StoneLock { + #[serde(default)] + pub upstreams: Vec, +} + pub struct Builder { pub targets: Vec, pub recipe: Recipe, @@ -42,6 +65,7 @@ pub struct Builder { pub ccache: bool, pub env: Env, profile: profile::Id, + new_lock_data: Mutex>, } pub struct Target { @@ -93,6 +117,7 @@ impl Builder { ccache, env, profile, + new_lock_data: Mutex::new(None), }) } @@ -121,10 +146,14 @@ impl Builder { // Populate rootfs root::populate(self, repos, timing, initialize_timer, update_repos)?; + // Resolve the upstreams + let (resolved_upstreams, new_lock_data) = resolve_upstreams(&self.recipe)?; + *self.new_lock_data.lock().unwrap() = Some(new_lock_data); + let timer = timing.begin(timing::Kind::Fetch); // Sync (fetch & share) upstreams to rootfs - upstream::sync(&self.recipe, &self.paths)?; + upstream::sync(&resolved_upstreams, &self.paths)?; timing.finish(timer); @@ -268,6 +297,155 @@ impl Builder { Ok(()) } + + pub fn write_lock_file(&self) -> Result<(), LockFileError> { + if let Some(lock_data) = self.new_lock_data.lock().unwrap().take() { + let lock_path = self.recipe.path.with_file_name("stone.lock"); + + let header = + "# This file is automatically generated by boulder.\n# It is not intended for manual editing.\n\n"; + let serialized_data = serde_yaml::to_string(&lock_data)?; + let new_lock_content = format!("{header}{serialized_data}"); + + fs::write(&lock_path, new_lock_content)?; + } + Ok(()) + } +} + +fn resolve_git_ref(uri: &url::Url, ref_id: &str) -> Result { + let refs_to_try = [format!("refs/tags/{ref_id}"), format!("refs/heads/{ref_id}")]; + let output = process::Command::new("git") + .args(["ls-remote", "--", uri.as_str()]) + .args(&refs_to_try) + .output()?; + + if !output.status.success() { + return Err(GitError::Failed); + } + let stdout = String::from_utf8(output.stdout)?; + + // git ls-remote output is in the format: \t + // so just grab the first word to get the hash. + stdout + .split_whitespace() + .next() + .ok_or_else(|| GitError::UnresolvedReference { + ref_id: ref_id.to_owned(), + uri: uri.to_string(), + }) + .map(|s| s.to_owned()) +} + +pub fn resolve_upstreams(recipe: &Recipe) -> Result<(Vec, StoneLock), Error> { + let lock_path = recipe.path.with_file_name("stone.lock"); + let existing_lock: StoneLock = if lock_path.exists() { + let content = fs::read_to_string(&lock_path)?; + serde_yaml::from_str(&content).unwrap_or_default() + } else { + Default::default() + }; + + let lock_map: BTreeMap = existing_lock + .upstreams + .iter() + .map(|u| match u { + LockedUpstream::Git { uri, .. } => (uri.clone(), u), + }) + .collect(); + + let mut resolved_upstreams = Vec::new(); + let mut new_locked_upstreams = Vec::new(); + + for upstream in &recipe.parsed.upstreams { + match upstream { + stone_recipe::Upstream::Plain { .. } => { + resolved_upstreams.push(upstream::Upstream::from_recipe(upstream.clone())?); + // Plain upstreams are passed directly through since they always define a hash + } + stone_recipe::Upstream::Git { + uri, tag, branch, rev, .. + } => { + let uri_string = format!("git|{uri}"); + let locked_entry = lock_map.get(&uri_string); + let resolved_rev = match (rev, tag, branch, locked_entry) { + // The stone.yaml directly specifies a rev, so lock file is irrelevant + (Some(r), ..) => r.clone(), + // The stone.yaml specifies a tag that exists in the stone.lock + ( + None, + Some(t), + None, + Some(LockedUpstream::Git { + tag: Some(locked_tag), + rev: locked_rev, + .. + }), + ) if t == locked_tag => { + let current_rev = resolve_git_ref(uri, t)?; + if current_rev != *locked_rev { + eprintln!( + "{} | The tag '{t}' for {uri} now points to a different commit hash.", + "Warning".yellow(), + ); + eprintln!(" Locked: {}", locked_rev.clone().dim()); + eprintln!(" Current: {}", current_rev.dim()); + eprintln!(" Using the locked commit hash for this build to ensure reproducibility."); + } + locked_rev.clone() + } + // The stone.yaml specifies a branch that exists in the stone.lock + ( + None, + None, + Some(b), + Some(LockedUpstream::Git { + branch: Some(locked_branch), + rev: locked_rev, + .. + }), + ) if b == locked_branch => { + let current_rev = resolve_git_ref(uri, b)?; + if current_rev != *locked_rev { + eprintln!( + "{} | The branch '{b}' for {uri} now points to a different commit hash.", + "Warning".yellow(), + ); + eprintln!(" Locked: {}", locked_rev.clone().dim()); + eprintln!(" Current: {}", current_rev.dim()); + eprintln!(" Using the locked commit hash for this build to ensure reproducibility."); + } + locked_rev.clone() + } + // Catch all if the lock file is missing or stale + // This covers: + // - No lock file exists. + // - stone.yml has a tag, but stone.lock has a different tag or a branch. + // - stone.yml has a branch, but stone.lock has a different branch or a tag. + // In all these cases, we resolve the reference from the remote repository. + (None, tag, branch, _) => { + let ref_id = tag.as_deref().or(branch.as_deref()).unwrap(); + resolve_git_ref(uri, ref_id)? + } + }; + resolved_upstreams + .push(upstream::Upstream::from_recipe(upstream.clone())?.with_resolved_rev(resolved_rev.clone())); + // Add a locked_entry if the stone.yaml does not specify a rev + if !rev.is_some() { + new_locked_upstreams.push(LockedUpstream::Git { + uri: uri_string, + tag: tag.clone(), + branch: branch.clone(), + rev: resolved_rev, + }); + } + } + } + } + let new_lock_data = StoneLock { + upstreams: new_locked_upstreams, + }; + Ok((resolved_upstreams, new_lock_data)) } pub fn build_target_prefix(target: BuildTarget, i: usize) -> String { @@ -429,6 +607,41 @@ fn breakpoint_line( }) } +#[derive(Debug, Error)] +pub enum LockFileError { + #[error("failed to serialize lock file")] + Serialize { + #[from] + source: serde_yaml::Error, + }, + #[error("failed to write lock file")] + Write { + #[from] + source: io::Error, + }, +} + +#[derive(Debug, Error)] +pub enum GitError { + #[error("failed to run git command")] + Command { + #[from] + source: io::Error, + }, + + #[error("command failed with non-zero status")] + Failed, + + #[error("output was not valid UTF-8")] + Utf8 { + #[from] + source: std::string::FromUtf8Error, + }, + + #[error("could not resolve '{ref_id}' for git repository '{uri}'")] + UnresolvedReference { ref_id: String, uri: String }, +} + #[derive(Debug, Error)] pub enum Error { #[error("no supported build targets for recipe")] @@ -459,4 +672,8 @@ pub enum Error { Io(#[from] io::Error), #[error("recreate artefacts dir")] RecreateArtefactsDir(#[source] io::Error), + #[error("git")] + Git(#[from] GitError), + #[error("lock file")] + LockFile(#[from] LockFileError), } diff --git a/boulder/src/build/upstream.rs b/boulder/src/build/upstream.rs index 00dd3e7cf..9c95122c4 100644 --- a/boulder/src/build/upstream.rs +++ b/boulder/src/build/upstream.rs @@ -19,25 +19,20 @@ use tokio::io::AsyncWriteExt; use tui::{MultiProgress, ProgressBar, ProgressStyle, Styled}; use url::Url; -use crate::{Paths, Recipe, util}; +use crate::{Paths, util}; -/// Cache all upstreams from the provided [`Recipe`] and make them available +/// Cache all resolved upstreams and make them available /// in the guest rootfs. -pub fn sync(recipe: &Recipe, paths: &Paths) -> Result<(), Error> { - let upstreams = recipe - .parsed - .upstreams - .iter() - .cloned() - .map(Upstream::from_recipe) - .collect::, _>>()?; - +pub fn sync(resolved_upstreams: &[Upstream], paths: &Paths) -> Result<(), Error> { println!(); - println!("Sharing {} upstream(s) with the build container", upstreams.len()); + println!( + "Sharing {} upstream(s) with the build container", + resolved_upstreams.len() + ); let mp = MultiProgress::new(); let tp = mp.add( - ProgressBar::new(upstreams.len() as u64).with_style( + ProgressBar::new(resolved_upstreams.len() as u64).with_style( ProgressStyle::with_template("\n|{bar:20.cyan/blue}| {pos}/{len}") .unwrap() .progress_chars("■≡=- "), @@ -49,7 +44,7 @@ pub fn sync(recipe: &Recipe, paths: &Paths) -> Result<(), Error> { util::ensure_dir_exists(&upstream_dir)?; runtime::block_on( - stream::iter(&upstreams) + stream::iter(resolved_upstreams) .map(|upstream| async { let pb = mp.insert_before( &tp, @@ -158,9 +153,17 @@ impl Upstream { hash: hash.parse()?, rename, })), - stone_recipe::Upstream::Git { - uri, ref_id, staging, .. - } => Ok(Self::Git(Git { uri, ref_id, staging })), + stone_recipe::Upstream::Git { uri, rev, staging, .. } => Ok(Self::Git(Git { uri, rev, staging })), + } + } + + pub fn with_resolved_rev(self, resolved_rev: String) -> Self { + match self { + Upstream::Git(mut git) => { + git.rev = Some(resolved_rev); + Upstream::Git(git) + } + plain => plain, } } @@ -303,7 +306,7 @@ impl Plain { #[derive(Debug, Clone)] pub struct Git { uri: Url, - ref_id: String, + rev: Option, staging: bool, } @@ -398,13 +401,15 @@ impl Git { self.run(&["fetch"], Some(path)).await?; - let result = self.run(&["cat-file", "-e", &self.ref_id], Some(path)).await; + let rev_to_check = self.rev.as_deref().expect("Git rev should be resolved before fetch"); + let result = self.run(&["cat-file", "-e", rev_to_check], Some(path)).await; Ok(result.is_ok()) } async fn reset_to_ref(&self, path: &Path) -> Result<(), Error> { - self.run(&["reset", "--hard", &self.ref_id], Some(path)).await?; + let rev_to_reset = self.rev.as_deref().expect("Git rev should be resolved before fetch"); + self.run(&["reset", "--hard", rev_to_reset], Some(path)).await?; self.run( &[ diff --git a/boulder/src/cli/build.rs b/boulder/src/cli/build.rs index ebed62e58..307577c82 100644 --- a/boulder/src/cli/build.rs +++ b/boulder/src/cli/build.rs @@ -121,6 +121,9 @@ pub fn handle(command: Command, env: Env) -> Result<(), Error> { // Copy artefacts to host recipe dir package::sync_artefacts(paths).map_err(Error::SyncArtefacts)?; + // Write the stone.lock file + builder.write_lock_file()?; + println!( "Build finished successfully at {}", Local::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) @@ -143,4 +146,6 @@ pub enum Error { Container(#[from] container::Error), #[error("setting thread priority")] Priority(#[from] thread_priority::Error), + #[error("lock file")] + LockFile(#[from] build::LockFileError), } diff --git a/crates/stone_recipe/src/lib.rs b/crates/stone_recipe/src/lib.rs index 60cf662fb..a24591002 100644 --- a/crates/stone_recipe/src/lib.rs +++ b/crates/stone_recipe/src/lib.rs @@ -18,6 +18,11 @@ pub mod macros; pub mod script; pub mod tuning; +fn is_valid_commit_hash(s: &str) -> bool { + // git commit hashes can be SHA-1 or SHA-256 hashes + (s.len() == 40 || s.len() == 64) && s.chars().all(|c| c.is_ascii_hexdigit()) +} + pub fn from_slice(bytes: &[u8]) -> Result { serde_yaml::from_slice(bytes) } @@ -125,7 +130,9 @@ pub enum Upstream { }, Git { uri: Url, - ref_id: String, + tag: Option, + branch: Option, + rev: Option, clone_dir: Option, staging: bool, }, @@ -150,8 +157,9 @@ impl<'de> Deserialize<'de> for Upstream { unpack_dir: Option, }, Git { - #[serde(rename = "ref")] - ref_id: String, + tag: Option, + branch: Option, + rev: Option, #[serde(rename = "clonedir")] clone_dir: Option, #[serde(default = "default_true", deserialize_with = "stringy_bool")] @@ -188,6 +196,19 @@ impl<'de> Deserialize<'de> for Upstream { #[error("invalid uri: {0}")] struct UriParseError(#[from] url::ParseError); + // Helper function to validate the rev for git type upstreams to ensure it is a valid commit hash. + fn validate_rev<'de, D>(rev: &str) -> Result<(), D::Error> + where + D: serde::Deserializer<'de>, + { + if !is_valid_commit_hash(rev) { + return Err(serde::de::Error::custom(format!( + "'{rev}' is not a valid git commit hash. If this is a tag or branch name use the corresponding 'tag' or 'branch' keys instead. For example: `tag: {rev}`" + ))); + } + Ok(()) + } + let raw_map = BTreeMap::::deserialize(deserializer)?; match raw_map.into_iter().next() { @@ -199,12 +220,17 @@ impl<'de> Deserialize<'de> for Upstream { unpack: default_true(), unpack_dir: None, }), - Some((Uri::Git(uri), Outer::String(ref_id))) => Ok(Upstream::Git { - uri, - ref_id, - clone_dir: None, - staging: default_true(), - }), + Some((Uri::Git(uri), Outer::String(rev))) => { + validate_rev::(&rev)?; + Ok(Upstream::Git { + uri, + tag: None, + branch: None, + rev: Some(rev), + clone_dir: None, + staging: default_true(), + }) + } Some(( Uri::Plain(uri), Outer::Inner(Inner::Plain { @@ -225,16 +251,34 @@ impl<'de> Deserialize<'de> for Upstream { Some(( Uri::Git(uri), Outer::Inner(Inner::Git { - ref_id, + tag, + branch, + rev, clone_dir, staging, }), - )) => Ok(Upstream::Git { - uri, - ref_id, - clone_dir, - staging, - }), + )) => { + // We prefer using rev, but allow tags and branches + // Regardless of which source is given here, the corresponding commit hash + // will be fetched and stored in the stone.lock file for reproducibility + let git_source_count = tag.is_some() as u32 + branch.is_some() as u32 + rev.is_some() as u32; + if git_source_count != 1 { + return Err(serde::de::Error::custom( + "For a git upstream, you must specify exactly one of: tag, branch, or rev", + )); + } + if let Some(rev_string) = &rev { + validate_rev::(rev_string)?; + } + Ok(Upstream::Git { + uri, + tag, + branch, + rev, + clone_dir, + staging, + }) + } Some((Uri::Plain(_), Outer::Inner(Inner::Git { .. }))) => Err(serde::de::Error::custom( "found git payload but missing 'git|' prefixed URI", )), @@ -385,4 +429,117 @@ mod test { dbg!(&recipe); } } + + #[test] + fn reject_git_tag() { + let input = r#" +name: test-pkg +version: 1.0.0 +release: 1 +license: MIT +homepage: https://example.com +upstreams: + - git|https://github.com/example/repo : v1.0.0 +"#; + let result = from_str(input); + assert!(result.is_err()); + } + + #[test] + fn accept_git_tag() { + let input = r#" +name: test-pkg +version: 1.0.0 +release: 1 +license: MIT +homepage: https://example.com +upstreams: + - git|https://github.com/example/repo: + tag: v1.0.0 +"#; + let result = from_str(input); + assert!(result.is_ok()); + } + + #[test] + fn accept_git_branch() { + let input = r#" +name: test-pkg +version: 1.0.0 +release: 1 +license: MIT +homepage: https://example.com +upstreams: + - git|https://github.com/example/repo: + branch: main +"#; + let result = from_str(input); + assert!(result.is_ok()); + } + + #[test] + fn accept_git_rev() { + let input = r#" +name: test-pkg +version: 1.0.0 +release: 1 +license: MIT +homepage: https://example.com +upstreams: + - git|https://github.com/example/repo: + rev: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3 +"#; + let result = from_str(input); + assert!(result.is_ok()); + } + + #[test] + fn reject_git_rev() { + let input = r#" +name: test-pkg +version: 1.0.0 +release: 1 +license: MIT +homepage: https://example.com +upstreams: + - git|https://github.com/example/repo: + rev: v1.0.0 +"#; + let result = from_str(input); + assert!(result.is_err()); + } + + #[test] + fn reject_git_rev_with_tag() { + let input = r#" +name: test-pkg +version: 1.0.0 +release: 1 +license: MIT +homepage: https://example.com +upstreams: + - git|https://github.com/example/repo: + tag: v1.0.0 + rev: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3 +"#; + let result = from_str(input); + assert!(result.is_err()); + } + + #[test] + fn reject_git_rev_with_branch() { + let input = r#" +name: test-pkg +version: 1.0.0 +release: 1 +license: MIT +homepage: https://example.com +upstreams: + - git|https://github.com/example/repo: + branch: main + rev: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3 +"#; + let result = from_str(input); + assert!(result.is_err()); + } } diff --git a/test/boulder-stone.yml b/test/boulder-stone.yml index 9290f143a..c75c686c1 100644 --- a/test/boulder-stone.yml +++ b/test/boulder-stone.yml @@ -8,7 +8,8 @@ description : | Extremely flexible and powerful, yet simple to use, package build tool for the Serpent OS project. upstreams : - - git|https://github.com/serpent-os/boulder : v1.0.1 + - git|https://github.com/serpent-os/boulder: + tag: v1.0.1 - https://github.com/serpent-os/libmoss/releases/download/v1.2.0/libmoss-1.2.0.tar.xz: hash: cbf684b5a37a3a433e0526beb04a7b3419b71e81b3709c3b0d7ed6a1987d3dcb unpackdir: libmoss